Fixes after feedback

apps/api/src/app/admin/admin.controller.ts

@@ -227,8 +227,8 @@ export class AdminController {
     @Param('symbol') symbol: string
   ): Promise<{ price: number }> {
     try {
-      const config = JSON.parse(data.scraperConfiguration);
+      const scraperConfiguration = JSON.parse(data.scraperConfiguration);
-      const price = await this.manualService.test(config);
+      const price = await this.manualService.test(scraperConfiguration);
 
       if (price) {
         return { price };

apps/api/src/services/data-provider/manual/manual.service.ts

@@ -222,11 +222,11 @@ export class ManualService implements DataProviderInterface {
     return { items };
   }
 
-  public async test(config: ScraperConfiguration) {
+  public async test(scraperConfiguration: ScraperConfiguration) {
-    return this.scrape(config);
+    return this.scrape(scraperConfiguration);
   }
 
-  private async scrape(config: ScraperConfiguration): Promise<number> {
+  private async scrape(scraperConfiguration: ScraperConfiguration): Promise<number> {
     try {
       const abortController = new AbortController();
 
@@ -234,18 +234,18 @@ export class ManualService implements DataProviderInterface {
         abortController.abort();
       }, this.configurationService.get('REQUEST_TIMEOUT'));
 
-      const { body } = await got(config.url, {
+      const { body, headers } = await got(scraperConfiguration.url, {
-        headers: config.headers as Headers,
+        headers: scraperConfiguration.headers as Headers,
         // @ts-ignore
         signal: abortController.signal
       });
-      if (config.type === 'json') {
+      if (headers["content-type"] === "application/json") {
         const data = JSON.parse(body);
-        const field = String(jsonpath.query(data, config.selector)[0]);
+        const field = String(jsonpath.query(data, scraperConfiguration.selector)[0]);
         return extractNumberFromString(field);
       } else {
         const $ = cheerio.load(body);
-        return extractNumberFromString($(config.selector).first().text());
+        return extractNumberFromString($(scraperConfiguration.selector).first().text());
       }
     } catch (error) {
       throw error;

apps/api/src/services/symbol-profile/symbol-profile.service.ts

@@ -204,8 +204,6 @@ export class SymbolProfileService {
           scraperConfiguration.headers as ScraperConfiguration['headers'],
         selector: scraperConfiguration.selector as string,
         url: scraperConfiguration.url as string,
-        type: (scraperConfiguration.type ??
-          'html') as ScraperConfiguration['type']
       };
     }
 

libs/common/src/lib/helper.ts

@@ -124,8 +124,7 @@ export function extractNumberFromString(aString: string): number {
   try {
     const [numberString] = aString.match(NUMERIC_REGEXP);
     return parseFloat(numberString.trim());
-  } catch (err) {
+  } catch {
-    console.error(err);
     return undefined;
   }
 }

libs/common/src/lib/interfaces/scraper-configuration.interface.ts

@@ -3,5 +3,4 @@ export interface ScraperConfiguration {
   headers?: { [key: string]: string };
   selector: string;
   url: string;
-  type: 'html' | 'json';
 }

package.json

@@ -110,7 +110,7 @@
     "helmet": "7.0.0",
     "http-status-codes": "2.3.0",
     "ionicons": "7.1.0",
-    "jsonpath": "^1.1.1",
+    "jsonpath": "1.1.1",
     "lodash": "4.17.21",
     "marked": "9.1.6",
     "ms": "3.0.0-canary.1",

yarn.lock

@@ -14262,7 +14262,7 @@ jsonparse@^1.3.1:
   resolved "https://registry.yarnpkg.com/jsonparse/-/jsonparse-1.3.1.tgz#3f4dae4a91fac315f71062f8521cc239f1366280"
   integrity sha512-POQXvpdL69+CluYsillJ7SUhKvytYjW9vG/GKpnf+xP8UWgYEM/RaMzHHofbALDiKbbP1W8UEYmgGl39WkPZsg==
 
-jsonpath@^1.1.1:
+jsonpath@1.1.1:
   version "1.1.1"
   resolved "https://registry.yarnpkg.com/jsonpath/-/jsonpath-1.1.1.tgz#0ca1ed8fb65bb3309248cc9d5466d12d5b0b9901"
   integrity sha512-l6Cg7jRpixfbgoWgkrl77dgEj8RPvND0wMH6TwQmi9Qs4TFfS9u5cUFnbeKTwj5ga5Y3BTGGNI28k117LJ009w==