From 8f9c8f70068ee82ef7f593b584198440386cab86 Mon Sep 17 00:00:00 2001
From: Amandee Ellawala <amamandee.manushika.e@gmail.com>
Date: Mon, 30 Dec 2024 11:06:08 +0530
Subject: [PATCH] Feature/Split scraper configuration into sub form #3729

---
 .../asset-profile-dialog.component.ts         |  86 ++++++++++--
 .../asset-profile-dialog.html                 | 122 ++++++++++++++----
 .../asset-profile-dialog.module.ts            |   2 +
 3 files changed, 175 insertions(+), 35 deletions(-)

diff --git a/apps/client/src/app/components/admin-market-data/asset-profile-dialog/asset-profile-dialog.component.ts b/apps/client/src/app/components/admin-market-data/asset-profile-dialog/asset-profile-dialog.component.ts
index a144bae87..53ad731b1 100644
--- a/apps/client/src/app/components/admin-market-data/asset-profile-dialog/asset-profile-dialog.component.ts
+++ b/apps/client/src/app/components/admin-market-data/asset-profile-dialog/asset-profile-dialog.component.ts
@@ -21,7 +21,8 @@ import {
   Component,
   Inject,
   OnDestroy,
-  OnInit
+  OnInit,
+  signal
 } from '@angular/core';
 import { FormBuilder, FormControl, Validators } from '@angular/forms';
 import { MAT_DIALOG_DATA, MatDialogRef } from '@angular/material/dialog';
@@ -64,7 +65,14 @@ export class AssetProfileDialog implements OnDestroy, OnInit {
       csvString: ''
     }),
     name: ['', Validators.required],
-    scraperConfiguration: '',
+    scraperConfiguration: this.formBuilder.group({
+      defaultMarketPrice: 0,
+      headers: '',
+      locale: '',
+      mode: '',
+      selector: '',
+      url: ''
+    }),
     sectors: '',
     symbolMapping: '',
     url: ''
@@ -79,9 +87,14 @@ export class AssetProfileDialog implements OnDestroy, OnInit {
   public historicalDataItems: LineChartItem[];
   public isBenchmark = false;
   public marketDataItems: MarketData[] = [];
+  public modeValues = [
+    { value: 'lazy', viewValue: 'Lazy' },
+    { value: 'instant', viewValue: 'Instant' }
+  ];
   public sectors: {
     [name: string]: { name: string; value: number };
   };
+  readonly panelOpenState = signal(false);
   public user: User;
 
   private static readonly HISTORICAL_DATA_TEMPLATE = `date;marketPrice\n${format(
@@ -181,9 +194,17 @@ export class AssetProfileDialog implements OnDestroy, OnInit {
             csvString: AssetProfileDialog.HISTORICAL_DATA_TEMPLATE
           },
           name: this.assetProfile.name ?? this.assetProfile.symbol,
-          scraperConfiguration: JSON.stringify(
-            this.assetProfile?.scraperConfiguration ?? {}
-          ),
+          scraperConfiguration: {
+            defaultMarketPrice:
+              this.assetProfile?.scraperConfiguration.defaultMarketPrice ?? 0,
+            headers: JSON.stringify(
+              this.assetProfile?.scraperConfiguration.headers ?? {}
+            ),
+            locale: this.assetProfile?.scraperConfiguration.locale ?? '',
+            mode: this.assetProfile?.scraperConfiguration.mode ?? '',
+            selector: this.assetProfile?.scraperConfiguration.selector ?? '',
+            url: this.assetProfile?.scraperConfiguration.url ?? ''
+          },
           sectors: JSON.stringify(this.assetProfile?.sectors ?? []),
           symbolMapping: JSON.stringify(this.assetProfile?.symbolMapping ?? {}),
           url: this.assetProfile?.url ?? ''
@@ -252,9 +273,31 @@ export class AssetProfileDialog implements OnDestroy, OnInit {
     } catch {}
 
     try {
-      scraperConfiguration = JSON.parse(
-        this.assetProfileForm.get('scraperConfiguration').value
-      );
+      scraperConfiguration = {
+        defaultMarketPrice:
+          this.assetProfileForm.controls['scraperConfiguration'].controls[
+            'defaultMarketPrice'
+          ].value,
+        headers: JSON.parse(
+          this.assetProfileForm.controls['scraperConfiguration'].controls[
+            'headers'
+          ].value
+        ),
+        locale:
+          this.assetProfileForm.controls['scraperConfiguration'].controls[
+            'locale'
+          ].value,
+        mode: this.assetProfileForm.controls['scraperConfiguration'].controls[
+          'mode'
+        ].value,
+        selector:
+          this.assetProfileForm.controls['scraperConfiguration'].controls[
+            'selector'
+          ].value,
+        url: this.assetProfileForm.controls['scraperConfiguration'].controls[
+          'url'
+        ].value
+      };
     } catch {}
 
     try {
@@ -306,8 +349,31 @@ export class AssetProfileDialog implements OnDestroy, OnInit {
     this.adminService
       .testMarketData({
         dataSource: this.data.dataSource,
-        scraperConfiguration: this.assetProfileForm.get('scraperConfiguration')
-          .value,
+        scraperConfiguration: JSON.stringify({
+          defaultMarketPrice:
+            this.assetProfileForm.controls['scraperConfiguration'].controls[
+              'defaultMarketPrice'
+            ].value,
+          headers: JSON.parse(
+            this.assetProfileForm.controls['scraperConfiguration'].controls[
+              'headers'
+            ].value
+          ),
+          locale:
+            this.assetProfileForm.controls['scraperConfiguration'].controls[
+              'locale'
+            ].value,
+          mode: this.assetProfileForm.controls['scraperConfiguration'].controls[
+            'mode'
+          ].value,
+          selector:
+            this.assetProfileForm.controls['scraperConfiguration'].controls[
+              'selector'
+            ].value,
+          url: this.assetProfileForm.controls['scraperConfiguration'].controls[
+            'url'
+          ].value
+        }),
         symbol: this.data.symbol
       })
       .pipe(
diff --git a/apps/client/src/app/components/admin-market-data/asset-profile-dialog/asset-profile-dialog.html b/apps/client/src/app/components/admin-market-data/asset-profile-dialog/asset-profile-dialog.html
index eeb43e932..93311ec33 100644
--- a/apps/client/src/app/components/admin-market-data/asset-profile-dialog/asset-profile-dialog.html
+++ b/apps/client/src/app/components/admin-market-data/asset-profile-dialog/asset-profile-dialog.html
@@ -277,32 +277,104 @@
         ></textarea>
       </mat-form-field>
     </div>
+    <div class="mb-3">
+      <mat-accordion class="my-3">
+        <mat-expansion-panel
+          (closed)="panelOpenState.set(false)"
+          (opened)="panelOpenState.set(true)"
+        >
+          <mat-expansion-panel-header>
+            <mat-panel-title> Scraper Configuration </mat-panel-title>
+          </mat-expansion-panel-header>
+          @if (assetProfile?.dataSource === 'MANUAL') {
+            <div formGroupName="scraperConfiguration">
+              <div class="mt-3">
+                <mat-form-field appearance="outline" class="w-100 without-hint">
+                  <mat-label i18n>Default Market Price</mat-label>
+                  <input
+                    formControlName="defaultMarketPrice"
+                    matInput
+                    type="number"
+                  />
+                  <div class="align-items-end d-flex"></div>
+                </mat-form-field>
+              </div>
+              <div class="mt-3">
+                <mat-form-field appearance="outline" class="w-100 without-hint">
+                  <mat-label i18n>Headers</mat-label>
+                  <div class="align-items-end d-flex">
+                    <textarea
+                      cdkTextareaAutosize
+                      formControlName="headers"
+                      matInput
+                      type="text"
+                      [matAutocomplete]="auto"
+                    ></textarea>
+                  </div>
+                </mat-form-field>
+              </div>
+              <div class="mt-3">
+                <mat-form-field appearance="outline" class="w-100 without-hint">
+                  <mat-label i18n>Locale</mat-label>
+                  <div class="align-items-end d-flex">
+                    <input formControlName="locale" matInput type="text" />
+                  </div>
+                </mat-form-field>
+              </div>
+              <div class="mt-3">
+                <mat-form-field appearance="outline" class="w-100 without-hint">
+                  <mat-label i18n>Mode</mat-label>
+                  <mat-select formControlName="mode">
+                    <mat-option [value]="null" />
+                    @for (modeVal of modeValues; track modeVal) {
+                      <mat-option [value]="modeVal.value">{{
+                        modeVal.viewValue
+                      }}</mat-option>
+                    }
+                  </mat-select>
+                </mat-form-field>
+              </div>
+              <div class="mt-3">
+                <mat-form-field appearance="outline" class="w-100 without-hint">
+                  <mat-label i18n>Selector</mat-label>
+                  <div class="align-items-end d-flex">
+                    <textarea
+                      cdkTextareaAutosize
+                      formControlName="selector"
+                      matInput
+                      type="text"
+                      (keyup.enter)="$event.stopPropagation()"
+                    ></textarea>
+                  </div>
+                </mat-form-field>
+              </div>
+              <div class="mt-3">
+                <mat-form-field appearance="outline" class="w-100 without-hint">
+                  <mat-label i18n>URL</mat-label>
+                  <div class="align-items-end d-flex">
+                    <input formControlName="url" matInput type="text" />
+                  </div>
+                </mat-form-field>
+              </div>
+              <div class="m-3">
+                <button
+                  color="accent"
+                  mat-flat-button
+                  type="button"
+                  [disabled]="
+                    assetProfileForm.get('scraperConfiguration').value === '{}'
+                  "
+                  (click)="onTestMarketData()"
+                >
+                  <ng-container i18n>Test</ng-container>
+                </button>
+              </div>
+            </div>
+          }
+        </mat-expansion-panel>
+      </mat-accordion>
+    </div>
     @if (assetProfile?.dataSource === 'MANUAL') {
-      <div>
-        <mat-form-field appearance="outline" class="w-100">
-          <mat-label i18n>Scraper Configuration</mat-label>
-          <div class="align-items-end d-flex">
-            <textarea
-              cdkTextareaAutosize
-              formControlName="scraperConfiguration"
-              matInput
-              type="text"
-              (keyup.enter)="$event.stopPropagation()"
-            ></textarea>
-            <button
-              color="accent"
-              mat-flat-button
-              type="button"
-              [disabled]="
-                assetProfileForm.get('scraperConfiguration').value === '{}'
-              "
-              (click)="onTestMarketData()"
-            >
-              <ng-container i18n>Test</ng-container>
-            </button>
-          </div>
-        </mat-form-field>
-      </div>
       <div>
         <mat-form-field appearance="outline" class="w-100">
           <mat-label i18n>Sectors</mat-label>
diff --git a/apps/client/src/app/components/admin-market-data/asset-profile-dialog/asset-profile-dialog.module.ts b/apps/client/src/app/components/admin-market-data/asset-profile-dialog/asset-profile-dialog.module.ts
index d5e14ecb5..9b9876dbc 100644
--- a/apps/client/src/app/components/admin-market-data/asset-profile-dialog/asset-profile-dialog.module.ts
+++ b/apps/client/src/app/components/admin-market-data/asset-profile-dialog/asset-profile-dialog.module.ts
@@ -13,6 +13,7 @@ import { FormsModule, ReactiveFormsModule } from '@angular/forms';
 import { MatButtonModule } from '@angular/material/button';
 import { MatCheckboxModule } from '@angular/material/checkbox';
 import { MatDialogModule } from '@angular/material/dialog';
+import { MatExpansionModule } from '@angular/material/expansion';
 import { MatInputModule } from '@angular/material/input';
 import { MatMenuModule } from '@angular/material/menu';
 import { MatSelectModule } from '@angular/material/select';
@@ -34,6 +35,7 @@ import { AssetProfileDialog } from './asset-profile-dialog.component';
     MatButtonModule,
     MatCheckboxModule,
     MatDialogModule,
+    MatExpansionModule,
     MatInputModule,
     MatMenuModule,
     MatSelectModule,