Feature/split scraper configuration into sub form (#4157)

* Split scraper configuration into sub form

* Update changelog

CHANGELOG.md

@@ -13,6 +13,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Changed
 
+- Extracted the scraper configuration to a sub form in the asset profile details dialog of the admin control
 - Migrated the database seeding to _TypeScript_
 - Improved the language localization for German (`de`)
 - Upgraded `@trivago/prettier-plugin-sort-imports` from version `4.3.0` to `5.2.1`

apps/client/src/app/components/admin-market-data/asset-profile-dialog/asset-profile-dialog.component.scss

@@ -7,5 +7,21 @@
     gf-line-chart {
       aspect-ratio: 16/9;
     }
+
+    .mat-expansion-panel {
+      --mat-expansion-container-background-color: transparent;
+
+      ::ng-deep {
+        .mat-expansion-panel-body {
+          padding: 0;
+        }
+      }
+
+      .mat-expansion-panel-header {
+        &:hover {
+          --mat-expansion-header-hover-state-layer-color: transparent;
+        }
+      }
+    }
   }
 }

apps/client/src/app/components/admin-market-data/asset-profile-dialog/asset-profile-dialog.component.ts

@@ -21,7 +21,8 @@ import {
   Component,
   Inject,
   OnDestroy,
-  OnInit
+  OnInit,
+  signal
 } from '@angular/core';
 import { FormBuilder, FormControl, Validators } from '@angular/forms';
 import { MAT_DIALOG_DATA, MatDialogRef } from '@angular/material/dialog';
@@ -64,7 +65,14 @@ export class AssetProfileDialog implements OnDestroy, OnInit {
       csvString: ''
     }),
     name: ['', Validators.required],
-    scraperConfiguration: '',
+    scraperConfiguration: this.formBuilder.group({
+      defaultMarketPrice: null,
+      headers: JSON.stringify({}),
+      locale: '',
+      mode: '',
+      selector: '',
+      url: ''
+    }),
     sectors: '',
     symbolMapping: '',
     url: ''
@@ -79,6 +87,11 @@ export class AssetProfileDialog implements OnDestroy, OnInit {
   public historicalDataItems: LineChartItem[];
   public isBenchmark = false;
   public marketDataItems: MarketData[] = [];
+  public modeValues = [
+    { value: 'lazy', viewValue: $localize`Lazy` },
+    { value: 'instant', viewValue: $localize`Instant` }
+  ];
+  public scraperConfiguationIsExpanded = signal(false);
   public sectors: {
     [name: string]: { name: string; value: number };
   };
@@ -181,9 +194,18 @@ export class AssetProfileDialog implements OnDestroy, OnInit {
             csvString: AssetProfileDialog.HISTORICAL_DATA_TEMPLATE
           },
           name: this.assetProfile.name ?? this.assetProfile.symbol,
-          scraperConfiguration: JSON.stringify(
-            this.assetProfile?.scraperConfiguration ?? {}
-          ),
+          scraperConfiguration: {
+            defaultMarketPrice:
+              this.assetProfile?.scraperConfiguration?.defaultMarketPrice ??
+              null,
+            headers: JSON.stringify(
+              this.assetProfile?.scraperConfiguration?.headers ?? {}
+            ),
+            locale: this.assetProfile?.scraperConfiguration?.locale ?? '',
+            mode: this.assetProfile?.scraperConfiguration?.mode ?? 'lazy',
+            selector: this.assetProfile?.scraperConfiguration?.selector ?? '',
+            url: this.assetProfile?.scraperConfiguration?.url ?? ''
+          },
           sectors: JSON.stringify(this.assetProfile?.sectors ?? []),
           symbolMapping: JSON.stringify(this.assetProfile?.symbolMapping ?? {}),
           url: this.assetProfile?.url ?? ''
@@ -252,9 +274,31 @@ export class AssetProfileDialog implements OnDestroy, OnInit {
     } catch {}
 
     try {
-      scraperConfiguration = JSON.parse(
-        this.assetProfileForm.get('scraperConfiguration').value
-      );
+      scraperConfiguration = {
+        defaultMarketPrice:
+          this.assetProfileForm.controls['scraperConfiguration'].controls[
+            'defaultMarketPrice'
+          ].value,
+        headers: JSON.parse(
+          this.assetProfileForm.controls['scraperConfiguration'].controls[
+            'headers'
+          ].value
+        ),
+        locale:
+          this.assetProfileForm.controls['scraperConfiguration'].controls[
+            'locale'
+          ].value,
+        mode: this.assetProfileForm.controls['scraperConfiguration'].controls[
+          'mode'
+        ].value,
+        selector:
+          this.assetProfileForm.controls['scraperConfiguration'].controls[
+            'selector'
+          ].value,
+        url: this.assetProfileForm.controls['scraperConfiguration'].controls[
+          'url'
+        ].value
+      };
     } catch {}
 
     try {
@@ -306,8 +350,31 @@ export class AssetProfileDialog implements OnDestroy, OnInit {
     this.adminService
       .testMarketData({
         dataSource: this.data.dataSource,
-        scraperConfiguration: this.assetProfileForm.get('scraperConfiguration')
-          .value,
+        scraperConfiguration: JSON.stringify({
+          defaultMarketPrice:
+            this.assetProfileForm.controls['scraperConfiguration'].controls[
+              'defaultMarketPrice'
+            ].value,
+          headers: JSON.parse(
+            this.assetProfileForm.controls['scraperConfiguration'].controls[
+              'headers'
+            ].value
+          ),
+          locale:
+            this.assetProfileForm.controls['scraperConfiguration'].controls[
+              'locale'
+            ].value,
+          mode: this.assetProfileForm.controls['scraperConfiguration'].controls[
+            'mode'
+          ].value,
+          selector:
+            this.assetProfileForm.controls['scraperConfiguration'].controls[
+              'selector'
+            ].value,
+          url: this.assetProfileForm.controls['scraperConfiguration'].controls[
+            'url'
+          ].value
+        }),
         symbol: this.data.symbol
       })
       .pipe(

apps/client/src/app/components/admin-market-data/asset-profile-dialog/asset-profile-dialog.html

@@ -278,31 +278,106 @@
       </mat-form-field>
     </div>
     @if (assetProfile?.dataSource === 'MANUAL') {
-      <div>
-        <mat-form-field appearance="outline" class="w-100">
-          <mat-label i18n>Scraper Configuration</mat-label>
-          <div class="align-items-end d-flex">
-            <textarea
-              cdkTextareaAutosize
-              formControlName="scraperConfiguration"
-              matInput
-              type="text"
-              (keyup.enter)="$event.stopPropagation()"
-            ></textarea>
-            <button
-              color="accent"
-              mat-flat-button
-              type="button"
-              [disabled]="
-                assetProfileForm.get('scraperConfiguration').value === '{}'
-              "
-              (click)="onTestMarketData()"
-            >
-              <ng-container i18n>Test</ng-container>
-            </button>
-          </div>
-        </mat-form-field>
+      <div class="mb-3">
+        <mat-accordion class="my-3">
+          <mat-expansion-panel
+            class="shadow-none"
+            [expanded]="
+              assetProfileForm.controls.scraperConfiguration.controls.selector
+                .value !== '' &&
+              assetProfileForm.controls.scraperConfiguration.controls.url
+                .value !== ''
+            "
+            (closed)="scraperConfiguationIsExpanded.set(false)"
+            (opened)="scraperConfiguationIsExpanded.set(true)"
+          >
+            <mat-expansion-panel-header class="p-0">
+              <mat-panel-title i18n>Scraper Configuration</mat-panel-title>
+            </mat-expansion-panel-header>
+            <div formGroupName="scraperConfiguration">
+              <div class="mt-3">
+                <mat-form-field appearance="outline" class="w-100 without-hint">
+                  <mat-label i18n>Default Market Price</mat-label>
+                  <input
+                    formControlName="defaultMarketPrice"
+                    matInput
+                    type="number"
+                  />
+                </mat-form-field>
+              </div>
+              <div class="mt-3">
+                <mat-form-field appearance="outline" class="w-100 without-hint">
+                  <mat-label i18n>Headers</mat-label>
+                  <textarea
+                    cdkTextareaAutosize
+                    formControlName="headers"
+                    matInput
+                    type="text"
+                    [matAutocomplete]="auto"
+                  ></textarea>
+                </mat-form-field>
+              </div>
+              <div class="mt-3">
+                <mat-form-field appearance="outline" class="w-100 without-hint">
+                  <mat-label i18n>Locale</mat-label>
+                  <input formControlName="locale" matInput type="text" />
+                </mat-form-field>
+              </div>
+              <div class="mt-3">
+                <mat-form-field appearance="outline" class="w-100 without-hint">
+                  <mat-label i18n>Mode</mat-label>
+                  <mat-select formControlName="mode">
+                    @for (modeValue of modeValues; track modeValue) {
+                      <mat-option [value]="modeValue.value">{{
+                        modeValue.viewValue
+                      }}</mat-option>
+                    }
+                  </mat-select>
+                </mat-form-field>
+              </div>
+              <div class="mt-3">
+                <mat-form-field appearance="outline" class="w-100 without-hint">
+                  <mat-label>
+                    <ng-container i18n>Selector</ng-container>*
+                  </mat-label>
+                  <textarea
+                    cdkTextareaAutosize
+                    formControlName="selector"
+                    matInput
+                    type="text"
+                  ></textarea>
+                </mat-form-field>
+              </div>
+              <div class="mt-3">
+                <mat-form-field appearance="outline" class="w-100 without-hint">
+                  <mat-label>
+                    <ng-container i18n>Url</ng-container>*
+                  </mat-label>
+                  <input formControlName="url" matInput type="text" />
+                </mat-form-field>
+              </div>
+              <div class="my-3 text-right">
+                <button
+                  color="accent"
+                  mat-flat-button
+                  type="button"
+                  [disabled]="
+                    assetProfileForm.controls.scraperConfiguration.controls
+                      .selector.value === '' ||
+                    assetProfileForm.controls.scraperConfiguration.controls.url
+                      .value === ''
+                  "
+                  (click)="onTestMarketData()"
+                >
+                  <ng-container i18n>Test</ng-container>
+                </button>
+              </div>
+            </div>
+          </mat-expansion-panel>
+        </mat-accordion>
       </div>
+    }
+    @if (assetProfile?.dataSource === 'MANUAL') {
       <div>
         <mat-form-field appearance="outline" class="w-100">
           <mat-label i18n>Sectors</mat-label>

apps/client/src/app/components/admin-market-data/asset-profile-dialog/asset-profile-dialog.module.ts

@@ -13,6 +13,7 @@ import { FormsModule, ReactiveFormsModule } from '@angular/forms';
 import { MatButtonModule } from '@angular/material/button';
 import { MatCheckboxModule } from '@angular/material/checkbox';
 import { MatDialogModule } from '@angular/material/dialog';
+import { MatExpansionModule } from '@angular/material/expansion';
 import { MatInputModule } from '@angular/material/input';
 import { MatMenuModule } from '@angular/material/menu';
 import { MatSelectModule } from '@angular/material/select';
@@ -34,6 +35,7 @@ import { AssetProfileDialog } from './asset-profile-dialog.component';
     MatButtonModule,
     MatCheckboxModule,
     MatDialogModule,
+    MatExpansionModule,
     MatInputModule,
     MatMenuModule,
     MatSelectModule,