Harden validation of market data in activities import

1 week ago · f8710ef7e7
10 changed files with 40 additions and 67 deletions
--- a/apps/api/src/app/import/import.service.ts
+++ b/apps/api/src/app/import/import.service.ts
@ -628,11 +628,12 @@ export class ImportService {
          // Insert or update market data
          const marketDataObjects = (
            assetProfileWithMarketData.marketData ?? []
-          ).map((marketData) => {
+          ).map(({ date, marketPrice }) => {
            return {
-              ...marketData,
+              marketPrice,
              symbol,
-              dataSource: assetProfileWithMarketData.dataSource
+              dataSource: assetProfileWithMarketData.dataSource,
+              date: parseISO(date)
            } as Prisma.MarketDataUpdateInput;
          });

--- a/apps/api/src/services/market-data/market-data.service.ts
+++ b/apps/api/src/services/market-data/market-data.service.ts
@ -2,8 +2,7 @@ import { DateQuery } from '@ghostfolio/api/app/portfolio/interfaces/date-query.i
 import { DataGatheringItem } from '@ghostfolio/api/services/interfaces/interfaces';
 import { PrismaService } from '@ghostfolio/api/services/prisma/prisma.service';
 import { DEFAULT_PROCESSOR_GATHER_HISTORICAL_MARKET_DATA_TIMEOUT } from '@ghostfolio/common/config';
-import { UpdateMarketDataDto } from '@ghostfolio/common/dtos';
-import { resetHours } from '@ghostfolio/common/helper';
+import { getStartOfUtcDate } from '@ghostfolio/common/helper';
 import { AssetProfileIdentifier } from '@ghostfolio/common/interfaces';

 import { Injectable } from '@nestjs/common';
@ -36,7 +35,7 @@ export class MarketDataService {
      where: {
        dataSource,
        symbol,
-        date: resetHours(date)
+        date: getStartOfUtcDate(date)
      }
    });
  }
@ -156,14 +155,24 @@ export class MarketDataService {
    dataSource,
    symbol
  }: AssetProfileIdentifier & { data: Prisma.MarketDataUpdateInput[] }) {
+    const marketDataItems = data.map(({ date, marketPrice, state }) => {
+      return {
+        dataSource,
+        symbol,
+        date: getStartOfUtcDate(date as Date),
+        marketPrice: marketPrice as number,
+        state: state as MarketDataState
+      };
+    });
+
    await this.prismaService.$transaction(
      async (prisma) => {
-        if (data.length > 0) {
+        if (marketDataItems.length > 0) {
          let minTime = Infinity;
          let maxTime = -Infinity;

-          for (const { date } of data) {
-            const time = (date as Date).getTime();
+          for (const { date } of marketDataItems) {
+            const time = date.getTime();

            if (time < minTime) {
              minTime = time;
@ -189,13 +198,7 @@ export class MarketDataService {
          });

          await prisma.marketData.createMany({
-            data: data.map(({ date, marketPrice, state }) => ({
-              dataSource,
-              symbol,
-              date: date as Date,
-              marketPrice: marketPrice as number,
-              state: state as MarketDataState
-            })),
+            data: marketDataItems,
            skipDuplicates: true
          });
        }
@ -220,27 +223,6 @@ export class MarketDataService {
    });
  }

-  public async updateMarketData(params: {
-    data: {
-      state: MarketDataState;
-    } & UpdateMarketDataDto;
-    where: Prisma.MarketDataWhereUniqueInput;
-  }): Promise<MarketData> {
-    const { data, where } = params;
-
-    return this.prismaService.marketData.upsert({
-      where,
-      create: {
-        dataSource: where.dataSource_date_symbol.dataSource,
-        date: where.dataSource_date_symbol.date,
-        marketPrice: data.marketPrice,
-        state: data.state,
-        symbol: where.dataSource_date_symbol.symbol
-      },
-      update: { marketPrice: data.marketPrice, state: data.state }
-    });
-  }
-
  /**
   * Upsert market data by imitating missing upsertMany functionality
   * with $transaction
@ -252,10 +234,12 @@ export class MarketDataService {
  }): Promise<MarketData[]> {
    const upsertPromises = data.map(
      ({ dataSource, date, marketPrice, symbol, state }) => {
+        const dateOfMarketData = getStartOfUtcDate(date as Date);
+
        return this.prismaService.marketData.upsert({
          create: {
            dataSource: dataSource as DataSource,
-            date: date as Date,
+            date: dateOfMarketData,
            marketPrice: marketPrice as number,
            state: state as MarketDataState,
            symbol: symbol as string
@ -267,7 +251,7 @@ export class MarketDataService {
          where: {
            dataSource_date_symbol: {
              dataSource: dataSource as DataSource,
-              date: date as Date,
+              date: dateOfMarketData,
              symbol: symbol as string
            }
          }
--- a/libs/common/src/lib/dtos/index.ts
+++ b/libs/common/src/lib/dtos/index.ts
@ -23,7 +23,6 @@ import { UpdateAccountDto } from './update-account.dto';
 import { UpdateAssetProfileDataDto } from './update-asset-profile-data.dto';
 import { UpdateAssetProfileDto } from './update-asset-profile.dto';
 import { UpdateBulkMarketDataDto } from './update-bulk-market-data.dto';
-import { UpdateMarketDataDto } from './update-market-data.dto';
 import { UpdateOrderDto } from './update-order.dto';
 import { UpdateOwnAccessTokenDto } from './update-own-access-token.dto';
 import { UpdatePlatformDto } from './update-platform.dto';
@ -57,7 +56,6 @@ export {
  UpdateAssetProfileDataDto,
  UpdateAssetProfileDto,
  UpdateBulkMarketDataDto,
-  UpdateMarketDataDto,
  UpdateOrderDto,
  UpdateOwnAccessTokenDto,
  UpdatePlatformDto,
--- a/libs/common/src/lib/dtos/market-data.dto.ts
+++ b/libs/common/src/lib/dtos/market-data.dto.ts
@ -1,9 +1,13 @@
-import { IsISO8601, IsNumber } from 'class-validator';
+import { IsAfter1970Constraint } from '@ghostfolio/common/validator-constraints/is-after-1970';
+
+import { IsISO8601, IsNumber, Min, Validate } from 'class-validator';

 export class MarketDataDto {
-  @IsISO8601()
+  @IsISO8601({ strict: true, strictSeparator: true })
+  @Validate(IsAfter1970Constraint)
  date: string;

  @IsNumber()
+  @Min(0)
  marketPrice: number;
 }
--- a/libs/common/src/lib/dtos/update-bulk-market-data.dto.ts
+++ b/libs/common/src/lib/dtos/update-bulk-market-data.dto.ts
@ -1,11 +1,12 @@
-import { UpdateMarketDataDto } from '@ghostfolio/common/dtos';
+import { MarketDataDto } from '@ghostfolio/common/dtos';

 import { Type } from 'class-transformer';
-import { ArrayNotEmpty, IsArray } from 'class-validator';
+import { ArrayNotEmpty, IsArray, ValidateNested } from 'class-validator';

 export class UpdateBulkMarketDataDto {
  @ArrayNotEmpty()
  @IsArray()
-  @Type(() => UpdateMarketDataDto)
-  marketData: UpdateMarketDataDto[];
+  @Type(() => MarketDataDto)
+  @ValidateNested({ each: true })
+  marketData: MarketDataDto[];
 }
--- a/libs/common/src/lib/dtos/update-market-data.dto.ts
+++ b/libs/common/src/lib/dtos/update-market-data.dto.ts
@ -1,10 +0,0 @@
-import { IsISO8601, IsNumber, IsOptional } from 'class-validator';
-
-export class UpdateMarketDataDto {
-  @IsISO8601()
-  @IsOptional()
-  date?: string;
-
-  @IsNumber()
-  marketPrice: number;
-}
--- a/libs/common/src/lib/interfaces/index.ts
+++ b/libs/common/src/lib/interfaces/index.ts
@ -28,7 +28,6 @@ import type {
  NullableLineChartItem
 } from './line-chart-item.interface';
 import type { LookupItem } from './lookup-item.interface';
-import type { MarketData } from './market-data.interface';
 import type { PortfolioChart } from './portfolio-chart.interface';
 import type { PortfolioDetails } from './portfolio-details.interface';
 import type { PortfolioPerformance } from './portfolio-performance.interface';
@ -159,7 +158,6 @@ export {
  LineChartItem,
  LookupItem,
  LookupResponse,
-  MarketData,
  MarketDataOfMarketsResponse,
  NullableLineChartItem,
  OAuthResponse,
--- a/libs/common/src/lib/interfaces/market-data.interface.ts
+++ b/libs/common/src/lib/interfaces/market-data.interface.ts
@ -1,4 +0,0 @@
-export interface MarketData {
-  date: string;
-  marketPrice: number;
-}
--- a/libs/common/src/lib/interfaces/responses/export-response.interface.ts
+++ b/libs/common/src/lib/interfaces/responses/export-response.interface.ts
@ -1,8 +1,9 @@
+import { MarketDataDto } from '@ghostfolio/common/dtos';
+
 import { Account, Order, Platform, SymbolProfile, Tag } from '@prisma/client';

 import { AccountBalance } from '../account-balance.interface';
 import { AssetProfileIdentifier } from '../asset-profile-identifier.interface';
-import { MarketData } from '../market-data.interface';
 import { UserSettings } from '../user-settings.interface';

 export interface ExportResponse {
@ -29,7 +30,7 @@ export interface ExportResponse {
    | 'updatedAt'
    | 'userId'
  > & {
-    marketData: MarketData[];
+    marketData: MarketDataDto[];
  })[];
  meta: {
    date: string;
--- a/libs/ui/src/lib/historical-market-data-editor/historical-market-data-editor.component.ts
+++ b/libs/ui/src/lib/historical-market-data-editor/historical-market-data-editor.component.ts
@ -1,4 +1,4 @@
-import { UpdateMarketDataDto } from '@ghostfolio/common/dtos';
+import { MarketDataDto } from '@ghostfolio/common/dtos';
 import {
  DATE_FORMAT,
  getDateFormatString,
@ -249,7 +249,7 @@ export class GfHistoricalMarketDataEditorComponent

  public onImportHistoricalData() {
    try {
-      const marketData = csvToJson<UpdateMarketDataDto>(
+      const marketData = csvToJson<MarketDataDto>(
        this.historicalDataForm.controls.historicalData.controls.csvString
          .value ?? '',
        {