From 7b77dc044a765ea9b13d799065e361fb0bb10d26 Mon Sep 17 00:00:00 2001 From: Thomas Kaul <4159106+dtslvr@users.noreply.github.com> Date: Tue, 25 Apr 2023 20:09:12 +0200 Subject: [PATCH] Feature/add state to market data database schema (#1893) * Add state (CLOSE / INTRADAY) to MarketData * Update changelog --- CHANGELOG.md | 1 + apps/api/src/app/admin/admin.controller.ts | 2 +- .../portfolio/current-rate.service.spec.ts | 5 +++- .../data-gathering.processor.ts | 27 +++++++++---------- .../data-gathering/data-gathering.service.ts | 3 ++- .../data-provider/data-provider.service.ts | 3 ++- .../market-data/market-data.service.ts | 22 ++++++++++----- .../migration.sql | 5 ++++ prisma/schema.prisma | 18 ++++++++----- 9 files changed, 56 insertions(+), 30 deletions(-) create mode 100644 prisma/migrations/20230424194009_added_state_to_market_data/migration.sql diff --git a/CHANGELOG.md b/CHANGELOG.md index f05e8b4f0..cdf7c57a6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Introduced a new button to delete all activities from the portfolio activities page +- Added `state` to the `MarketData` database schema to distinguish `CLOSE` and `INTRADAY` in the data gathering ## 1.260.0 - 2023-04-23 diff --git a/apps/api/src/app/admin/admin.controller.ts b/apps/api/src/app/admin/admin.controller.ts index fc9f779a4..3898a8876 100644 --- a/apps/api/src/app/admin/admin.controller.ts +++ b/apps/api/src/app/admin/admin.controller.ts @@ -317,7 +317,7 @@ export class AdminController { const date = new Date(dateString); return this.marketDataService.updateMarketData({ - data: { ...data, dataSource }, + data: { marketPrice: data.marketPrice, state: 'CLOSE' }, where: { dataSource_date_symbol: { dataSource, diff --git a/apps/api/src/app/portfolio/current-rate.service.spec.ts b/apps/api/src/app/portfolio/current-rate.service.spec.ts index 00062ee60..0dd2b3a68 100644 --- a/apps/api/src/app/portfolio/current-rate.service.spec.ts +++ b/apps/api/src/app/portfolio/current-rate.service.spec.ts @@ -18,7 +18,8 @@ jest.mock('@ghostfolio/api/services/market-data/market-data.service', () => { createdAt: date, dataSource: DataSource.YAHOO, id: 'aefcbe3a-ee10-4c4f-9f2d-8ffad7b05584', - marketPrice: 1847.839966 + marketPrice: 1847.839966, + state: 'CLOSE' }); }, getRange: ({ @@ -37,6 +38,7 @@ jest.mock('@ghostfolio/api/services/market-data/market-data.service', () => { date: dateRangeStart, id: '8fa48fde-f397-4b0d-adbc-fb940e830e6d', marketPrice: 1841.823902, + state: 'CLOSE', symbol: symbols[0] }, { @@ -45,6 +47,7 @@ jest.mock('@ghostfolio/api/services/market-data/market-data.service', () => { date: dateRangeEnd, id: '082d6893-df27-4c91-8a5d-092e84315b56', marketPrice: 1847.839966, + state: 'CLOSE', symbol: symbols[0] } ]); diff --git a/apps/api/src/services/data-gathering/data-gathering.processor.ts b/apps/api/src/services/data-gathering/data-gathering.processor.ts index b18bc3969..063666ab0 100644 --- a/apps/api/src/services/data-gathering/data-gathering.processor.ts +++ b/apps/api/src/services/data-gathering/data-gathering.processor.ts @@ -6,8 +6,9 @@ import { GATHER_ASSET_PROFILE_PROCESS, GATHER_HISTORICAL_MARKET_DATA_PROCESS } from '@ghostfolio/common/config'; -import { DATE_FORMAT } from '@ghostfolio/common/helper'; +import { DATE_FORMAT, getStartOfUtcDate } from '@ghostfolio/common/helper'; import { UniqueAsset } from '@ghostfolio/common/interfaces'; +import { MarketDataService } from '@ghostfolio/api/services/market-data/market-data.service'; import { Process, Processor } from '@nestjs/bull'; import { Injectable, Logger } from '@nestjs/common'; import { Job } from 'bull'; @@ -28,7 +29,7 @@ export class DataGatheringProcessor { public constructor( private readonly dataGatheringService: DataGatheringService, private readonly dataProviderService: DataProviderService, - private readonly prismaService: PrismaService + private readonly marketDataService: MarketDataService ) {} @Process(GATHER_ASSET_PROFILE_PROCESS) @@ -83,19 +84,17 @@ export class DataGatheringProcessor { if (lastMarketPrice) { try { - await this.prismaService.marketData.create({ + await this.marketDataService.updateMarketData({ data: { - dataSource, - symbol, - date: new Date( - Date.UTC( - getYear(currentDate), - getMonth(currentDate), - getDate(currentDate), - 0 - ) - ), - marketPrice: lastMarketPrice + marketPrice: lastMarketPrice, + state: 'CLOSE' + }, + where: { + dataSource_date_symbol: { + dataSource, + symbol, + date: getStartOfUtcDate(currentDate) + } } }); } catch {} diff --git a/apps/api/src/services/data-gathering/data-gathering.service.ts b/apps/api/src/services/data-gathering/data-gathering.service.ts index 8b1ea705d..a238e02fc 100644 --- a/apps/api/src/services/data-gathering/data-gathering.service.ts +++ b/apps/api/src/services/data-gathering/data-gathering.service.ts @@ -271,7 +271,8 @@ export class DataGatheringService { by: ['symbol'], orderBy: [{ symbol: 'asc' }], where: { - date: { gt: startDate } + date: { gt: startDate }, + state: 'CLOSE' } }) ) diff --git a/apps/api/src/services/data-provider/data-provider.service.ts b/apps/api/src/services/data-provider/data-provider.service.ts index 6455252c3..db53d6341 100644 --- a/apps/api/src/services/data-provider/data-provider.service.ts +++ b/apps/api/src/services/data-provider/data-provider.service.ts @@ -291,7 +291,8 @@ export class DataProviderService { symbol, dataSource: response[symbol].dataSource, date: getStartOfUtcDate(new Date()), - marketPrice: response[symbol].marketPrice + marketPrice: response[symbol].marketPrice, + state: 'INTRADAY' }; }) }); diff --git a/apps/api/src/services/market-data/market-data.service.ts b/apps/api/src/services/market-data/market-data.service.ts index f7bffe431..414c247aa 100644 --- a/apps/api/src/services/market-data/market-data.service.ts +++ b/apps/api/src/services/market-data/market-data.service.ts @@ -5,7 +5,12 @@ import { PrismaService } from '@ghostfolio/api/services/prisma/prisma.service'; import { resetHours } from '@ghostfolio/common/helper'; import { UniqueAsset } from '@ghostfolio/common/interfaces'; import { Injectable } from '@nestjs/common'; -import { DataSource, MarketData, Prisma } from '@prisma/client'; +import { + DataSource, + MarketData, + MarketDataState, + Prisma +} from '@prisma/client'; @Injectable() export class MarketDataService { @@ -92,7 +97,9 @@ export class MarketDataService { } public async updateMarketData(params: { - data: { dataSource: DataSource } & UpdateMarketDataDto; + data: { + state: MarketDataState; + } & UpdateMarketDataDto; where: Prisma.MarketDataWhereUniqueInput; }): Promise { const { data, where } = params; @@ -100,12 +107,13 @@ export class MarketDataService { return this.prismaService.marketData.upsert({ where, create: { - dataSource: data.dataSource, + dataSource: where.dataSource_date_symbol.dataSource, date: where.dataSource_date_symbol.date, marketPrice: data.marketPrice, + state: data.state, symbol: where.dataSource_date_symbol.symbol }, - update: { marketPrice: data.marketPrice } + update: { marketPrice: data.marketPrice, state: data.state } }); } @@ -119,16 +127,18 @@ export class MarketDataService { data: Prisma.MarketDataUpdateInput[]; }): Promise { const upsertPromises = data.map( - ({ dataSource, date, marketPrice, symbol }) => { + ({ dataSource, date, marketPrice, symbol, state }) => { return this.prismaService.marketData.upsert({ create: { dataSource: dataSource, date: date, marketPrice: marketPrice, + state: state, symbol: symbol }, update: { - marketPrice: marketPrice + marketPrice: marketPrice, + state: state }, where: { dataSource_date_symbol: { diff --git a/prisma/migrations/20230424194009_added_state_to_market_data/migration.sql b/prisma/migrations/20230424194009_added_state_to_market_data/migration.sql new file mode 100644 index 000000000..e030d8109 --- /dev/null +++ b/prisma/migrations/20230424194009_added_state_to_market_data/migration.sql @@ -0,0 +1,5 @@ +-- CreateEnum +CREATE TYPE "MarketDataState" AS ENUM ('CLOSE', 'INTRADAY'); + +-- AlterTable +ALTER TABLE "MarketData" ADD COLUMN "state" "MarketDataState" NOT NULL DEFAULT 'CLOSE'; diff --git a/prisma/schema.prisma b/prisma/schema.prisma index 6e099d5ee..d74a7ae54 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -59,12 +59,13 @@ model AuthDevice { } model MarketData { - createdAt DateTime @default(now()) + createdAt DateTime @default(now()) dataSource DataSource date DateTime - id String @id @default(uuid()) - symbol String + id String @id @default(uuid()) marketPrice Float + state MarketDataState @default(CLOSE) + symbol String @@unique([dataSource, date, symbol]) @@index([symbol]) @@ -214,9 +215,9 @@ enum DataSource { YAHOO } -enum ViewMode { - DEFAULT - ZEN +enum MarketDataState { + CLOSE + INTRADAY } enum Provider { @@ -237,3 +238,8 @@ enum Type { ITEM SELL } + +enum ViewMode { + DEFAULT + ZEN +}