Added Market Data upsert locks

apps/api/src/services/data-gathering/data-gathering.service.ts

@@ -24,6 +24,7 @@ import { DataSource } from '@prisma/client';
 import { JobOptions, Queue } from 'bull';
 import { format, min, subDays, subYears } from 'date-fns';
 import { isEmpty } from 'lodash';
+import { Lock } from 'async-lock';
 
 @Injectable()
 export class DataGatheringService {
@@ -89,6 +90,7 @@ export class DataGatheringService {
     date: Date;
     symbol: string;
   }) {
+    const lock = new Lock();
     try {
       const historicalData = await this.dataProviderService.getHistoricalRaw(
         [{ dataSource, symbol }],
@@ -100,15 +102,17 @@ export class DataGatheringService {
         historicalData[symbol][format(date, DATE_FORMAT)].marketPrice;
 
       if (marketPrice) {
-        return await this.prismaService.marketData.upsert({
+        return await lock.acquire('marketData', async function () {
-          create: {
+          return await this.prismaService.marketData.upsert({
-            dataSource,
+            create: {
-            date,
+              dataSource,
-            marketPrice,
+              date,
-            symbol
+              marketPrice,
-          },
+              symbol
-          update: { marketPrice },
+            },
-          where: { dataSource_date_symbol: { dataSource, date, symbol } }
+            update: { marketPrice },
+            where: { dataSource_date_symbol: { dataSource, date, symbol } }
+          });
         });
       }
     } catch (error) {

apps/api/src/services/market-data/market-data.service.ts

@@ -13,11 +13,14 @@ import {
   Prisma
 } from '@prisma/client';
 import { DateQueryHelper } from '@ghostfolio/api/helper/dateQueryHelper';
+import AwaitLock from 'await-lock';
 
 @Injectable()
 export class MarketDataService {
   public constructor(private readonly prismaService: PrismaService) {}
 
+  lock = new AwaitLock();
+
   private dateQueryHelper = new DateQueryHelper();
 
   public async deleteMany({ dataSource, symbol }: UniqueAsset) {
@@ -129,18 +132,22 @@ export class MarketDataService {
     where: Prisma.MarketDataWhereUniqueInput;
   }): Promise<MarketData> {
     const { data, where } = params;
-
+    await this.lock.acquireAsync();
-    return this.prismaService.marketData.upsert({
+    try {
-      where,
+      return this.prismaService.marketData.upsert({
-      create: {
+        where,
-        dataSource: where.dataSource_date_symbol.dataSource,
+        create: {
-        date: where.dataSource_date_symbol.date,
+          dataSource: where.dataSource_date_symbol.dataSource,
-        marketPrice: data.marketPrice,
+          date: where.dataSource_date_symbol.date,
-        state: data.state,
+          marketPrice: data.marketPrice,
-        symbol: where.dataSource_date_symbol.symbol
+          state: data.state,
-      },
+          symbol: where.dataSource_date_symbol.symbol
-      update: { marketPrice: data.marketPrice, state: data.state }
+        },
-    });
+        update: { marketPrice: data.marketPrice, state: data.state }
+      });
+    } finally {
+      this.lock.release();
+    }
   }
 
   /**
@@ -153,30 +160,34 @@ export class MarketDataService {
     data: Prisma.MarketDataUpdateInput[];
   }): Promise<MarketData[]> {
     const upsertPromises = data.map(
-      ({ dataSource, date, marketPrice, symbol, state }) => {
+      async ({ dataSource, date, marketPrice, symbol, state }) => {
-        return this.prismaService.marketData.upsert({
+        await this.lock.acquireAsync();
-          create: {
+        try {
-            dataSource: <DataSource>dataSource,
+          return this.prismaService.marketData.upsert({
-            date: <Date>date,
+            create: {
-            marketPrice: <number>marketPrice,
-            state: <MarketDataState>state,
-            symbol: <string>symbol
-          },
-          update: {
-            marketPrice: <number>marketPrice,
-            state: <MarketDataState>state
-          },
-          where: {
-            dataSource_date_symbol: {
               dataSource: <DataSource>dataSource,
               date: <Date>date,
+              marketPrice: <number>marketPrice,
+              state: <MarketDataState>state,
               symbol: <string>symbol
+            },
+            update: {
+              marketPrice: <number>marketPrice,
+              state: <MarketDataState>state
+            },
+            where: {
+              dataSource_date_symbol: {
+                dataSource: <DataSource>dataSource,
+                date: <Date>date,
+                symbol: <string>symbol
+              }
             }
-          }
+          });
-        });
+        } finally {
+          this.lock.release();
+        }
       }
     );
-
+    return await Promise.all(upsertPromises);
-    return this.prismaService.$transaction(upsertPromises);
   }
 }

package.json

@@ -86,6 +86,7 @@
     "@simplewebauthn/server": "8.3.2",
     "@stripe/stripe-js": "1.47.0",
     "alphavantage": "2.2.0",
+    "await-lock": "^2.2.2",
     "big.js": "6.2.1",
     "body-parser": "1.20.1",
     "bootstrap": "4.6.0",