@ -11,12 +11,16 @@ import { DataGatheringService } from '@ghostfolio/api/services/queues/data-gathe
import { SymbolProfileService } from '@ghostfolio/api/services/symbol-profile/symbol-profile.service' ; import { SymbolProfileService } from '@ghostfolio/api/services/symbol-profile/symbol-profile.service' ;
import { import {
DATA_GATHERING_QUEUE_PRIORITY_HIGH , DATA_GATHERING_QUEUE_PRIORITY_HIGH ,
DEFAULT_CURRENCY ,
GATHER_ASSET_PROFILE_PROCESS_JOB_NAME , GATHER_ASSET_PROFILE_PROCESS_JOB_NAME ,
GATHER_ASSET_PROFILE_PROCESS_JOB_OPTIONS , GATHER_ASSET_PROFILE_PROCESS_JOB_OPTIONS ,
ghostfolioPrefix , ghostfolioPrefix
TAG_ID_EXCLUDE_FROM_ANALYSIS
} from '@ghostfolio/common/config' ; } from '@ghostfolio/common/config' ;
import { getAssetProfileIdentifier } from '@ghostfolio/common/helper' ; import {
DATE_FORMAT ,
getAssetProfileIdentifier ,
resetHours
} from '@ghostfolio/common/helper' ;
import { import {
ActivitiesResponse , ActivitiesResponse ,
Activity , Activity ,
@ -39,8 +43,8 @@ import {
} from '@prisma/client' ; } from '@prisma/client' ;
import { Big } from 'big.js' ; import { Big } from 'big.js' ;
import { isUUID } from 'class-validator' ; import { isUUID } from 'class-validator' ;
import { endOfToday , isAfter } from 'date-fns' ; import { endOfToday , format , isAfter , subDays } from 'date-fns' ;
import { groupBy , uniqBy } from 'lodash' ; import { uniqBy } from 'lodash' ;
import { randomUUID } from 'node:crypto' ; import { randomUUID } from 'node:crypto' ;
@Injectable ( ) @Injectable ( )
@ -469,7 +473,7 @@ export class ActivitiesService {
sortColumn , sortColumn ,
sortDirection = 'asc' , sortDirection = 'asc' ,
startDate , startDate ,
take = Number . MAX_SAFE_INTEGER , take ,
types , types ,
userCurrency , userCurrency ,
userId , userId ,
@ -488,166 +492,112 @@ export class ActivitiesService {
userId : string ; userId : string ;
withExcludedAccountsAndActivities? : boolean ; withExcludedAccountsAndActivities? : boolean ;
} ) : Promise < ActivitiesResponse > { } ) : Promise < ActivitiesResponse > {
let orderBy : Prisma.Enumerable < Prisma.OrderOrderByWithRelationInput > = [ const where : Prisma.OrderWhereInput = {
{ date : 'asc' } . . . ( includeDrafts === false ? { isDraft : false } : { } ) ,
] ; . . . ( types ? . length > 0 ? { type : { in : types } } : { } ) ,
. . . ( userId ? { userId } : { } )
const where : Prisma.OrderWhereInput = { userId } ; } ;
if ( endDate || startDate ) {
where . AND = [ ] ;
if ( endDate ) {
where . AND . push ( { date : { lte : endDate } } ) ;
}
if ( startDate ) { if ( startDate ) {
where . AND . push ( { date : { gt : startDate } } ) ; where . date = { gte : resetHours ( startDate ) } ;
}
} }
const { if ( endDate ) {
ACCOUNT : filtersByAccount , where . date = {
ASSET_CLASS : filtersByAssetClass , . . . ( where . date as Prisma . DateTimeFilter ) ,
TAG : filtersByTag lte : resetHours ( endDate )
} = groupBy ( filters , ( { type } ) = > {
return type ;
} ) ;
const filterByDataSource = filters ? . find ( ( { type } ) = > {
return type === 'DATA_SOURCE' ;
} ) ? . id ;
const filterBySymbol = filters ? . find ( ( { type } ) = > {
return type === 'SYMBOL' ;
} ) ? . id ;
const searchQuery = filters ? . find ( ( { type } ) = > {
return type === 'SEARCH_QUERY' ;
} ) ? . id ;
if ( filtersByAccount ? . length > 0 ) {
where . accountId = {
in : filtersByAccount . map ( ( { id } ) = > {
return id ;
} )
} ; } ;
} }
if ( includeDraft s=== false ) { if ( withExcludedAccountsAndActivities === false ) {
where . isDraft = false ; where . account = { isExcluded : false } ;
} }
if ( filtersByAssetClass ? . length > 0 ) { if ( filters ? . length > 0 ) {
where . SymbolProfile = { where . OR = [ ] ;
OR : [
{ const accountIds = filters
AND : [ . filter ( ( { type } ) = > type === 'ACCOUNT' )
{ . map ( ( { id } ) = > id ) ;
OR : filtersByAssetClass.map ( ( { id } ) = > { if ( accountIds . length > 0 ) {
return { assetClass : AssetClass [ id ] } ; where . OR . push ( { accountId : { in : accountIds } } ) ;
} )
} ,
{
OR : [
{ SymbolProfileOverrides : { is : null } } ,
{ SymbolProfileOverrides : { assetClass : null } }
]
}
]
} ,
{
SymbolProfileOverrides : {
OR : filtersByAssetClass.map ( ( { id } ) = > {
return { assetClass : AssetClass [ id ] } ;
} )
}
}
]
} ;
} }
if ( filterByDataSource && filterBySymbol ) { const assetClasses = filters
if ( where . SymbolProfile ) { . filter ( ( { type } ) = > type === 'ASSET_CLASS' )
where . SymbolProfile = { . map ( ( { id } ) = > id ) as AssetClass [ ] ;
AND : [ if ( assetClasses . length > 0 ) {
where . SymbolProfile , where . OR . push ( { SymbolProfile : { assetClass : { in : assetClasses } } } ) ;
{
AND : [
{ dataSource : filterByDataSource as DataSource } ,
{ symbol : filterBySymbol }
]
} }
]
} ; const tags = filters
} else { . filter ( ( { type } ) = > type === 'TAG' )
where . SymbolProfile = { . map ( ( { id } ) = > id ) ;
AND : [ if ( tags . length > 0 ) {
{ dataSource : filterByDataSource as DataSource } , where . OR . push ( { tags : { some : { id : { in : tags } } } } ) ;
{ symbol : filterBySymbol }
]
} ;
} }
} }
if ( searchQuery ) { let orderBy : Prisma.OrderOrderByWithRelationInput [ ] = [
const searchQueryWhereInput : Prisma.SymbolProfileWhereInput [ ] = [ { date : sortDirection }
{ id : { mode : 'insensitive' , startsWith : searchQuery } } ,
{ isin : { mode : 'insensitive' , startsWith : searchQuery } } ,
{ name : { mode : 'insensitive' , startsWith : searchQuery } } ,
{ symbol : { mode : 'insensitive' , startsWith : searchQuery } }
] ; ] ;
if ( where . SymbolProfile ) { if ( sortColumn ) {
where . SymbolProfile = { orderBy = [ ] ;
AND : [
where . SymbolProfile , if (
{ [ 'currency' , 'fee' , 'quantity' , 'type' , 'unitPrice' ] . includes (
OR : searchQueryWhereInput sortColumn
} )
] ) {
} ; orderBy . push ( { [ sortColumn ] : sortDirection } ) ;
} else { } else {
where . SymbolProfile = { if ( sortColumn === 'SymbolProfile.name' ) {
OR : searchQueryWhereInput orderBy . push ( { SymbolProfile : { name : sortDirection } } ) ;
} ; } else if ( sortColumn === 'account.name' ) {
orderBy . push ( { account : { name : sortDirection } } ) ;
} }
} }
if ( filtersByTag ? . length > 0 ) {
where . tags = {
some : {
OR : filtersByTag.map ( ( { id } ) = > {
return { id } ;
} )
}
} ;
} }
if ( sortColumn ) { const count = await this . prismaService . order . count ( { where } ) ;
orderBy = [ { [ sortColumn ] : sortDirection } ] ;
}
if ( types ? . length > 0 ) { let orders : OrderWithAccount [ ] = [ ] ;
where . type = { in : types } ;
}
if ( withExcludedAccountsAndActivities === false ) { // If take is undefined and count is extremely large, batch fetch to prevent Prisma P2029 limits
where . OR = [ const BATCH_SIZE = 5000 ;
{ account : null } ,
{ account : { NOT : { isExcluded : true } } } if ( take === undefined && count > BATCH_SIZE ) {
] ; let currentSkip = skip || 0 ;
const totalToFetch = count - currentSkip ;
where . tags = { while ( orders . length < totalToFetch ) {
. . . where . tags , const batch = await this . orders ( {
none : { skip : currentSkip ,
id : TAG_ID_EXCLUDE_FROM_ANALYSIS take : BATCH_SIZE ,
where ,
include : {
account : {
include : {
platform : true
} }
} ; } ,
// eslint-disable-next-line @typescript-eslint/naming-convention
SymbolProfile : true ,
tags : true
} ,
orderBy : [ . . . orderBy , { id : sortDirection } ]
} ) ;
if ( batch . length === 0 ) {
break ;
} }
const [ orders , count ] = await Promise . all ( [ orders = orders . concat ( batch ) ;
this . orders ( { currentSkip += batch . length ;
}
} else {
orders = await this . orders ( {
skip , skip ,
take , take ,
where , where ,
@ -662,9 +612,8 @@ export class ActivitiesService {
tags : true tags : true
} , } ,
orderBy : [ . . . orderBy , { id : sortDirection } ] orderBy : [ . . . orderBy , { id : sortDirection } ]
} ) , } ) ;
this . prismaService . order . count ( { where } ) }
] ) ;
const assetProfileIdentifiers = uniqBy ( const assetProfileIdentifiers = uniqBy (
orders . map ( ( { SymbolProfile } ) = > { orders . map ( ( { SymbolProfile } ) = > {
@ -685,8 +634,122 @@ export class ActivitiesService {
assetProfileIdentifiers assetProfileIdentifiers
) ; ) ;
const activities = await Promise . all ( let exchangeRatesToUser : any = { } ;
orders . map ( async ( order ) = > { let exchangeRatesToDefault : any = { } ;
if ( orders . length > 0 ) {
let minDate = orders [ 0 ] . date ;
let maxDate = orders [ 0 ] . date ;
const uniqueCurrencies = new Set < string > ( ) ;
uniqueCurrencies . add ( userCurrency ) ;
uniqueCurrencies . add ( DEFAULT_CURRENCY ) ;
const uniqueDatesSet = new Set < number > ( ) ;
for ( const order of orders ) {
if ( order . date < minDate ) {
minDate = order . date ;
}
if ( order . date > maxDate ) {
maxDate = order . date ;
}
uniqueDatesSet . add ( resetHours ( order . date ) . getTime ( ) ) ;
if ( order . currency ) {
uniqueCurrencies . add ( order . currency ) ;
}
if ( order . SymbolProfile ? . currency ) {
uniqueCurrencies . add ( order . SymbolProfile . currency ) ;
}
}
const currenciesList = Array . from ( uniqueCurrencies ) . filter ( Boolean ) ;
const uniqueDates = Array . from ( uniqueDatesSet ) . map (
( time ) = > new Date ( time )
) ;
const startDatePreload = subDays ( resetHours ( minDate ) , 1 ) ;
const endDatePreload = resetHours ( maxDate ) ;
const [ ratesUser , ratesDefault ] = await Promise . all ( [
this . exchangeRateDataService . getExchangeRatesByCurrency ( {
currencies : currenciesList ,
dates : uniqueDates ,
endDate : endDatePreload ,
startDate : startDatePreload ,
targetCurrency : userCurrency
} ) ,
this . exchangeRateDataService . getExchangeRatesByCurrency ( {
currencies : currenciesList ,
dates : uniqueDates ,
endDate : endDatePreload ,
startDate : startDatePreload ,
targetCurrency : DEFAULT_CURRENCY
} )
] ) ;
exchangeRatesToUser = ratesUser ;
exchangeRatesToDefault = ratesDefault ;
}
const getPreloadedRate = (
from : string ,
to : string ,
dateStr : string
) : number | undefined = > {
if ( from === to ) {
return 1 ;
}
if ( to === userCurrency ) {
return exchangeRatesToUser [ ` ${ from } ${ userCurrency } ` ] ? . [ dateStr ] ;
}
if ( to === DEFAULT_CURRENCY ) {
return exchangeRatesToDefault [ ` ${ from } ${ DEFAULT_CURRENCY } ` ] ? . [ dateStr ] ;
}
if ( from === DEFAULT_CURRENCY ) {
const rateToDefault =
exchangeRatesToDefault [ ` ${ to } ${ DEFAULT_CURRENCY } ` ] ? . [ dateStr ] ;
return rateToDefault ? 1 / rateToDefault : undefined ;
}
const rateFromToDefault =
exchangeRatesToDefault [ ` ${ from } ${ DEFAULT_CURRENCY } ` ] ? . [ dateStr ] ;
const rateToToDefault =
exchangeRatesToDefault [ ` ${ to } ${ DEFAULT_CURRENCY } ` ] ? . [ dateStr ] ;
if ( rateFromToDefault !== undefined && rateToToDefault ) {
return rateFromToDefault / rateToToDefault ;
}
return undefined ;
} ;
const convertValue = async (
val : number ,
from : string ,
to : string ,
date : Date
) : Promise < number > = > {
if ( val === 0 ) {
return 0 ;
}
const dateStr = format ( date , DATE_FORMAT ) ;
const rate = getPreloadedRate ( from , to , dateStr ) ;
if ( rate !== undefined && ! isNaN ( rate ) ) {
return rate * val ;
}
return this . exchangeRateDataService . toCurrencyAtDate ( val , from , to , date ) ;
} ;
const activities = [ ] ;
const chunkSize = 500 ;
for ( let i = 0 ; i < orders . length ; i += chunkSize ) {
const chunk = orders . slice ( i , i + chunkSize ) ;
const processedChunk = await Promise . all (
chunk . map ( async ( order ) = > {
const assetProfile = assetProfiles . find ( ( { dataSource , symbol } ) = > { const assetProfile = assetProfiles . find ( ( { dataSource , symbol } ) = > {
return ( return (
dataSource === order . SymbolProfile . dataSource && dataSource === order . SymbolProfile . dataSource &&
@ -702,25 +765,25 @@ export class ActivitiesService {
unitPriceInAssetProfileCurrency , unitPriceInAssetProfileCurrency ,
valueInBaseCurrency valueInBaseCurrency
] = await Promise . all ( [ ] = await Promise . all ( [
this . exchangeRateDataService . toCurrencyAtDat e( convertValu e (
order . fee , order . fee ,
order . currency ? ? order . SymbolProfile . currency , order . currency ? ? order . SymbolProfile . currency ,
order . SymbolProfile . currency , order . SymbolProfile . currency ,
order . date order . date
) , ) ,
this . exchangeRateDataService . toCurrencyAtDat e( convertValu e (
order . fee , order . fee ,
order . currency ? ? order . SymbolProfile . currency , order . currency ? ? order . SymbolProfile . currency ,
userCurrency , userCurrency ,
order . date order . date
) , ) ,
this . exchangeRateDataService . toCurrencyAtDat e( convertValu e (
order . unitPrice , order . unitPrice ,
order . currency ? ? order . SymbolProfile . currency , order . currency ? ? order . SymbolProfile . currency ,
order . SymbolProfile . currency , order . SymbolProfile . currency ,
order . date order . date
) , ) ,
this . exchangeRateDataService . toCurrencyAtDat e( convertValu e (
value , value ,
order . currency ? ? order . SymbolProfile . currency , order . currency ? ? order . SymbolProfile . currency ,
userCurrency , userCurrency ,
@ -730,16 +793,20 @@ export class ActivitiesService {
return { return {
. . . order , . . . order ,
feeInAssetProfileCurrency , feeInAssetProfileCurrency : feeInAssetProfileCurrency ? ? order . fee ,
feeInBaseCurrency , feeInBaseCurrency : feeInBaseCurrency ? ? order . fee ,
unitPriceInAssetProfileCurrency , unitPriceInAssetProfileCurrency :
unitPriceInAssetProfileCurrency ? ? order . unitPrice ,
value , value ,
valueInBaseCurrency , valueInBaseCurrency : valueInBaseCurrency ? ? value ,
SymbolProfile : assetProfile SymbolProfile : assetProfile
} ; } ;
} ) } )
) ; ) ;
activities . push ( . . . processedChunk ) ;
}
return { activities , count } ; return { activities , count } ;
} }
Andrea Bugeja
1 day ago
GitHub