@ -11,12 +11,16 @@ import { DataGatheringService } from '@ghostfolio/api/services/queues/data-gathe
import { SymbolProfileService } from '@ghostfolio/api/services/symbol-profile/symbol-profile.service' ; import { SymbolProfileService } from '@ghostfolio/api/services/symbol-profile/symbol-profile.service' ;
import { import {
DATA_GATHERING_QUEUE_PRIORITY_HIGH , DATA_GATHERING_QUEUE_PRIORITY_HIGH ,
DEFAULT_CURRENCY ,
GATHER_ASSET_PROFILE_PROCESS_JOB_NAME , GATHER_ASSET_PROFILE_PROCESS_JOB_NAME ,
GATHER_ASSET_PROFILE_PROCESS_JOB_OPTIONS , GATHER_ASSET_PROFILE_PROCESS_JOB_OPTIONS ,
ghostfolioPrefix , ghostfolioPrefix
TAG_ID_EXCLUDE_FROM_ANALYSIS
} from '@ghostfolio/common/config' ; } from '@ghostfolio/common/config' ;
import { getAssetProfileIdentifier } from '@ghostfolio/common/helper' ; import {
DATE_FORMAT ,
getAssetProfileIdentifier ,
resetHours
} from '@ghostfolio/common/helper' ;
import { import {
ActivitiesResponse , ActivitiesResponse ,
Activity , Activity ,
@ -39,8 +43,8 @@ import {
} from '@prisma/client' ; } from '@prisma/client' ;
import { Big } from 'big.js' ; import { Big } from 'big.js' ;
import { isUUID } from 'class-validator' ; import { isUUID } from 'class-validator' ;
import { endOfToday , isAfter } from 'date-fns' ; import { endOfToday , format , isAfter , subDays } from 'date-fns' ;
import { groupBy , uniqBy } from 'lodash' ; import { uniqBy } from 'lodash' ;
import { randomUUID } from 'node:crypto' ; import { randomUUID } from 'node:crypto' ;
@Injectable ( ) @Injectable ( )
@ -469,7 +473,7 @@ export class ActivitiesService {
sortColumn , sortColumn ,
sortDirection = 'asc' , sortDirection = 'asc' ,
startDate , startDate ,
take = Number . MAX_SAFE_INTEGER , take ,
types , types ,
userCurrency , userCurrency ,
userId , userId ,
@ -488,166 +492,112 @@ export class ActivitiesService {
userId : string ; userId : string ;
withExcludedAccountsAndActivities? : boolean ; withExcludedAccountsAndActivities? : boolean ;
} ) : Promise < ActivitiesResponse > { } ) : Promise < ActivitiesResponse > {
let orderBy : Prisma.Enumerable < Prisma.OrderOrderByWithRelationInput > = [ const where : Prisma.OrderWhereInput = {
{ date : 'asc' } . . . ( includeDrafts === false ? { isDraft : false } : { } ) ,
] ; . . . ( types ? . length > 0 ? { type : { in : types } } : { } ) ,
. . . ( userId ? { userId } : { } )
const where : Prisma.OrderWhereInput = { userId } ; } ;
if ( endDate || startDate ) {
where . AND = [ ] ;
if ( end Date) { if ( startDate ) {
where . AND . push ( { date : { lte : endDate } } ) ; where . date = { gte : resetHours ( startDate ) } ;
} }
if ( startDate ) { if ( endDate ) {
where . AND . push ( { date : { gt : startDate } } ) ; where . date = {
} . . . ( where . date as Prisma . DateTimeFilter ) ,
lte : resetHours ( endDate )
} ;
} }
const { if ( withExcludedAccountsAndActivities === false ) {
ACCOUNT : filtersByAccount , where . account = { isExcluded : false } ;
ASSET_CLASS : filtersByAssetClass , }
TAG : filtersByTag
} = groupBy ( filters , ( { type } ) = > {
return type ;
} ) ;
const filterByDataSource = filters ? . find ( ( { type } ) = > { if ( filters ? . length > 0 ) {
return type === 'DATA_SOURCE' ; where . OR = [ ] ;
} ) ? . id ;
const filterBySymbol = filters ? . find ( ( { type } ) = > { const accountIds = filters
return type === 'SYMBOL' ; . filter ( ( { type } ) = > type === 'ACCOUNT' )
} ) ? . id ; . map ( ( { id } ) = > id ) ;
if ( accountIds . length > 0 ) {
where . OR . push ( { accountId : { in : accountIds } } ) ;
}
const searchQuery = filters ? . find ( ( { type } ) = > { const assetClasses = filters
return type === 'SEARCH_QUERY' ; . filter ( ( { type } ) = > type === 'ASSET_CLASS' )
} ) ? . id ; . map ( ( { id } ) = > id ) as AssetClass [ ] ;
if ( assetClasses . length > 0 ) {
where . OR . push ( { SymbolProfile : { assetClass : { in : assetClasses } } } ) ;
}
if ( filtersByAccount ? . length > 0 ) { const tags = filters
where . accountId = { . filter ( ( { type } ) = > type === 'TAG' )
in : filtersByAccount . map ( ( { id } ) = > { . map ( ( { id } ) = > id ) ;
return id ; if ( tags . length > 0 ) {
} ) where . OR . push ( { tags : { some : { id : { in : tags } } } } ) ;
} ; }
} }
if ( includeDrafts === false ) { let orderBy : Prisma.OrderOrderByWithRelationInput [ ] = [
where . isDraft = false ; { date : sortDirection }
} ] ;
if ( filtersByAssetClass ? . length > 0 ) { if ( sortColumn ) {
where . SymbolProfile = { orderBy = [ ] ;
OR : [
{
AND : [
{
OR : filtersByAssetClass.map ( ( { id } ) = > {
return { assetClass : AssetClass [ id ] } ;
} )
} ,
{
OR : [
{ SymbolProfileOverrides : { is : null } } ,
{ SymbolProfileOverrides : { assetClass : null } }
]
}
]
} ,
{
SymbolProfileOverrides : {
OR : filtersByAssetClass.map ( ( { id } ) = > {
return { assetClass : AssetClass [ id ] } ;
} )
}
}
]
} ;
}
if ( filterByDataSource && filterBySymbol ) { if (
if ( where . SymbolProfile ) { [ 'currency' , 'fee' , 'quantity' , 'type' , 'unitPrice' ] . includes (
where . SymbolProfile = { sortColumn
AND : [ )
where . SymbolProfile , ) {
{ orderBy . push ( { [ sortColumn ] : sortDirection } ) ;
AND : [
{ dataSource : filterByDataSource as DataSource } ,
{ symbol : filterBySymbol }
]
}
]
} ;
} else { } else {
where . SymbolProfile = { if ( sortColumn === 'SymbolProfile.name' ) {
AND : [ orderBy . push ( { SymbolProfile : { name : sortDirection } } ) ;
{ dataSource : filterByDataSource as DataSource } , } else if ( sortColumn === 'account.name' ) {
{ symbol : filterBySymbol } orderBy . push ( { account : { name : sortDirection } } ) ;
] }
} ;
} }
} }
if ( searchQuery ) { const count = await this . prismaService . order . count ( { where } ) ;
const searchQueryWhereInput : Prisma.SymbolProfileWhereInput [ ] = [
{ id : { mode : 'insensitive' , startsWith : searchQuery } } ,
{ isin : { mode : 'insensitive' , startsWith : searchQuery } } ,
{ name : { mode : 'insensitive' , startsWith : searchQuery } } ,
{ symbol : { mode : 'insensitive' , startsWith : searchQuery } }
] ;
if ( where . SymbolProfile ) {
where . SymbolProfile = {
AND : [
where . SymbolProfile ,
{
OR : searchQueryWhereInput
}
]
} ;
} else {
where . SymbolProfile = {
OR : searchQueryWhereInput
} ;
}
}
if ( filtersByTag ? . length > 0 ) { let orders : OrderWithAccount [ ] = [ ] ;
where . tags = {
some : {
OR : filtersByTag.map ( ( { id } ) = > {
return { id } ;
} )
}
} ;
}
if ( sortColumn ) { // If take is undefined and count is extremely large, batch fetch to prevent Prisma P2029 limits
orderBy = [ { [ sortColumn ] : sortDirection } ] ; const BATCH_SIZE = 5000 ;
}
if ( types ? . length > 0 ) { if ( take === undefined && count > BATCH_SIZE ) {
where . type = { in : types } ; let currentSkip = skip || 0 ;
} const totalToFetch = count - currentSkip ;
if ( withExcludedAccountsAndActivities === false ) { while ( orders . length < totalToFetch ) {
where . OR = [ const batch = await this . orders ( {
{ account : null } , skip : currentSkip ,
{ account : { NOT : { isExcluded : true } } } take : BATCH_SIZE ,
] ; where ,
include : {
where . tags = { account : {
. . . where . tags , include : {
none : { platform : true
id : TAG_ID_EXCLUDE_FROM_ANALYSIS }
} ,
// eslint-disable-next-line @typescript-eslint/naming-convention
SymbolProfile : true ,
tags : true
} ,
orderBy : [ . . . orderBy , { id : sortDirection } ]
} ) ;
if ( batch . length === 0 ) {
break ;
} }
} ;
}
const [ orders , count ] = await Promise . all ( [ orders = orders . concat ( batch ) ;
this . orders ( { currentSkip += batch . length ;
}
} else {
orders = await this . orders ( {
skip , skip ,
take , take ,
where , where ,
@ -662,9 +612,8 @@ export class ActivitiesService {
tags : true tags : true
} , } ,
orderBy : [ . . . orderBy , { id : sortDirection } ] orderBy : [ . . . orderBy , { id : sortDirection } ]
} ) , } ) ;
this . prismaService . order . count ( { where } ) }
] ) ;
const assetProfileIdentifiers = uniqBy ( const assetProfileIdentifiers = uniqBy (
orders . map ( ( { SymbolProfile } ) = > { orders . map ( ( { SymbolProfile } ) = > {
@ -685,60 +634,178 @@ export class ActivitiesService {
assetProfileIdentifiers assetProfileIdentifiers
) ; ) ;
const activities = await Promise . all ( let exchangeRatesToUser : any = { } ;
orders . map ( async ( order ) = > { let exchangeRatesToDefault : any = { } ;
const assetProfile = assetProfiles . find ( ( { dataSource , symbol } ) = > {
return ( if ( orders . length > 0 ) {
dataSource === order . SymbolProfile . dataSource && let minDate = orders [ 0 ] . date ;
symbol === order . SymbolProfile . symbol let maxDate = orders [ 0 ] . date ;
) ; const uniqueCurrencies = new Set < string > ( ) ;
} ) ; uniqueCurrencies . add ( userCurrency ) ;
uniqueCurrencies . add ( DEFAULT_CURRENCY ) ;
const uniqueDatesSet = new Set < number > ( ) ;
for ( const order of orders ) {
if ( order . date < minDate ) {
minDate = order . date ;
}
if ( order . date > maxDate ) {
maxDate = order . date ;
}
uniqueDatesSet . add ( resetHours ( order . date ) . getTime ( ) ) ;
if ( order . currency ) {
uniqueCurrencies . add ( order . currency ) ;
}
if ( order . SymbolProfile ? . currency ) {
uniqueCurrencies . add ( order . SymbolProfile . currency ) ;
}
}
const currenciesList = Array . from ( uniqueCurrencies ) . filter ( Boolean ) ;
const uniqueDates = Array . from ( uniqueDatesSet ) . map (
( time ) = > new Date ( time )
) ;
const startDatePreload = subDays ( resetHours ( minDate ) , 1 ) ;
const endDatePreload = resetHours ( maxDate ) ;
const [ ratesUser , ratesDefault ] = await Promise . all ( [
this . exchangeRateDataService . getExchangeRatesByCurrency ( {
currencies : currenciesList ,
dates : uniqueDates ,
endDate : endDatePreload ,
startDate : startDatePreload ,
targetCurrency : userCurrency
} ) ,
this . exchangeRateDataService . getExchangeRatesByCurrency ( {
currencies : currenciesList ,
dates : uniqueDates ,
endDate : endDatePreload ,
startDate : startDatePreload ,
targetCurrency : DEFAULT_CURRENCY
} )
] ) ;
exchangeRatesToUser = ratesUser ;
exchangeRatesToDefault = ratesDefault ;
}
const getPreloadedRate = (
from : string ,
to : string ,
dateStr : string
) : number | undefined = > {
if ( from === to ) {
return 1 ;
}
const value = new Big ( order . quantity ) . mul ( order . unitPrice ) . toNumber ( ) ; if ( to === userCurrency ) {
return exchangeRatesToUser [ ` ${ from } ${ userCurrency } ` ] ? . [ dateStr ] ;
const [ }
feeInAssetProfileCurrency ,
feeInBaseCurrency , if ( to === DEFAULT_CURRENCY ) {
unitPriceInAssetProfileCurrency , return exchangeRatesToDefault [ ` ${ from } ${ DEFAULT_CURRENCY } ` ] ? . [ dateStr ] ;
valueInBaseCurrency }
] = await Promise . all ( [
this . exchangeRateDataService . toCurrencyAtDate ( if ( from === DEFAULT_CURRENCY ) {
order . fee , const rateToDefault =
order . currency ? ? order . SymbolProfile . currency , exchangeRatesToDefault [ ` ${ to } ${ DEFAULT_CURRENCY } ` ] ? . [ dateStr ] ;
order . SymbolProfile . currency , return rateToDefault ? 1 / rateToDefault : undefined ;
order . date }
) ,
this . exchangeRateDataService . toCurrencyAtDate ( const rateFromToDefault =
order . fee , exchangeRatesToDefault [ ` ${ from } ${ DEFAULT_CURRENCY } ` ] ? . [ dateStr ] ;
order . currency ? ? order . SymbolProfile . currency , const rateToToDefault =
userCurrency , exchangeRatesToDefault [ ` ${ to } ${ DEFAULT_CURRENCY } ` ] ? . [ dateStr ] ;
order . date if ( rateFromToDefault !== undefined && rateToToDefault ) {
) , return rateFromToDefault / rateToToDefault ;
this . exchangeRateDataService . toCurrencyAtDate ( }
order . unitPrice ,
order . currency ? ? order . SymbolProfile . currency , return undefined ;
order . SymbolProfile . currency , } ;
order . date
) , const convertValue = async (
this . exchangeRateDataService . toCurrencyAtDate ( val : number ,
from : string ,
to : string ,
date : Date
) : Promise < number > = > {
if ( val === 0 ) {
return 0 ;
}
const dateStr = format ( date , DATE_FORMAT ) ;
const rate = getPreloadedRate ( from , to , dateStr ) ;
if ( rate !== undefined && ! isNaN ( rate ) ) {
return rate * val ;
}
return this . exchangeRateDataService . toCurrencyAtDate ( val , from , to , date ) ;
} ;
const activities = [ ] ;
const chunkSize = 500 ;
for ( let i = 0 ; i < orders . length ; i += chunkSize ) {
const chunk = orders . slice ( i , i + chunkSize ) ;
const processedChunk = await Promise . all (
chunk . map ( async ( order ) = > {
const assetProfile = assetProfiles . find ( ( { dataSource , symbol } ) = > {
return (
dataSource === order . SymbolProfile . dataSource &&
symbol === order . SymbolProfile . symbol
) ;
} ) ;
const value = new Big ( order . quantity ) . mul ( order . unitPrice ) . toNumber ( ) ;
const [
feeInAssetProfileCurrency ,
feeInBaseCurrency ,
unitPriceInAssetProfileCurrency ,
valueInBaseCurrency
] = await Promise . all ( [
convertValue (
order . fee ,
order . currency ? ? order . SymbolProfile . currency ,
order . SymbolProfile . currency ,
order . date
) ,
convertValue (
order . fee ,
order . currency ? ? order . SymbolProfile . currency ,
userCurrency ,
order . date
) ,
convertValue (
order . unitPrice ,
order . currency ? ? order . SymbolProfile . currency ,
order . SymbolProfile . currency ,
order . date
) ,
convertValue (
value ,
order . currency ? ? order . SymbolProfile . currency ,
userCurrency ,
order . date
)
] ) ;
return {
. . . order ,
feeInAssetProfileCurrency : feeInAssetProfileCurrency ? ? order . fee ,
feeInBaseCurrency : feeInBaseCurrency ? ? order . fee ,
unitPriceInAssetProfileCurrency :
unitPriceInAssetProfileCurrency ? ? order . unitPrice ,
value , value ,
order . currency ? ? order . SymbolProfile . currency , valueInBaseCurrency : valueInBaseCurrency ? ? value ,
userCurrency , SymbolProfile : assetProfile
order . date } ;
) } )
] ) ; ) ;
return { activities . push ( . . . processedChunk ) ;
. . . order , }
feeInAssetProfileCurrency ,
feeInBaseCurrency ,
unitPriceInAssetProfileCurrency ,
value ,
valueInBaseCurrency ,
SymbolProfile : assetProfile
} ;
} )
) ;
return { activities , count } ; return { activities , count } ;
} }
Andrea Bugeja
22 hours ago
GitHub