Browse Source

Merge pull request #39 from dandevaud/bugfix/several-bugfixes

Bugfix/several bugfixes
pull/5027/head
dandevaud 2 years ago
committed by GitHub
parent
commit
28281c9988
No known key found for this signature in database GPG Key ID: 4AEE18F83AFDEB23
  1. 6
      apps/api/src/app/admin/admin.service.ts
  2. 14
      apps/api/src/app/portfolio/current-rate.service.ts
  3. 23
      apps/api/src/helper/dateQueryHelper.ts
  4. 32
      apps/api/src/services/data-provider/manual/manual.service.ts
  5. 61
      apps/api/src/services/market-data/market-data.service.ts
  6. 46
      libs/common/src/lib/chunkhelper.ts

6
apps/api/src/app/admin/admin.service.ts

@ -369,13 +369,13 @@ export class AdminService {
symbolProfileId
});
} else {
symbolProfileId = await this.symbolProfileService.getSymbolProfiles([
let profiles = await this.symbolProfileService.getSymbolProfiles([
{
dataSource,
symbol
}
])[0];
]);
symbolProfileId = profiles[0].id;
await this.symbolProfileOverwriteService.add({
SymbolProfile: {
connect: {

14
apps/api/src/app/portfolio/current-rate.service.ts

@ -14,9 +14,12 @@ import { flatten, isEmpty, uniqBy } from 'lodash';
import { GetValueObject } from './interfaces/get-value-object.interface';
import { GetValuesObject } from './interfaces/get-values-object.interface';
import { GetValuesParams } from './interfaces/get-values-params.interface';
import { DateQueryHelper } from '@ghostfolio/api/helper/dateQueryHelper';
@Injectable()
export class CurrentRateService {
private dateQueryHelper = new DateQueryHelper();
public constructor(
private readonly dataProviderService: DataProviderService,
private readonly exchangeRateDataService: ExchangeRateDataService,
@ -34,7 +37,7 @@ export class CurrentRateService {
(!dateQuery.lt || isBefore(new Date(), dateQuery.lt)) &&
(!dateQuery.gte || isBefore(dateQuery.gte, new Date())) &&
(!dateQuery.in || this.containsToday(dateQuery.in));
let { query, dates } = this.dateQueryHelper.handleDateQueryIn(dateQuery);
const promises: Promise<GetValueObject[]>[] = [];
const quoteErrors: ResponseError['errors'] = [];
const today = resetHours(new Date());
@ -89,7 +92,7 @@ export class CurrentRateService {
promises.push(
this.marketDataService
.getRange({
dateQuery,
dateQuery: query,
uniqueAssets
})
.then((data) => {
@ -116,9 +119,12 @@ export class CurrentRateService {
errors: quoteErrors.map(({ dataSource, symbol }) => {
return { dataSource, symbol };
}),
values: uniqBy(values, ({ date, symbol }) => `${date}-${symbol}`)
values: uniqBy(values, ({ date, symbol }) => `${date}-${symbol}`).filter(
(v) =>
dates?.length === 0 ||
dates.some((d: Date) => d.getTime() === v.date.getTime())
)
};
if (!isEmpty(quoteErrors)) {
for (const { dataSource, symbol } of quoteErrors) {
try {

23
apps/api/src/helper/dateQueryHelper.ts

@ -0,0 +1,23 @@
import { resetHours } from '@ghostfolio/common/helper';
import { DateQuery } from '../app/portfolio/interfaces/date-query.interface';
import { addDays } from 'date-fns';
export class DateQueryHelper {
public handleDateQueryIn(dateQuery: DateQuery): {
query: DateQuery;
dates: Date[];
} {
let dates = [];
let query = dateQuery;
if (dateQuery.in?.length > 0) {
dates = dateQuery.in;
let end = Math.max(...dates.map((d) => d.getTime()));
let start = Math.min(...dates.map((d) => d.getTime()));
query = {
gte: resetHours(new Date(start)),
lt: resetHours(addDays(end, 1))
};
}
return { query, dates };
}
}

32
apps/api/src/services/data-provider/manual/manual.service.ts

@ -6,6 +6,7 @@ import {
} from '@ghostfolio/api/services/interfaces/interfaces';
import { PrismaService } from '@ghostfolio/api/services/prisma/prisma.service';
import { SymbolProfileService } from '@ghostfolio/api/services/symbol-profile/symbol-profile.service';
import { BatchPrismaClient } from '@ghostfolio/common/chunkhelper';
import { DEFAULT_REQUEST_TIMEOUT } from '@ghostfolio/common/config';
import {
DATE_FORMAT,
@ -153,18 +154,25 @@ export class ManualService implements DataProviderInterface {
})
);
const marketData = await this.prismaService.marketData.findMany({
distinct: ['symbol'],
orderBy: {
date: 'desc'
},
take: symbols.length,
where: {
symbol: {
in: symbols
}
}
});
const batch = new BatchPrismaClient(this.prismaService);
const marketData = await batch
.over(symbols)
.with((prisma, _symbols) =>
prisma.marketData.findMany({
distinct: ['symbol'],
orderBy: {
date: 'desc'
},
take: symbols.length,
where: {
symbol: {
in: _symbols
}
}
})
)
.then((_result) => _result.flat());
for (const symbolProfile of symbolProfiles) {
response[symbolProfile.symbol] = {

61
apps/api/src/services/market-data/market-data.service.ts

@ -3,6 +3,7 @@ import { DateQuery } from '@ghostfolio/api/app/portfolio/interfaces/date-query.i
import { IDataGatheringItem } from '@ghostfolio/api/services/interfaces/interfaces';
import { PrismaService } from '@ghostfolio/api/services/prisma/prisma.service';
import { resetHours } from '@ghostfolio/common/helper';
import { BatchPrismaClient } from '@ghostfolio/common/chunkhelper';
import { UniqueAsset } from '@ghostfolio/common/interfaces';
import { Injectable } from '@nestjs/common';
import {
@ -11,11 +12,14 @@ import {
MarketDataState,
Prisma
} from '@prisma/client';
import { DateQueryHelper } from '@ghostfolio/api/helper/dateQueryHelper';
@Injectable()
export class MarketDataService {
public constructor(private readonly prismaService: PrismaService) {}
private dateQueryHelper = new DateQueryHelper();
public async deleteMany({ dataSource, symbol }: UniqueAsset) {
return this.prismaService.marketData.deleteMany({
where: {
@ -64,29 +68,41 @@ export class MarketDataService {
dateQuery: DateQuery;
uniqueAssets: UniqueAsset[];
}): Promise<MarketData[]> {
return await this.prismaService.marketData.findMany({
orderBy: [
{
date: 'asc'
},
{
symbol: 'asc'
}
],
where: {
OR: uniqueAssets.map(({ dataSource, symbol }) => {
return {
AND: [
{
dataSource,
symbol,
date: dateQuery
}
]
};
const batch = new BatchPrismaClient(this.prismaService);
let { query, dates } = this.dateQueryHelper.handleDateQueryIn(dateQuery);
let marketData = await batch
.over(uniqueAssets)
.with((prisma, _assets) =>
prisma.marketData.findMany({
orderBy: [
{
date: 'asc'
},
{
symbol: 'asc'
}
],
where: {
OR: _assets.map(({ dataSource, symbol }) => {
return {
AND: [
{
dataSource,
symbol,
date: query
}
]
};
})
}
})
}
});
)
.then((data) => data.flat());
return marketData.filter(
(m) =>
dates?.length === 0 ||
dates.some((d) => m.date.getTime() === d.getTime())
);
}
public async marketDataItems(params: {
@ -97,7 +113,6 @@ export class MarketDataService {
orderBy?: Prisma.MarketDataOrderByWithRelationInput;
}): Promise<MarketData[]> {
const { skip, take, cursor, where, orderBy } = params;
return this.prismaService.marketData.findMany({
cursor,
orderBy,

46
libs/common/src/lib/chunkhelper.ts

@ -0,0 +1,46 @@
import { Prisma, PrismaClient } from '@prisma/client';
class Chunk<T> implements Iterable<T[] | undefined> {
protected constructor(
private readonly values: readonly T[],
private readonly size: number
) {}
*[Symbol.iterator]() {
const copy = [...this.values];
if (copy.length === 0) yield undefined;
while (copy.length) yield copy.splice(0, this.size);
}
map<U>(mapper: (items?: T[]) => U): U[] {
return Array.from(this).map((items) => mapper(items));
}
static of<U>(values: readonly U[]) {
return {
by: (size: number) => new Chunk(values, size)
};
}
}
export type Queryable<T, Result> = (
p: PrismaClient,
vs?: T[]
) => Prisma.PrismaPromise<Result>;
export class BatchPrismaClient {
constructor(
private readonly prisma: PrismaClient,
private readonly size = 32_000
) {}
over<T>(values: readonly T[]) {
return {
with: <Result>(queryable: Queryable<T, Result>) =>
this.prisma.$transaction(
Chunk.of(values)
.by(this.size)
.map((vs) => queryable(this.prisma, vs))
)
};
}
}
Loading…
Cancel
Save