topaLE
/
ghostfolio
mirror of https://github.com/ghostfolio/ghostfolio
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

Implement logic for asset profile gathering

pull/4524/head
Thomas Kaul 4 months ago
parent
7dad8df632
commit
529ad22ab9
10 changed files with 78 additions and 32 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 16
      apps/api/src/app/admin/admin.controller.ts
  2. 8
      apps/api/src/app/order/order.service.ts
  3. 8
      apps/api/src/services/cron.service.ts
  4. 9
      apps/api/src/services/data-provider/data-enhancer/yahoo-finance/yahoo-finance.service.ts
  5. 8
      apps/api/src/services/data-provider/data-provider.service.ts
  6. 1
      apps/api/src/services/data-provider/errors/asset-profile-delisted.error.ts
  7. 4
      apps/api/src/services/data-provider/yahoo-finance/yahoo-finance.service.ts
  8. 44
      apps/api/src/services/queues/data-gathering/data-gathering.processor.ts
  9. 8
      apps/client/src/app/components/admin-market-data/asset-profile-dialog/asset-profile-dialog.html
  10. 4
      libs/common/src/lib/config.ts

16
apps/api/src/app/admin/admin.controller.ts
View File

@ -9,8 +9,8 @@ import { DataGatheringService } from '@ghostfolio/api/services/queues/data-gathe
import {
DATA_GATHERING_QUEUE_PRIORITY_HIGH,
DATA_GATHERING_QUEUE_PRIORITY_MEDIUM,
GATHER_ASSET_PROFILE_PROCESS,
GATHER_ASSET_PROFILE_PROCESS_OPTIONS
GATHER_ASSET_PROFILE_PROCESS_JOB_NAME,
GATHER_ASSET_PROFILE_PROCESS_JOB_OPTIONS
} from '@ghostfolio/common/config';
import { getAssetProfileIdentifier } from '@ghostfolio/common/helper';
import {
@ -92,9 +92,9 @@ export class AdminController {
dataSource,
symbol
},
name: GATHER_ASSET_PROFILE_PROCESS,
name: GATHER_ASSET_PROFILE_PROCESS_JOB_NAME,
opts: {
...GATHER_ASSET_PROFILE_PROCESS_OPTIONS,
...GATHER_ASSET_PROFILE_PROCESS_JOB_OPTIONS,
jobId: getAssetProfileIdentifier({ dataSource, symbol }),
priority: DATA_GATHERING_QUEUE_PRIORITY_MEDIUM
}
@ -119,9 +119,9 @@ export class AdminController {
dataSource,
symbol
},
name: GATHER_ASSET_PROFILE_PROCESS,
name: GATHER_ASSET_PROFILE_PROCESS_JOB_NAME,
opts: {
...GATHER_ASSET_PROFILE_PROCESS_OPTIONS,
...GATHER_ASSET_PROFILE_PROCESS_JOB_OPTIONS,
jobId: getAssetProfileIdentifier({ dataSource, symbol }),
priority: DATA_GATHERING_QUEUE_PRIORITY_MEDIUM
}
@ -142,9 +142,9 @@ export class AdminController {
dataSource,
symbol
},
name: GATHER_ASSET_PROFILE_PROCESS,
name: GATHER_ASSET_PROFILE_PROCESS_JOB_NAME,
opts: {
...GATHER_ASSET_PROFILE_PROCESS_OPTIONS,
...GATHER_ASSET_PROFILE_PROCESS_JOB_OPTIONS,
jobId: getAssetProfileIdentifier({ dataSource, symbol }),
priority: DATA_GATHERING_QUEUE_PRIORITY_HIGH
}

8
apps/api/src/app/order/order.service.ts
View File

@ -7,8 +7,8 @@ import { DataGatheringService } from '@ghostfolio/api/services/queues/data-gathe
import { SymbolProfileService } from '@ghostfolio/api/services/symbol-profile/symbol-profile.service';
import {
DATA_GATHERING_QUEUE_PRIORITY_HIGH,
GATHER_ASSET_PROFILE_PROCESS,
GATHER_ASSET_PROFILE_PROCESS_OPTIONS
GATHER_ASSET_PROFILE_PROCESS_JOB_NAME,
GATHER_ASSET_PROFILE_PROCESS_JOB_OPTIONS
} from '@ghostfolio/common/config';
import { getAssetProfileIdentifier } from '@ghostfolio/common/helper';
import {
@ -144,9 +144,9 @@ export class OrderService {
dataSource: data.SymbolProfile.connectOrCreate.create.dataSource,
symbol: data.SymbolProfile.connectOrCreate.create.symbol
},
name: GATHER_ASSET_PROFILE_PROCESS,
name: GATHER_ASSET_PROFILE_PROCESS_JOB_NAME,
opts: {
...GATHER_ASSET_PROFILE_PROCESS_OPTIONS,
...GATHER_ASSET_PROFILE_PROCESS_JOB_OPTIONS,
jobId: getAssetProfileIdentifier({
dataSource: data.SymbolProfile.connectOrCreate.create.dataSource,
symbol: data.SymbolProfile.connectOrCreate.create.symbol

8
apps/api/src/services/cron.service.ts
View File

@ -1,8 +1,8 @@
import { UserService } from '@ghostfolio/api/app/user/user.service';
import {
DATA_GATHERING_QUEUE_PRIORITY_LOW,
GATHER_ASSET_PROFILE_PROCESS,
GATHER_ASSET_PROFILE_PROCESS_OPTIONS,
GATHER_ASSET_PROFILE_PROCESS_JOB_NAME,
GATHER_ASSET_PROFILE_PROCESS_JOB_OPTIONS,
PROPERTY_IS_DATA_GATHERING_ENABLED
} from '@ghostfolio/common/config';
import { getAssetProfileIdentifier } from '@ghostfolio/common/helper';
@ -66,9 +66,9 @@ export class CronService {
dataSource,
symbol
},
name: GATHER_ASSET_PROFILE_PROCESS,
name: GATHER_ASSET_PROFILE_PROCESS_JOB_NAME,
opts: {
...GATHER_ASSET_PROFILE_PROCESS_OPTIONS,
...GATHER_ASSET_PROFILE_PROCESS_JOB_OPTIONS,
jobId: getAssetProfileIdentifier({ dataSource, symbol }),
priority: DATA_GATHERING_QUEUE_PRIORITY_LOW
}

9
apps/api/src/services/data-provider/data-enhancer/yahoo-finance/yahoo-finance.service.ts
View File

@ -1,4 +1,5 @@
import { CryptocurrencyService } from '@ghostfolio/api/services/cryptocurrency/cryptocurrency.service';
import { AssetProfileDelistedError } from '@ghostfolio/api/services/data-provider/errors/asset-profile-delisted.error';
import { DataEnhancerInterface } from '@ghostfolio/api/services/data-provider/interfaces/data-enhancer.interface';
import {
DEFAULT_CURRENCY,
@ -236,7 +237,13 @@ export class YahooFinanceDataEnhancerService implements DataEnhancerInterface {
response.url = url;
}
} catch (error) {
Logger.error(error, 'YahooFinanceService');
if (error.message === `Quote not found for symbol: ${aSymbol}`) {
throw new AssetProfileDelistedError(
`No data found, ${aSymbol} (${this.getName()}) may be delisted`
);
} else {
Logger.error(error, 'YahooFinanceService');
}
}
return response;

8
apps/api/src/services/data-provider/data-provider.service.ts
View File

@ -114,7 +114,13 @@ export class DataProviderService {
}
}
await Promise.all(promises);
try {
await Promise.all(promises);
} catch (error) {
Logger.error(error, 'DataProviderService');
throw error;
}
return response;
}

1
apps/api/src/services/data-provider/errors/asset-profile-delisted.error.ts
View File

@ -1,6 +1,7 @@
export class AssetProfileDelistedError extends Error {
public constructor(message: string) {
super(message);
this.name = 'AssetProfileDelistedError';
}
}

4
apps/api/src/services/data-provider/yahoo-finance/yahoo-finance.service.ts
View File

@ -145,7 +145,9 @@ export class YahooFinanceService implements DataProviderInterface {
return response;
} catch (error) {
if (error.message === 'No data found, symbol may be delisted') {
throw new AssetProfileDelistedError(error.message);
throw new AssetProfileDelistedError(
`No data found, ${symbol} (${this.getName()}) may be delisted`
);
} else {
throw new Error(
`Could not get historical market data for ${symbol} (${this.getName()}) from ${format(

44
apps/api/src/services/queues/data-gathering/data-gathering.processor.ts
View File

@ -7,7 +7,7 @@ import {
DATA_GATHERING_QUEUE,
DEFAULT_PROCESSOR_GATHER_ASSET_PROFILE_CONCURRENCY,
DEFAULT_PROCESSOR_GATHER_HISTORICAL_MARKET_DATA_CONCURRENCY,
GATHER_ASSET_PROFILE_PROCESS,
GATHER_ASSET_PROFILE_PROCESS_JOB_NAME,
GATHER_HISTORICAL_MARKET_DATA_PROCESS_JOB_NAME
} from '@ghostfolio/common/config';
import { DATE_FORMAT, getStartOfUtcDate } from '@ghostfolio/common/helper';
@ -45,28 +45,49 @@ export class DataGatheringProcessor {
DEFAULT_PROCESSOR_GATHER_ASSET_PROFILE_CONCURRENCY.toString(),
10
),
name: GATHER_ASSET_PROFILE_PROCESS
name: GATHER_ASSET_PROFILE_PROCESS_JOB_NAME
})
public async gatherAssetProfile(job: Job<AssetProfileIdentifier>) {
const { dataSource, symbol } = job.data;
try {
Logger.log(
`Asset profile data gathering has been started for ${job.data.symbol} (${job.data.dataSource})`,
`DataGatheringProcessor (${GATHER_ASSET_PROFILE_PROCESS})`
`Asset profile data gathering has been started for ${symbol} (${dataSource})`,
`DataGatheringProcessor (${GATHER_ASSET_PROFILE_PROCESS_JOB_NAME})`
);
await this.dataGatheringService.gatherAssetProfiles([job.data]);
Logger.log(
`Asset profile data gathering has been completed for ${job.data.symbol} (${job.data.dataSource})`,
`DataGatheringProcessor (${GATHER_ASSET_PROFILE_PROCESS})`
`Asset profile data gathering has been completed for ${symbol} (${dataSource})`,
`DataGatheringProcessor (${GATHER_ASSET_PROFILE_PROCESS_JOB_NAME})`
);
} catch (error) {
if (error instanceof AssetProfileDelistedError) {
await this.symbolProfileService.updateSymbolProfile(
{
dataSource,
symbol
},
{
isActive: false
}
);
Logger.log(
`Asset profile data gathering has been discarded for ${symbol} (${dataSource})`,
`DataGatheringProcessor (${GATHER_ASSET_PROFILE_PROCESS_JOB_NAME})`
);
return job.discard();
}
Logger.error(
error,
`DataGatheringProcessor (${GATHER_ASSET_PROFILE_PROCESS})`
`DataGatheringProcessor (${GATHER_ASSET_PROFILE_PROCESS_JOB_NAME})`
);
throw new Error(error);
throw error;
}
}
@ -157,7 +178,12 @@ export class DataGatheringProcessor {
}
);
await job.discard();
Logger.log(
`Historical market data gathering has been discarded for ${symbol} (${dataSource})`,
`DataGatheringProcessor (${GATHER_HISTORICAL_MARKET_DATA_PROCESS_JOB_NAME})`
);
return job.discard();
}
Logger.error(

8
apps/client/src/app/components/admin-market-data/asset-profile-dialog/asset-profile-dialog.html
View File

@ -19,7 +19,9 @@
<button
mat-menu-item
type="button"
[disabled]="assetProfileForm.dirty"
[disabled]="
assetProfileForm.dirty || !assetProfileForm.controls.isActive.value
"
(click)="
onGatherSymbol({ dataSource: data.dataSource, symbol: data.symbol })
"
@ -29,7 +31,9 @@
<button
mat-menu-item
type="button"
[disabled]="assetProfileForm.dirty"
[disabled]="
assetProfileForm.dirty || !assetProfileForm.controls.isActive.value
"
(click)="
onGatherProfileDataBySymbol({
dataSource: data.dataSource,

4
libs/common/src/lib/config.ts
View File

@ -78,8 +78,8 @@ export const DERIVED_CURRENCIES = [
export const EMERGENCY_FUND_TAG_ID = '4452656d-9fa4-4bd0-ba38-70492e31d180';
export const GATHER_ASSET_PROFILE_PROCESS = 'GATHER_ASSET_PROFILE';
export const GATHER_ASSET_PROFILE_PROCESS_OPTIONS: JobOptions = {
export const GATHER_ASSET_PROFILE_PROCESS_JOB_NAME = 'GATHER_ASSET_PROFILE';
export const GATHER_ASSET_PROFILE_PROCESS_JOB_OPTIONS: JobOptions = {
attempts: 12,
backoff: {
delay: ms('1 minute'),

Write Preview
Loading…
Cancel
Save