Browse Source

Migrate to yahoo-finance2 (part 2)

pull/722/head
Thomas 3 years ago
parent
commit
bec72090c0
  1. 2
      apps/api/src/app/portfolio/current-rate.service.ts
  2. 7
      apps/api/src/app/portfolio/portfolio.service-new.ts
  3. 12
      apps/api/src/app/portfolio/portfolio.service.ts
  4. 6
      apps/api/src/app/symbol/symbol.service.ts
  5. 12
      apps/api/src/services/data-gathering.service.ts
  6. 26
      apps/api/src/services/data-provider/alpha-vantage/alpha-vantage.service.ts
  7. 17
      apps/api/src/services/data-provider/data-enhancer/trackinsight/trackinsight.service.ts
  8. 118
      apps/api/src/services/data-provider/data-provider.service.ts
  9. 76
      apps/api/src/services/data-provider/ghostfolio-scraper-api/ghostfolio-scraper-api.service.ts
  10. 94
      apps/api/src/services/data-provider/google-sheets/google-sheets.service.ts
  11. 6
      apps/api/src/services/data-provider/interfaces/data-enhancer.interface.ts
  12. 8
      apps/api/src/services/data-provider/interfaces/data-provider.interface.ts
  13. 18
      apps/api/src/services/data-provider/manual/manual.service.ts
  14. 71
      apps/api/src/services/data-provider/rakuten-rapid-api/rakuten-rapid-api.service.ts
  15. 21
      apps/api/src/services/data-provider/yahoo-finance/interfaces/interfaces.ts
  16. 190
      apps/api/src/services/data-provider/yahoo-finance/yahoo-finance.service.ts
  17. 2
      apps/api/src/services/exchange-rate-data.service.ts
  18. 9
      apps/api/src/services/interfaces/interfaces.ts
  19. 5
      apps/client/src/app/components/position/position.component.html
  20. 3
      package.json
  21. 8
      yarn.lock

2
apps/api/src/app/portfolio/current-rate.service.ts

@ -40,7 +40,7 @@ export class CurrentRateService {
const today = resetHours(new Date()); const today = resetHours(new Date());
promises.push( promises.push(
this.dataProviderService this.dataProviderService
.get(dataGatheringItems) .getQuotes(dataGatheringItems)
.then((dataResultProvider) => { .then((dataResultProvider) => {
const result = []; const result = [];
for (const dataGatheringItem of dataGatheringItems) { for (const dataGatheringItem of dataGatheringItems) {

7
apps/api/src/app/portfolio/portfolio.service-new.ts

@ -327,7 +327,7 @@ export class PortfolioServiceNew {
); );
const [dataProviderResponses, symbolProfiles] = await Promise.all([ const [dataProviderResponses, symbolProfiles] = await Promise.all([
this.dataProviderService.get(dataGatheringItems), this.dataProviderService.getQuotes(dataGatheringItems),
this.symbolProfileService.getSymbolProfiles(symbols) this.symbolProfileService.getSymbolProfiles(symbols)
]); ]);
@ -358,7 +358,6 @@ export class PortfolioServiceNew {
countries: symbolProfile.countries, countries: symbolProfile.countries,
currency: item.currency, currency: item.currency,
dataSource: symbolProfile.dataSource, dataSource: symbolProfile.dataSource,
exchange: dataProviderResponse.exchange,
grossPerformance: item.grossPerformance?.toNumber() ?? 0, grossPerformance: item.grossPerformance?.toNumber() ?? 0,
grossPerformancePercent: grossPerformancePercent:
item.grossPerformancePercentage?.toNumber() ?? 0, item.grossPerformancePercentage?.toNumber() ?? 0,
@ -578,7 +577,7 @@ export class PortfolioServiceNew {
) )
}; };
} else { } else {
const currentData = await this.dataProviderService.get([ const currentData = await this.dataProviderService.getQuotes([
{ dataSource: DataSource.YAHOO, symbol: aSymbol } { dataSource: DataSource.YAHOO, symbol: aSymbol }
]); ]);
const marketPrice = currentData[aSymbol]?.marketPrice; const marketPrice = currentData[aSymbol]?.marketPrice;
@ -679,7 +678,7 @@ export class PortfolioServiceNew {
const symbols = positions.map((position) => position.symbol); const symbols = positions.map((position) => position.symbol);
const [dataProviderResponses, symbolProfiles] = await Promise.all([ const [dataProviderResponses, symbolProfiles] = await Promise.all([
this.dataProviderService.get(dataGatheringItem), this.dataProviderService.getQuotes(dataGatheringItem),
this.symbolProfileService.getSymbolProfiles(symbols) this.symbolProfileService.getSymbolProfiles(symbols)
]); ]);

12
apps/api/src/app/portfolio/portfolio.service.ts

@ -315,7 +315,7 @@ export class PortfolioService {
); );
const [dataProviderResponses, symbolProfiles] = await Promise.all([ const [dataProviderResponses, symbolProfiles] = await Promise.all([
this.dataProviderService.get(dataGatheringItems), this.dataProviderService.getQuotes(dataGatheringItems),
this.symbolProfileService.getSymbolProfiles(symbols) this.symbolProfileService.getSymbolProfiles(symbols)
]); ]);
@ -346,7 +346,6 @@ export class PortfolioService {
countries: symbolProfile.countries, countries: symbolProfile.countries,
currency: item.currency, currency: item.currency,
dataSource: symbolProfile.dataSource, dataSource: symbolProfile.dataSource,
exchange: dataProviderResponse.exchange,
grossPerformance: item.grossPerformance?.toNumber() ?? 0, grossPerformance: item.grossPerformance?.toNumber() ?? 0,
grossPerformancePercent: grossPerformancePercent:
item.grossPerformancePercentage?.toNumber() ?? 0, item.grossPerformancePercentage?.toNumber() ?? 0,
@ -552,9 +551,10 @@ export class PortfolioService {
SymbolProfile, SymbolProfile,
transactionCount, transactionCount,
averagePrice: averagePrice.toNumber(), averagePrice: averagePrice.toNumber(),
grossPerformancePercent: position.grossPerformancePercentage.toNumber(), grossPerformancePercent:
position.grossPerformancePercentage?.toNumber(),
historicalData: historicalDataArray, historicalData: historicalDataArray,
netPerformancePercent: position.netPerformancePercentage.toNumber(), netPerformancePercent: position.netPerformancePercentage?.toNumber(),
quantity: quantity.toNumber(), quantity: quantity.toNumber(),
value: this.exchangeRateDataService.toCurrency( value: this.exchangeRateDataService.toCurrency(
quantity.mul(marketPrice).toNumber(), quantity.mul(marketPrice).toNumber(),
@ -563,7 +563,7 @@ export class PortfolioService {
) )
}; };
} else { } else {
const currentData = await this.dataProviderService.get([ const currentData = await this.dataProviderService.getQuotes([
{ dataSource: DataSource.YAHOO, symbol: aSymbol } { dataSource: DataSource.YAHOO, symbol: aSymbol }
]); ]);
const marketPrice = currentData[aSymbol]?.marketPrice; const marketPrice = currentData[aSymbol]?.marketPrice;
@ -660,7 +660,7 @@ export class PortfolioService {
const symbols = positions.map((position) => position.symbol); const symbols = positions.map((position) => position.symbol);
const [dataProviderResponses, symbolProfiles] = await Promise.all([ const [dataProviderResponses, symbolProfiles] = await Promise.all([
this.dataProviderService.get(dataGatheringItem), this.dataProviderService.getQuotes(dataGatheringItem),
this.symbolProfileService.getSymbolProfiles(symbols) this.symbolProfileService.getSymbolProfiles(symbols)
]); ]);

6
apps/api/src/app/symbol/symbol.service.ts

@ -27,8 +27,10 @@ export class SymbolService {
dataGatheringItem: IDataGatheringItem; dataGatheringItem: IDataGatheringItem;
includeHistoricalData?: number; includeHistoricalData?: number;
}): Promise<SymbolItem> { }): Promise<SymbolItem> {
const response = await this.dataProviderService.get([dataGatheringItem]); const quotes = await this.dataProviderService.getQuotes([
const { currency, marketPrice } = response[dataGatheringItem.symbol] ?? {}; dataGatheringItem
]);
const { currency, marketPrice } = quotes[dataGatheringItem.symbol] ?? {};
if (dataGatheringItem.dataSource && marketPrice) { if (dataGatheringItem.dataSource && marketPrice) {
let historicalData: HistoricalDataItem[] = []; let historicalData: HistoricalDataItem[] = [];

12
apps/api/src/services/data-gathering.service.ts

@ -226,22 +226,24 @@ export class DataGatheringService {
dataGatheringItems = await this.getSymbolsProfileData(); dataGatheringItems = await this.getSymbolsProfileData();
} }
const currentData = await this.dataProviderService.get(dataGatheringItems); const assetProfiles = await this.dataProviderService.getAssetProfiles(
dataGatheringItems
);
const symbolProfiles = await this.symbolProfileService.getSymbolProfiles( const symbolProfiles = await this.symbolProfileService.getSymbolProfiles(
dataGatheringItems.map(({ symbol }) => { dataGatheringItems.map(({ symbol }) => {
return symbol; return symbol;
}) })
); );
for (const [symbol, response] of Object.entries(currentData)) { for (const [symbol, assetProfile] of Object.entries(assetProfiles)) {
const symbolMapping = symbolProfiles.find((symbolProfile) => { const symbolMapping = symbolProfiles.find((symbolProfile) => {
return symbolProfile.symbol === symbol; return symbolProfile.symbol === symbol;
})?.symbolMapping; })?.symbolMapping;
for (const dataEnhancer of this.dataEnhancers) { for (const dataEnhancer of this.dataEnhancers) {
try { try {
currentData[symbol] = await dataEnhancer.enhance({ assetProfiles[symbol] = await dataEnhancer.enhance({
response, response: assetProfile,
symbol: symbolMapping?.[dataEnhancer.getName()] ?? symbol symbol: symbolMapping?.[dataEnhancer.getName()] ?? symbol
}); });
} catch (error) { } catch (error) {
@ -257,7 +259,7 @@ export class DataGatheringService {
dataSource, dataSource,
name, name,
sectors sectors
} = currentData[symbol]; } = assetProfiles[symbol];
try { try {
await this.prismaService.symbolProfile.upsert({ await this.prismaService.symbolProfile.upsert({

26
apps/api/src/services/data-provider/alpha-vantage/alpha-vantage.service.ts

@ -1,15 +1,15 @@
import { LookupItem } from '@ghostfolio/api/app/symbol/interfaces/lookup-item.interface'; import { LookupItem } from '@ghostfolio/api/app/symbol/interfaces/lookup-item.interface';
import { ConfigurationService } from '@ghostfolio/api/services/configuration.service'; import { ConfigurationService } from '@ghostfolio/api/services/configuration.service';
import {
IDataProviderHistoricalResponse,
IDataProviderResponse
} from '@ghostfolio/api/services/interfaces/interfaces';
import { DATE_FORMAT } from '@ghostfolio/common/helper'; import { DATE_FORMAT } from '@ghostfolio/common/helper';
import { Granularity } from '@ghostfolio/common/types'; import { Granularity } from '@ghostfolio/common/types';
import { Injectable, Logger } from '@nestjs/common'; import { Injectable, Logger } from '@nestjs/common';
import { DataSource } from '@prisma/client'; import { DataSource, SymbolProfile } from '@prisma/client';
import { isAfter, isBefore, parse } from 'date-fns'; import { isAfter, isBefore, parse } from 'date-fns';
import {
IDataProviderHistoricalResponse,
IDataProviderResponse
} from '../../interfaces/interfaces';
import { DataProviderInterface } from '../interfaces/data-provider.interface'; import { DataProviderInterface } from '../interfaces/data-provider.interface';
import { IAlphaVantageHistoricalResponse } from './interfaces/interfaces'; import { IAlphaVantageHistoricalResponse } from './interfaces/interfaces';
@ -29,10 +29,12 @@ export class AlphaVantageService implements DataProviderInterface {
return !!this.configurationService.get('ALPHA_VANTAGE_API_KEY'); return !!this.configurationService.get('ALPHA_VANTAGE_API_KEY');
} }
public async get( public async getAssetProfile(
aSymbols: string[] aSymbol: string
): Promise<{ [symbol: string]: IDataProviderResponse }> { ): Promise<Partial<SymbolProfile>> {
return {}; return {
dataSource: this.getName()
};
} }
public async getHistorical( public async getHistorical(
@ -84,6 +86,12 @@ export class AlphaVantageService implements DataProviderInterface {
return DataSource.ALPHA_VANTAGE; return DataSource.ALPHA_VANTAGE;
} }
public async getQuotes(
aSymbols: string[]
): Promise<{ [symbol: string]: IDataProviderResponse }> {
return {};
}
public async search(aQuery: string): Promise<{ items: LookupItem[] }> { public async search(aQuery: string): Promise<{ items: LookupItem[] }> {
const result = await this.alphaVantage.data.search(aQuery); const result = await this.alphaVantage.data.search(aQuery);

17
apps/api/src/services/data-provider/data-enhancer/trackinsight/trackinsight.service.ts

@ -1,5 +1,8 @@
import { DataEnhancerInterface } from '@ghostfolio/api/services/data-provider/interfaces/data-enhancer.interface'; import { DataEnhancerInterface } from '@ghostfolio/api/services/data-provider/interfaces/data-enhancer.interface';
import { IDataProviderResponse } from '@ghostfolio/api/services/interfaces/interfaces'; import { IDataProviderResponse } from '@ghostfolio/api/services/interfaces/interfaces';
import { Country } from '@ghostfolio/common/interfaces/country.interface';
import { Sector } from '@ghostfolio/common/interfaces/sector.interface';
import { SymbolProfile } from '@prisma/client';
import bent from 'bent'; import bent from 'bent';
const getJSON = bent('json'); const getJSON = bent('json');
@ -21,9 +24,9 @@ export class TrackinsightDataEnhancerService implements DataEnhancerInterface {
response, response,
symbol symbol
}: { }: {
response: IDataProviderResponse; response: Partial<SymbolProfile>;
symbol: string; symbol: string;
}): Promise<IDataProviderResponse> { }): Promise<Partial<SymbolProfile>> {
if ( if (
!(response.assetClass === 'EQUITY' && response.assetSubClass === 'ETF') !(response.assetClass === 'EQUITY' && response.assetSubClass === 'ETF')
) { ) {
@ -40,7 +43,10 @@ export class TrackinsightDataEnhancerService implements DataEnhancerInterface {
); );
}); });
if (!response.countries || response.countries.length === 0) { if (
!response.countries ||
(response.countries as unknown as Country[]).length === 0
) {
response.countries = []; response.countries = [];
for (const [name, value] of Object.entries<any>(holdings.countries)) { for (const [name, value] of Object.entries<any>(holdings.countries)) {
let countryCode: string; let countryCode: string;
@ -65,7 +71,10 @@ export class TrackinsightDataEnhancerService implements DataEnhancerInterface {
} }
} }
if (!response.sectors || response.sectors.length === 0) { if (
!response.sectors ||
(response.sectors as unknown as Sector[]).length === 0
) {
response.sectors = []; response.sectors = [];
for (const [name, value] of Object.entries<any>(holdings.sectors)) { for (const [name, value] of Object.entries<any>(holdings.sectors)) {
response.sectors.push({ response.sectors.push({

118
apps/api/src/services/data-provider/data-provider.service.ts

@ -10,7 +10,7 @@ import { PrismaService } from '@ghostfolio/api/services/prisma.service';
import { DATE_FORMAT } from '@ghostfolio/common/helper'; import { DATE_FORMAT } from '@ghostfolio/common/helper';
import { Granularity } from '@ghostfolio/common/types'; import { Granularity } from '@ghostfolio/common/types';
import { Inject, Injectable, Logger } from '@nestjs/common'; import { Inject, Injectable, Logger } from '@nestjs/common';
import { DataSource, MarketData } from '@prisma/client'; import { DataSource, MarketData, SymbolProfile } from '@prisma/client';
import { format, isValid } from 'date-fns'; import { format, isValid } from 'date-fns';
import { groupBy, isEmpty } from 'lodash'; import { groupBy, isEmpty } from 'lodash';
@ -23,42 +23,6 @@ export class DataProviderService {
private readonly prismaService: PrismaService private readonly prismaService: PrismaService
) {} ) {}
public async get(items: IDataGatheringItem[]): Promise<{
[symbol: string]: IDataProviderResponse;
}> {
const response: {
[symbol: string]: IDataProviderResponse;
} = {};
const itemsGroupedByDataSource = groupBy(items, (item) => item.dataSource);
const promises = [];
for (const [dataSource, dataGatheringItems] of Object.entries(
itemsGroupedByDataSource
)) {
const symbols = dataGatheringItems.map((dataGatheringItem) => {
return dataGatheringItem.symbol;
});
const promise = Promise.resolve(
this.getDataProvider(DataSource[dataSource]).get(symbols)
);
promises.push(
promise.then((result) => {
for (const [symbol, dataProviderResponse] of Object.entries(result)) {
response[symbol] = dataProviderResponse;
}
})
);
}
await Promise.all(promises);
return response;
}
public async getHistorical( public async getHistorical(
aItems: IDataGatheringItem[], aItems: IDataGatheringItem[],
aGranularity: Granularity = 'month', aGranularity: Granularity = 'month',
@ -158,6 +122,82 @@ export class DataProviderService {
return result; return result;
} }
public getPrimaryDataSource(): DataSource {
return DataSource[this.configurationService.get('DATA_SOURCE_PRIMARY')];
}
public async getAssetProfiles(items: IDataGatheringItem[]): Promise<{
[symbol: string]: Partial<SymbolProfile>;
}> {
const response: {
[symbol: string]: Partial<SymbolProfile>;
} = {};
const itemsGroupedByDataSource = groupBy(items, (item) => item.dataSource);
const promises = [];
for (const [dataSource, dataGatheringItems] of Object.entries(
itemsGroupedByDataSource
)) {
const symbols = dataGatheringItems.map((dataGatheringItem) => {
return dataGatheringItem.symbol;
});
for (const symbol of symbols) {
const promise = Promise.resolve(
this.getDataProvider(DataSource[dataSource]).getAssetProfile(symbol)
);
promises.push(
promise.then((symbolProfile) => {
response[symbol] = symbolProfile;
})
);
}
}
await Promise.all(promises);
return response;
}
public async getQuotes(items: IDataGatheringItem[]): Promise<{
[symbol: string]: IDataProviderResponse;
}> {
const response: {
[symbol: string]: IDataProviderResponse;
} = {};
const itemsGroupedByDataSource = groupBy(items, (item) => item.dataSource);
const promises = [];
for (const [dataSource, dataGatheringItems] of Object.entries(
itemsGroupedByDataSource
)) {
const symbols = dataGatheringItems.map((dataGatheringItem) => {
return dataGatheringItem.symbol;
});
const promise = Promise.resolve(
this.getDataProvider(DataSource[dataSource]).getQuotes(symbols)
);
promises.push(
promise.then((result) => {
for (const [symbol, dataProviderResponse] of Object.entries(result)) {
response[symbol] = dataProviderResponse;
}
})
);
}
await Promise.all(promises);
return response;
}
public async search(aQuery: string): Promise<{ items: LookupItem[] }> { public async search(aQuery: string): Promise<{ items: LookupItem[] }> {
const promises: Promise<{ items: LookupItem[] }>[] = []; const promises: Promise<{ items: LookupItem[] }>[] = [];
let lookupItems: LookupItem[] = []; let lookupItems: LookupItem[] = [];
@ -184,10 +224,6 @@ export class DataProviderService {
}; };
} }
public getPrimaryDataSource(): DataSource {
return DataSource[this.configurationService.get('DATA_SOURCE_PRIMARY')];
}
private getDataProvider(providerName: DataSource) { private getDataProvider(providerName: DataSource) {
for (const dataProviderInterface of this.dataProviderInterfaces) { for (const dataProviderInterface of this.dataProviderInterfaces) {
if (dataProviderInterface.getName() === providerName) { if (dataProviderInterface.getName() === providerName) {

76
apps/api/src/services/data-provider/ghostfolio-scraper-api/ghostfolio-scraper-api.service.ts

@ -14,7 +14,7 @@ import {
} from '@ghostfolio/common/helper'; } from '@ghostfolio/common/helper';
import { Granularity } from '@ghostfolio/common/types'; import { Granularity } from '@ghostfolio/common/types';
import { Injectable, Logger } from '@nestjs/common'; import { Injectable, Logger } from '@nestjs/common';
import { DataSource } from '@prisma/client'; import { DataSource, SymbolProfile } from '@prisma/client';
import * as bent from 'bent'; import * as bent from 'bent';
import * as cheerio from 'cheerio'; import * as cheerio from 'cheerio';
import { format } from 'date-fns'; import { format } from 'date-fns';
@ -32,41 +32,12 @@ export class GhostfolioScraperApiService implements DataProviderInterface {
return isGhostfolioScraperApiSymbol(symbol); return isGhostfolioScraperApiSymbol(symbol);
} }
public async get( public async getAssetProfile(
aSymbols: string[] aSymbol: string
): Promise<{ [symbol: string]: IDataProviderResponse }> { ): Promise<Partial<SymbolProfile>> {
if (aSymbols.length <= 0) {
return {};
}
try {
const [symbol] = aSymbols;
const [symbolProfile] = await this.symbolProfileService.getSymbolProfiles(
[symbol]
);
const { marketPrice } = await this.prismaService.marketData.findFirst({
orderBy: {
date: 'desc'
},
where: {
symbol
}
});
return { return {
[symbol]: { dataSource: this.getName()
marketPrice,
currency: symbolProfile?.currency,
dataSource: this.getName(),
marketState: MarketState.delayed
}
}; };
} catch (error) {
Logger.error(error);
}
return {};
} }
public async getHistorical( public async getHistorical(
@ -112,6 +83,43 @@ export class GhostfolioScraperApiService implements DataProviderInterface {
return DataSource.GHOSTFOLIO; return DataSource.GHOSTFOLIO;
} }
public async getQuotes(
aSymbols: string[]
): Promise<{ [symbol: string]: IDataProviderResponse }> {
if (aSymbols.length <= 0) {
return {};
}
try {
const [symbol] = aSymbols;
const [symbolProfile] = await this.symbolProfileService.getSymbolProfiles(
[symbol]
);
const { marketPrice } = await this.prismaService.marketData.findFirst({
orderBy: {
date: 'desc'
},
where: {
symbol
}
});
return {
[symbol]: {
marketPrice,
currency: symbolProfile?.currency,
dataSource: this.getName(),
marketState: MarketState.delayed
}
};
} catch (error) {
Logger.error(error);
}
return {};
}
public async search(aQuery: string): Promise<{ items: LookupItem[] }> { public async search(aQuery: string): Promise<{ items: LookupItem[] }> {
const items = await this.prismaService.symbolProfile.findMany({ const items = await this.prismaService.symbolProfile.findMany({
select: { select: {

94
apps/api/src/services/data-provider/google-sheets/google-sheets.service.ts

@ -11,7 +11,7 @@ import { SymbolProfileService } from '@ghostfolio/api/services/symbol-profile.se
import { DATE_FORMAT, parseDate } from '@ghostfolio/common/helper'; import { DATE_FORMAT, parseDate } from '@ghostfolio/common/helper';
import { Granularity } from '@ghostfolio/common/types'; import { Granularity } from '@ghostfolio/common/types';
import { Injectable, Logger } from '@nestjs/common'; import { Injectable, Logger } from '@nestjs/common';
import { DataSource } from '@prisma/client'; import { DataSource, SymbolProfile } from '@prisma/client';
import { format } from 'date-fns'; import { format } from 'date-fns';
import { GoogleSpreadsheet } from 'google-spreadsheet'; import { GoogleSpreadsheet } from 'google-spreadsheet';
@ -27,50 +27,13 @@ export class GoogleSheetsService implements DataProviderInterface {
return true; return true;
} }
public async get( public async getAssetProfile(
aSymbols: string[] aSymbol: string
): Promise<{ [symbol: string]: IDataProviderResponse }> { ): Promise<Partial<SymbolProfile>> {
if (aSymbols.length <= 0) { return {
return {}; dataSource: this.getName()
}
try {
const response: { [symbol: string]: IDataProviderResponse } = {};
const symbolProfiles = await this.symbolProfileService.getSymbolProfiles(
aSymbols
);
const sheet = await this.getSheet({
sheetId: this.configurationService.get('GOOGLE_SHEETS_ID'),
symbol: 'Overview'
});
const rows = await sheet.getRows();
for (const row of rows) {
const marketPrice = parseFloat(row['marketPrice']);
const symbol = row['symbol'];
if (aSymbols.includes(symbol)) {
response[symbol] = {
marketPrice,
currency: symbolProfiles.find((symbolProfile) => {
return symbolProfile.symbol === symbol;
})?.currency,
dataSource: this.getName(),
marketState: MarketState.delayed
}; };
} }
}
return response;
} catch (error) {
Logger.error(error);
}
return {};
}
public async getHistorical( public async getHistorical(
aSymbol: string, aSymbol: string,
@ -119,6 +82,51 @@ export class GoogleSheetsService implements DataProviderInterface {
return DataSource.GOOGLE_SHEETS; return DataSource.GOOGLE_SHEETS;
} }
public async getQuotes(
aSymbols: string[]
): Promise<{ [symbol: string]: IDataProviderResponse }> {
if (aSymbols.length <= 0) {
return {};
}
try {
const response: { [symbol: string]: IDataProviderResponse } = {};
const symbolProfiles = await this.symbolProfileService.getSymbolProfiles(
aSymbols
);
const sheet = await this.getSheet({
sheetId: this.configurationService.get('GOOGLE_SHEETS_ID'),
symbol: 'Overview'
});
const rows = await sheet.getRows();
for (const row of rows) {
const marketPrice = parseFloat(row['marketPrice']);
const symbol = row['symbol'];
if (aSymbols.includes(symbol)) {
response[symbol] = {
marketPrice,
currency: symbolProfiles.find((symbolProfile) => {
return symbolProfile.symbol === symbol;
})?.currency,
dataSource: this.getName(),
marketState: MarketState.delayed
};
}
}
return response;
} catch (error) {
Logger.error(error);
}
return {};
}
public async search(aQuery: string): Promise<{ items: LookupItem[] }> { public async search(aQuery: string): Promise<{ items: LookupItem[] }> {
const items = await this.prismaService.symbolProfile.findMany({ const items = await this.prismaService.symbolProfile.findMany({
select: { select: {

6
apps/api/src/services/data-provider/interfaces/data-enhancer.interface.ts

@ -1,13 +1,13 @@
import { IDataProviderResponse } from '@ghostfolio/api/services/interfaces/interfaces'; import { SymbolProfile } from '@prisma/client';
export interface DataEnhancerInterface { export interface DataEnhancerInterface {
enhance({ enhance({
response, response,
symbol symbol
}: { }: {
response: IDataProviderResponse; response: Partial<SymbolProfile>;
symbol: string; symbol: string;
}): Promise<IDataProviderResponse>; }): Promise<Partial<SymbolProfile>>;
getName(): string; getName(): string;
} }

8
apps/api/src/services/data-provider/interfaces/data-provider.interface.ts

@ -4,12 +4,12 @@ import {
IDataProviderResponse IDataProviderResponse
} from '@ghostfolio/api/services/interfaces/interfaces'; } from '@ghostfolio/api/services/interfaces/interfaces';
import { Granularity } from '@ghostfolio/common/types'; import { Granularity } from '@ghostfolio/common/types';
import { DataSource } from '@prisma/client'; import { DataSource, SymbolProfile } from '@prisma/client';
export interface DataProviderInterface { export interface DataProviderInterface {
canHandle(symbol: string): boolean; canHandle(symbol: string): boolean;
get(aSymbols: string[]): Promise<{ [symbol: string]: IDataProviderResponse }>; getAssetProfile(aSymbol: string): Promise<Partial<SymbolProfile>>;
getHistorical( getHistorical(
aSymbol: string, aSymbol: string,
@ -22,5 +22,9 @@ export interface DataProviderInterface {
getName(): DataSource; getName(): DataSource;
getQuotes(
aSymbols: string[]
): Promise<{ [symbol: string]: IDataProviderResponse }>;
search(aQuery: string): Promise<{ items: LookupItem[] }>; search(aQuery: string): Promise<{ items: LookupItem[] }>;
} }

18
apps/api/src/services/data-provider/manual/manual.service.ts

@ -6,7 +6,7 @@ import {
} from '@ghostfolio/api/services/interfaces/interfaces'; } from '@ghostfolio/api/services/interfaces/interfaces';
import { Granularity } from '@ghostfolio/common/types'; import { Granularity } from '@ghostfolio/common/types';
import { Injectable } from '@nestjs/common'; import { Injectable } from '@nestjs/common';
import { DataSource } from '@prisma/client'; import { DataSource, SymbolProfile } from '@prisma/client';
@Injectable() @Injectable()
export class ManualService implements DataProviderInterface { export class ManualService implements DataProviderInterface {
@ -16,10 +16,12 @@ export class ManualService implements DataProviderInterface {
return false; return false;
} }
public async get( public async getAssetProfile(
aSymbols: string[] aSymbol: string
): Promise<{ [symbol: string]: IDataProviderResponse }> { ): Promise<Partial<SymbolProfile>> {
return {}; return {
dataSource: this.getName()
};
} }
public async getHistorical( public async getHistorical(
@ -37,6 +39,12 @@ export class ManualService implements DataProviderInterface {
return DataSource.MANUAL; return DataSource.MANUAL;
} }
public async getQuotes(
aSymbols: string[]
): Promise<{ [symbol: string]: IDataProviderResponse }> {
return {};
}
public async search(aQuery: string): Promise<{ items: LookupItem[] }> { public async search(aQuery: string): Promise<{ items: LookupItem[] }> {
return { items: [] }; return { items: [] };
} }

71
apps/api/src/services/data-provider/rakuten-rapid-api/rakuten-rapid-api.service.ts

@ -1,19 +1,19 @@
import { LookupItem } from '@ghostfolio/api/app/symbol/interfaces/lookup-item.interface'; import { LookupItem } from '@ghostfolio/api/app/symbol/interfaces/lookup-item.interface';
import { ConfigurationService } from '@ghostfolio/api/services/configuration.service'; import { ConfigurationService } from '@ghostfolio/api/services/configuration.service';
import {
IDataProviderHistoricalResponse,
IDataProviderResponse,
MarketState
} from '@ghostfolio/api/services/interfaces/interfaces';
import { PrismaService } from '@ghostfolio/api/services/prisma.service'; import { PrismaService } from '@ghostfolio/api/services/prisma.service';
import { ghostfolioFearAndGreedIndexSymbol } from '@ghostfolio/common/config'; import { ghostfolioFearAndGreedIndexSymbol } from '@ghostfolio/common/config';
import { DATE_FORMAT, getToday, getYesterday } from '@ghostfolio/common/helper'; import { DATE_FORMAT, getToday, getYesterday } from '@ghostfolio/common/helper';
import { Granularity } from '@ghostfolio/common/types'; import { Granularity } from '@ghostfolio/common/types';
import { Injectable, Logger } from '@nestjs/common'; import { Injectable, Logger } from '@nestjs/common';
import { DataSource } from '@prisma/client'; import { DataSource, SymbolProfile } from '@prisma/client';
import * as bent from 'bent'; import * as bent from 'bent';
import { format, subMonths, subWeeks, subYears } from 'date-fns'; import { format, subMonths, subWeeks, subYears } from 'date-fns';
import {
IDataProviderHistoricalResponse,
IDataProviderResponse,
MarketState
} from '../../interfaces/interfaces';
import { DataProviderInterface } from '../interfaces/data-provider.interface'; import { DataProviderInterface } from '../interfaces/data-provider.interface';
@Injectable() @Injectable()
@ -29,35 +29,13 @@ export class RakutenRapidApiService implements DataProviderInterface {
return !!this.configurationService.get('RAKUTEN_RAPID_API_KEY'); return !!this.configurationService.get('RAKUTEN_RAPID_API_KEY');
} }
public async get( public async getAssetProfile(
aSymbols: string[] aSymbol: string
): Promise<{ [symbol: string]: IDataProviderResponse }> { ): Promise<Partial<SymbolProfile>> {
if (aSymbols.length <= 0) {
return {};
}
try {
const symbol = aSymbols[0];
if (symbol === ghostfolioFearAndGreedIndexSymbol) {
const fgi = await this.getFearAndGreedIndex();
return { return {
[ghostfolioFearAndGreedIndexSymbol]: { dataSource: this.getName()
currency: undefined,
dataSource: this.getName(),
marketPrice: fgi.now.value,
marketState: MarketState.open,
name: RakutenRapidApiService.FEAR_AND_GREED_INDEX_NAME
}
}; };
} }
} catch (error) {
Logger.error(error);
}
return {};
}
public async getHistorical( public async getHistorical(
aSymbol: string, aSymbol: string,
@ -125,6 +103,35 @@ export class RakutenRapidApiService implements DataProviderInterface {
return DataSource.RAKUTEN; return DataSource.RAKUTEN;
} }
public async getQuotes(
aSymbols: string[]
): Promise<{ [symbol: string]: IDataProviderResponse }> {
if (aSymbols.length <= 0) {
return {};
}
try {
const symbol = aSymbols[0];
if (symbol === ghostfolioFearAndGreedIndexSymbol) {
const fgi = await this.getFearAndGreedIndex();
return {
[ghostfolioFearAndGreedIndexSymbol]: {
currency: undefined,
dataSource: this.getName(),
marketPrice: fgi.now.value,
marketState: MarketState.open
}
};
}
} catch (error) {
Logger.error(error);
}
return {};
}
public async search(aQuery: string): Promise<{ items: LookupItem[] }> { public async search(aQuery: string): Promise<{ items: LookupItem[] }> {
return { items: [] }; return { items: [] };
} }

21
apps/api/src/services/data-provider/yahoo-finance/interfaces/interfaces.ts

@ -1,21 +0,0 @@
export interface IYahooFinanceQuoteResponse {
price: IYahooFinancePrice;
summaryProfile: IYahooFinanceSummaryProfile;
}
export interface IYahooFinancePrice {
currency: string;
exchangeName: string;
longName: string;
marketState: string;
quoteType: string;
regularMarketPrice: number;
shortName: string;
}
export interface IYahooFinanceSummaryProfile {
country?: string;
industry?: string;
sector?: string;
website?: string;
}

190
apps/api/src/services/data-provider/yahoo-finance/yahoo-finance.service.ts

@ -1,27 +1,27 @@
import { LookupItem } from '@ghostfolio/api/app/symbol/interfaces/lookup-item.interface'; import { LookupItem } from '@ghostfolio/api/app/symbol/interfaces/lookup-item.interface';
import { CryptocurrencyService } from '@ghostfolio/api/services/cryptocurrency/cryptocurrency.service'; import { CryptocurrencyService } from '@ghostfolio/api/services/cryptocurrency/cryptocurrency.service';
import { UNKNOWN_KEY, baseCurrency } from '@ghostfolio/common/config'; import {
IDataProviderHistoricalResponse,
IDataProviderResponse,
MarketState
} from '@ghostfolio/api/services/interfaces/interfaces';
import { baseCurrency } from '@ghostfolio/common/config';
import { DATE_FORMAT, isCurrency } from '@ghostfolio/common/helper'; import { DATE_FORMAT, isCurrency } from '@ghostfolio/common/helper';
import { Granularity } from '@ghostfolio/common/types'; import { Granularity } from '@ghostfolio/common/types';
import { Injectable, Logger } from '@nestjs/common'; import { Injectable, Logger } from '@nestjs/common';
import { AssetClass, AssetSubClass, DataSource } from '@prisma/client'; import {
AssetClass,
AssetSubClass,
DataSource,
SymbolProfile
} from '@prisma/client';
import * as bent from 'bent'; import * as bent from 'bent';
import Big from 'big.js'; import Big from 'big.js';
import { countries } from 'countries-list'; import { countries } from 'countries-list';
import { addDays, format, isSameDay } from 'date-fns'; import { addDays, format, isSameDay } from 'date-fns';
import * as yahooFinance from 'yahoo-finance';
import yahooFinance2 from 'yahoo-finance2'; import yahooFinance2 from 'yahoo-finance2';
import {
IDataProviderHistoricalResponse,
IDataProviderResponse,
MarketState
} from '../../interfaces/interfaces';
import { DataProviderInterface } from '../interfaces/data-provider.interface'; import { DataProviderInterface } from '../interfaces/data-provider.interface';
import {
IYahooFinancePrice,
IYahooFinanceQuoteResponse
} from './interfaces/interfaces';
@Injectable() @Injectable()
export class YahooFinanceService implements DataProviderInterface { export class YahooFinanceService implements DataProviderInterface {
@ -73,92 +73,60 @@ export class YahooFinanceService implements DataProviderInterface {
return aSymbol; return aSymbol;
} }
public async get( public async getAssetProfile(
aSymbols: string[] aSymbol: string
): Promise<{ [symbol: string]: IDataProviderResponse }> { ): Promise<Partial<SymbolProfile>> {
if (aSymbols.length <= 0) { const response: Partial<SymbolProfile> = {};
return {};
}
const yahooFinanceSymbols = aSymbols.map((symbol) =>
this.convertToYahooFinanceSymbol(symbol)
);
try { try {
const response: { [symbol: string]: IDataProviderResponse } = {}; const symbol = this.convertToYahooFinanceSymbol(aSymbol);
const assetProfile = await yahooFinance2.quoteSummary(symbol, {
const data: { modules: ['price', 'summaryProfile']
[symbol: string]: IYahooFinanceQuoteResponse;
} = await yahooFinance.quote({
modules: ['price', 'summaryProfile'],
symbols: yahooFinanceSymbols
}); });
for (const [yahooFinanceSymbol, value] of Object.entries(data)) { const { assetClass, assetSubClass } = this.parseAssetClass(
// Convert symbols back assetProfile.price
const symbol = this.convertFromYahooFinanceSymbol(yahooFinanceSymbol); );
const { assetClass, assetSubClass } = this.parseAssetClass(value.price);
response[symbol] = {
assetClass,
assetSubClass,
currency: value.price?.currency,
dataSource: this.getName(),
exchange: this.parseExchange(value.price?.exchangeName),
marketState:
value.price?.marketState === 'REGULAR' ||
this.cryptocurrencyService.isCryptocurrency(symbol)
? MarketState.open
: MarketState.closed,
marketPrice: value.price?.regularMarketPrice || 0,
name: value.price?.longName || value.price?.shortName || symbol
};
if (value.price?.currency === 'GBp') { response.assetClass = assetClass;
// Convert GBp (pence) to GBP response.assetSubClass = assetSubClass;
response[symbol].currency = 'GBP'; response.currency = assetProfile.price.currency;
response[symbol].marketPrice = new Big( response.dataSource = this.getName();
value.price?.regularMarketPrice ?? 0 response.name =
) assetProfile.price.longName || assetProfile.price.shortName || symbol;
.div(100) response.symbol = aSymbol;
.toNumber();
}
// Add country if stock and available
if ( if (
assetSubClass === AssetSubClass.STOCK && assetSubClass === AssetSubClass.STOCK &&
value.summaryProfile?.country assetProfile.summaryProfile?.country
) { ) {
// Add country if asset is stock and country available
try { try {
const [code] = Object.entries(countries).find(([, country]) => { const [code] = Object.entries(countries).find(([, country]) => {
return country.name === value.summaryProfile?.country; return country.name === assetProfile.summaryProfile?.country;
}); });
if (code) { if (code) {
response[symbol].countries = [{ code, weight: 1 }]; response.countries = [{ code, weight: 1 }];
} }
} catch {} } catch {}
if (value.summaryProfile?.sector) { if (assetProfile.summaryProfile?.sector) {
response[symbol].sectors = [ response.sectors = [
{ name: value.summaryProfile?.sector, weight: 1 } { name: assetProfile.summaryProfile?.sector, weight: 1 }
]; ];
} }
} }
// Add url if available // TODO: Add url if available
const url = value.summaryProfile?.website; /*const url = assetProfile.summaryProfile?.website;
if (url) { if (url) {
response[symbol].url = url; response.url = url;
} }*/
} } catch {}
return response; return response;
} catch (error) {
Logger.error(error);
return {};
}
} }
public async getHistorical( public async getHistorical(
@ -215,6 +183,53 @@ export class YahooFinanceService implements DataProviderInterface {
return DataSource.YAHOO; return DataSource.YAHOO;
} }
public async getQuotes(
aSymbols: string[]
): Promise<{ [symbol: string]: IDataProviderResponse }> {
if (aSymbols.length <= 0) {
return {};
}
const yahooFinanceSymbols = aSymbols.map((symbol) =>
this.convertToYahooFinanceSymbol(symbol)
);
try {
const response: { [symbol: string]: IDataProviderResponse } = {};
const quotes = await yahooFinance2.quote(yahooFinanceSymbols, {}, {});
for (const quote of quotes) {
// Convert symbols back
const symbol = this.convertFromYahooFinanceSymbol(quote.symbol);
response[symbol] = {
currency: quote.currency,
dataSource: this.getName(),
marketState:
quote.marketState === 'REGULAR' ||
this.cryptocurrencyService.isCryptocurrency(symbol)
? MarketState.open
: MarketState.closed,
marketPrice: quote.regularMarketPrice || 0
};
if (quote.currency === 'GBp') {
// Convert GBp (pence) to GBP
response[symbol].currency = 'GBP';
response[symbol].marketPrice = new Big(quote.regularMarketPrice ?? 0)
.div(100)
.toNumber();
}
}
return response;
} catch (error) {
Logger.error(error);
return {};
}
}
public async search(aQuery: string): Promise<{ items: LookupItem[] }> { public async search(aQuery: string): Promise<{ items: LookupItem[] }> {
const items: LookupItem[] = []; const items: LookupItem[] = [];
@ -230,7 +245,7 @@ export class YahooFinanceService implements DataProviderInterface {
const searchResult = await get(); const searchResult = await get();
const symbols: string[] = searchResult.quotes const quotes = searchResult.quotes
.filter((quote) => { .filter((quote) => {
// filter out undefined symbols // filter out undefined symbols
return quote.symbol; return quote.symbol;
@ -253,19 +268,24 @@ export class YahooFinanceService implements DataProviderInterface {
} }
return true; return true;
})
.map(({ symbol }) => {
return symbol;
}); });
const marketData = await this.get(symbols); const marketData = await this.getQuotes(
quotes.map(({ symbol }) => {
return symbol;
})
);
for (const [symbol, value] of Object.entries(marketData)) { for (const [symbol, value] of Object.entries(marketData)) {
const quote = quotes.find((currentQuote: any) => {
return currentQuote.symbol === symbol;
});
items.push({ items.push({
symbol, symbol,
currency: value.currency, currency: value.currency,
dataSource: this.getName(), dataSource: this.getName(),
name: value.name name: quote?.longname || quote?.shortname || symbol
}); });
} }
} catch (error) { } catch (error) {
@ -275,7 +295,7 @@ export class YahooFinanceService implements DataProviderInterface {
return { items }; return { items };
} }
private parseAssetClass(aPrice: IYahooFinancePrice): { private parseAssetClass(aPrice: any): {
assetClass: AssetClass; assetClass: AssetClass;
assetSubClass: AssetSubClass; assetSubClass: AssetSubClass;
} { } {
@ -299,12 +319,4 @@ export class YahooFinanceService implements DataProviderInterface {
return { assetClass, assetSubClass }; return { assetClass, assetSubClass };
} }
private parseExchange(aString: string): string {
if (aString?.toLowerCase() === 'ccc') {
return UNKNOWN_KEY;
}
return aString;
}
} }

2
apps/api/src/services/exchange-rate-data.service.ts

@ -61,7 +61,7 @@ export class ExchangeRateDataService {
if (Object.keys(result).length !== this.currencyPairs.length) { if (Object.keys(result).length !== this.currencyPairs.length) {
// Load currencies directly from data provider as a fallback // Load currencies directly from data provider as a fallback
// if historical data is not fully available // if historical data is not fully available
const historicalData = await this.dataProviderService.get( const historicalData = await this.dataProviderService.getQuotes(
this.currencyPairs.map(({ dataSource, symbol }) => { this.currencyPairs.map(({ dataSource, symbol }) => {
return { dataSource, symbol }; return { dataSource, symbol };
}) })

9
apps/api/src/services/interfaces/interfaces.ts

@ -33,19 +33,10 @@ export interface IDataProviderHistoricalResponse {
} }
export interface IDataProviderResponse { export interface IDataProviderResponse {
assetClass?: AssetClass;
assetSubClass?: AssetSubClass;
countries?: { code: string; weight: number }[];
currency: string; currency: string;
dataSource: DataSource; dataSource: DataSource;
exchange?: string;
marketChange?: number;
marketChangePercent?: number;
marketPrice: number; marketPrice: number;
marketState: MarketState; marketState: MarketState;
name?: string;
sectors?: { name: string; weight: number }[];
url?: string;
} }
export interface IDataGatheringItem { export interface IDataGatheringItem {

5
apps/client/src/app/components/position/position.component.html

@ -39,11 +39,6 @@
<div class="h6 m-0 text-truncate">{{ position?.name }}</div> <div class="h6 m-0 text-truncate">{{ position?.name }}</div>
<div class="d-flex"> <div class="d-flex">
<span>{{ position?.symbol | gfSymbol }}</span> <span>{{ position?.symbol | gfSymbol }}</span>
<span
*ngIf="position?.exchange && position?.exchange !== unknownKey"
class="ml-2 text-muted"
>({{ position.exchange }})</span
>
</div> </div>
<div class="d-flex mt-1"> <div class="d-flex mt-1">
<gf-value <gf-value

3
package.json

@ -117,8 +117,7 @@
"tslib": "2.0.0", "tslib": "2.0.0",
"twitter-api-v2": "1.10.3", "twitter-api-v2": "1.10.3",
"uuid": "8.3.2", "uuid": "8.3.2",
"yahoo-finance": "0.3.6", "yahoo-finance2": "2.1.8",
"yahoo-finance2": "2.1.5",
"zone.js": "0.11.4" "zone.js": "0.11.4"
}, },
"devDependencies": { "devDependencies": {

8
yarn.lock

@ -18781,10 +18781,10 @@ y18n@^5.0.5:
resolved "https://registry.yarnpkg.com/y18n/-/y18n-5.0.8.tgz#7f4934d0f7ca8c56f95314939ddcd2dd91ce1d55" resolved "https://registry.yarnpkg.com/y18n/-/y18n-5.0.8.tgz#7f4934d0f7ca8c56f95314939ddcd2dd91ce1d55"
integrity sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA== integrity sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==
yahoo-finance2@2.1.5: yahoo-finance2@2.1.8:
version "2.1.5" version "2.1.8"
resolved "https://registry.yarnpkg.com/yahoo-finance2/-/yahoo-finance2-2.1.5.tgz#f0ff4025b56d4b9c63a3edfd3b28df292c171e90" resolved "https://registry.yarnpkg.com/yahoo-finance2/-/yahoo-finance2-2.1.8.tgz#4e48f01318b5592aa9e275e27b32fdf5175e57a5"
integrity sha512-yN+BvaJ1IRWoOaZkZMHpH6Ll2NtKGeC9RRFDo6FcrC7llkCkL0RV/Fcvd4jbMIZ6pOhuZqsnxF0Mm7h5EZQIqg== integrity sha512-bjSgr43gpDHErB90s22Kl06nZkGApAtnLNdBeZZSX5DcWSqOQAJRsZ+RXhg/5SduCzcYivIEonA1x0+TKe4F9w==
dependencies: dependencies:
ajv "8.10.0" ajv "8.10.0"
ajv-formats "2.1.1" ajv-formats "2.1.1"

Loading…
Cancel
Save