From 6b0906f1f2b3e817eeafa4b54cd0df7c1476e24b Mon Sep 17 00:00:00 2001 From: Anatoly Popov Date: Sat, 7 Jun 2025 23:48:30 +0300 Subject: [PATCH 1/4] fix: don't fail whole request if one of providers fails, but log it Signed-off-by: Anatoly Popov --- .../data-providers/ghostfolio/ghostfolio.service.ts | 9 +++++++-- .../src/services/data-provider/data-provider.service.ts | 9 +++++++-- .../data-provider/yahoo-finance/yahoo-finance.service.ts | 1 + 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/apps/api/src/app/endpoints/data-providers/ghostfolio/ghostfolio.service.ts b/apps/api/src/app/endpoints/data-providers/ghostfolio/ghostfolio.service.ts index d088bf3ac..8c7e2ec3e 100644 --- a/apps/api/src/app/endpoints/data-providers/ghostfolio/ghostfolio.service.ts +++ b/apps/api/src/app/endpoints/data-providers/ghostfolio/ghostfolio.service.ts @@ -319,9 +319,14 @@ export class GhostfolioService { ); } - const searchResults = await Promise.all(promises); + const searchResults = await Promise.allSettled(promises); - for (const { items } of searchResults) { + for (const result of searchResults) { + if (result.status === 'rejected') { + Logger.warn(result.reason, 'GhostfolioService'); + continue; + } + const { items } = result.value; if (items?.length > 0) { lookupItems = lookupItems.concat(items); } diff --git a/apps/api/src/services/data-provider/data-provider.service.ts b/apps/api/src/services/data-provider/data-provider.service.ts index a6b12cce2..35df5d551 100644 --- a/apps/api/src/services/data-provider/data-provider.service.ts +++ b/apps/api/src/services/data-provider/data-provider.service.ts @@ -762,9 +762,14 @@ export class DataProviderService implements OnModuleInit { ); } - const searchResults = await Promise.all(promises); + const searchResults = await Promise.allSettled(promises); - for (const { items } of searchResults) { + for (const result of searchResults) { + if (result.status === 'rejected') { + Logger.warn(result.reason, 'DataProviderService'); + continue; + } + const { items } = result.value; if (items?.length > 0) { lookupItems = lookupItems.concat(items); } diff --git a/apps/api/src/services/data-provider/yahoo-finance/yahoo-finance.service.ts b/apps/api/src/services/data-provider/yahoo-finance/yahoo-finance.service.ts index de8807098..61feaa49a 100644 --- a/apps/api/src/services/data-provider/yahoo-finance/yahoo-finance.service.ts +++ b/apps/api/src/services/data-provider/yahoo-finance/yahoo-finance.service.ts @@ -335,6 +335,7 @@ export class YahooFinanceService implements DataProviderInterface { } } catch (error) { Logger.error(error, 'YahooFinanceService'); + return { items: [] }; } return { items }; From 2daf2093e689b7c978446bad865d339fe592deb7 Mon Sep 17 00:00:00 2001 From: Anatoly Popov Date: Sat, 7 Jun 2025 23:48:30 +0300 Subject: [PATCH 2/4] fix: if yahoo service returns zero quotes do not fail enhance request Signed-off-by: Anatoly Popov --- .../data-enhancer/yahoo-finance/yahoo-finance.service.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/apps/api/src/services/data-provider/data-enhancer/yahoo-finance/yahoo-finance.service.ts b/apps/api/src/services/data-provider/data-enhancer/yahoo-finance/yahoo-finance.service.ts index 72136dc04..1a3ebfa58 100644 --- a/apps/api/src/services/data-provider/data-enhancer/yahoo-finance/yahoo-finance.service.ts +++ b/apps/api/src/services/data-provider/data-enhancer/yahoo-finance/yahoo-finance.service.ts @@ -105,6 +105,9 @@ export class YahooFinanceDataEnhancerService implements DataEnhancerInterface { yahooSymbol = symbol; } else { const { quotes } = await this.yahooFinance.search(response.isin); + if (quotes.length === 0) { + return response; + } yahooSymbol = quotes[0].symbol as string; } From 522d1135f273aa8a4db75840b0ba3680d4c5ab27 Mon Sep 17 00:00:00 2001 From: Anatoly Popov Date: Sun, 18 May 2025 15:27:57 +0300 Subject: [PATCH 3/4] feat: support security from MOEX via MOEX api **Justification** My main portfolio is on MOEX, so I need a way to manage it. Ghostfolio looks cool but there's no way to proper manage assets on MOEX. So, first I've found `moex-iss-api` npm package. Turns out it needed some help, so couple of PR there. After that I was ready to work on ghostfolio. This commit implements full `DataProviderInterface` spec: - Disabled by default, so only those who need it would enabled it. - We `canHandle` all symbols - We aren't premium feature - `getQuotes`, `getHistorical` and `search` were pretty straightforward - `getTestSymbol` return `SBER` because if something happens to `SBER` stocks then MOEX would definetely doesn't matter anymore for sometime. - `getAssetProfile` and `getDividends` proved to be tricky to implement, so I'll cover them below separately. **getAssetProfile** **Currency** This is main method to get info about assets. Unfortunately, due to fall of USSR and ruble denomination in 1998, we have three currency tickers: 'RUR', 'SUR', 'RUB' For convenience we use 'RUB' instead of all of them. I don't see practical value to differentiate between them here, but I'm open to suggestions. Also, some of the tickers do not return currency in which they're listed on MOEX. Assumed that it's also 'RUB'. **Name** Every asset can have several things to identify it. And all of them are optional in MOEX API, except `secid` which is `Security ID`. So we use them for name in this order of preference: 1. Latin (usually English) name. 2. Latin short name. 3. Russian name. 4. Security ID. **Country** I try to detect country, parsing ISIN: first two letters should be country code. **Sectors** MOEX supports some industry related indices, so when we first encounter some symbol, I check whether it's in those indices and assign sectors accordingly. **AssetClass and AssetSubClass** At first, I was tempted to leave them empty, but finally decided to look into. I downloaded all asset types from MOEX and tried to best of my knowledge assign asset classes and subclasses. If I wasn't able to find proper relation, I left the cell empty. After that I took the table (you can check it in the comments in the code) and made `SecurityTypeMap` interface. **getDividends** MOEX API for dividends isn't documented at all (or probably I didn't find proper docs) and sometimes it doesn't return the newest dividends. Surprisingly, you can get dividends for MOEX-related assets from YAHOO, but the date can differ. So, there is heurestic implemented: if those date are no more than two days apart and payout is the same, then it's the exact same payout and we merge them. Signed-off-by: Anatoly Popov --- .../data-provider/data-provider.module.ts | 18 +- .../data-provider/moex/moex.service.ts | 604 ++++++++++++++++++ package-lock.json | 77 +-- package.json | 1 + .../migration.sql | 2 + prisma/schema.prisma | 1 + 6 files changed, 643 insertions(+), 60 deletions(-) create mode 100644 apps/api/src/services/data-provider/moex/moex.service.ts create mode 100644 prisma/migrations/20240501071657_added_moex_to_data_sources/migration.sql diff --git a/apps/api/src/services/data-provider/data-provider.module.ts b/apps/api/src/services/data-provider/data-provider.module.ts index 71b54f01e..0881fd036 100644 --- a/apps/api/src/services/data-provider/data-provider.module.ts +++ b/apps/api/src/services/data-provider/data-provider.module.ts @@ -8,6 +8,7 @@ import { FinancialModelingPrepService } from '@ghostfolio/api/services/data-prov import { GhostfolioService } from '@ghostfolio/api/services/data-provider/ghostfolio/ghostfolio.service'; import { GoogleSheetsService } from '@ghostfolio/api/services/data-provider/google-sheets/google-sheets.service'; import { ManualService } from '@ghostfolio/api/services/data-provider/manual/manual.service'; +import { MoexService } from '@ghostfolio/api/services/data-provider/moex/moex.service'; import { RapidApiService } from '@ghostfolio/api/services/data-provider/rapid-api/rapid-api.service'; import { YahooFinanceService } from '@ghostfolio/api/services/data-provider/yahoo-finance/yahoo-finance.service'; import { MarketDataModule } from '@ghostfolio/api/services/market-data/market-data.module'; @@ -43,6 +44,7 @@ import { DataProviderService } from './data-provider.service'; ManualService, RapidApiService, YahooFinanceService, + MoexService, { inject: [ AlphaVantageService, @@ -53,7 +55,8 @@ import { DataProviderService } from './data-provider.service'; GoogleSheetsService, ManualService, RapidApiService, - YahooFinanceService + YahooFinanceService, + MoexService ], provide: 'DataProviderInterfaces', useFactory: ( @@ -65,7 +68,8 @@ import { DataProviderService } from './data-provider.service'; googleSheetsService, manualService, rapidApiService, - yahooFinanceService + yahooFinanceService, + moexService ) => [ alphaVantageService, coinGeckoService, @@ -75,11 +79,17 @@ import { DataProviderService } from './data-provider.service'; googleSheetsService, manualService, rapidApiService, - yahooFinanceService + yahooFinanceService, + moexService ] }, YahooFinanceDataEnhancerService ], - exports: [DataProviderService, ManualService, YahooFinanceService] + exports: [ + DataProviderService, + ManualService, + YahooFinanceService, + MoexService + ] }) export class DataProviderModule {} diff --git a/apps/api/src/services/data-provider/moex/moex.service.ts b/apps/api/src/services/data-provider/moex/moex.service.ts new file mode 100644 index 000000000..4587ebff5 --- /dev/null +++ b/apps/api/src/services/data-provider/moex/moex.service.ts @@ -0,0 +1,604 @@ +import { + DataProviderInterface, + GetDividendsParams, + GetHistoricalParams, + GetQuotesParams, + GetSearchParams +} from '@ghostfolio/api/services/data-provider/interfaces/data-provider.interface'; +import { YahooFinanceService } from '@ghostfolio/api/services/data-provider/yahoo-finance/yahoo-finance.service'; +import { + DataProviderHistoricalResponse, + DataProviderResponse, + DataProviderInfo, + LookupResponse +} from '@ghostfolio/common/interfaces'; + +import { Injectable, Logger } from '@nestjs/common'; +import { $Enums, SymbolProfile } from '@prisma/client'; +import { isISO4217CurrencyCode } from 'class-validator'; +import { + subYears, + format, + startOfYesterday, + subDays, + differenceInDays, + addYears, + differenceInMilliseconds +} from 'date-fns'; +import { createMoexCLient } from 'moex-iss-api-client'; +import { + IGetSecuritiesParams, + TGetSecuritiesParamsGroupBy +} from 'moex-iss-api-client/dist/client/security/requestTypes'; +import { ISecuritiesResponse } from 'moex-iss-api-client/dist/client/security/responseTypes'; + +const moexClient = createMoexCLient(); + +declare interface ResponseData { + columns: string[]; + data: (string | number | null)[][]; +} + +declare interface Response { + data: T; + issError: string; +} + +function response_data_to_map( + response: ResponseData, + keyColumnName: string +): Map> { + const result = new Map>(); + + response.data.forEach((x) => { + const item = new Map(); + response.columns.forEach((c, i) => item.set(c, x[i])); + result.set(item.get(keyColumnName), item); + }); + + return result; +} + +function getCurrency(currency: string): string { + if (currency === 'SUR' || currency === 'RUR') return 'RUB'; + + return currency; +} + +/// So, we try to guess sectors of security by looking into indexes, +/// in which this security was put by MOEX +const indexToSectorMapping = new Map([ + ['MOEXOG', ['Energy']], // MOEX Oil and Gas Index + ['MOEXEU', ['Utilities']], // MOEX Electric Utilities Index + ['MOEXTL', ['Communication Services']], // MOEX Telecommunication Index + ['MOEXMM', ['Basic Materials', 'Industrial']], // MOEX Metals and Mining Index + ['MOEXFN', ['Financial Services']], // MOEX Financials Index + ['MOEXCN', ['Consumer Defensive', 'Consumer Cyclical', 'Healthcare']], // MOEX Consumer Index + ['MOEXCH', ['Basic Materials']], // MOEX Chemicals Index + ['MOEXIT', ['Technology']], // MOEX Information Technologies Index + ['MOEXRE', ['Real Estate']], // MOEX Real Estate Index + ['MOEXTN', ['Consumer Cyclical', 'Industrial']] // MOEX Transportation Index +]); + +async function getSectors( + symbol: string +): Promise<{ name: string; weight: number }[]> { + const indicesResponse: Response<{ indices: ResponseData }> = + await moexClient.security.getSecurityIndexes({ security: symbol }); + const errorMessage = indicesResponse.issError; + if (errorMessage) { + Logger.warn(errorMessage, 'MoexService.getSectors'); + return []; + } + + const indices = response_data_to_map(indicesResponse.data.indices, 'SECID'); + + const sectorIncluded = new Set(); + const sectors = new Array<{ name: string; weight: number }>(); + + for (const [indexCode, indexSectors] of indexToSectorMapping.entries()) { + const index = indices.get(indexCode); + if (!index) { + continue; + } + + if (sectorIncluded.has(indexCode)) { + continue; + } + + sectorIncluded.add(indexCode); + indexSectors.forEach((x) => sectors.push({ name: x, weight: 1.0 })); + } + + return sectors; +} + +async function getDividendsFromMoex({ + from, + symbol, + to +}: GetDividendsParams): Promise<{ + [date: string]: DataProviderHistoricalResponse; +}> { + const response: Response<{ dividends: ResponseData }> = + await moexClient.request(`securities/${symbol}/dividends`); + if (response.issError) { + Logger.warn(response.issError, 'MoexService.getDividends'); + return {}; + } + + const dividends = response_data_to_map( + response.data.dividends, + 'registryclosedate' + ); + const result: { + [date: string]: DataProviderHistoricalResponse; + } = {}; + + for (const [key, value] of dividends.entries()) { + const date = new Date(key); + + if (date < from || date > to) { + continue; + } + + const price = value.get('value'); + if (typeof price === 'number') result[key] = { marketPrice: price }; + } + + return result; +} + +async function readBatchedResponse( + getNextBatch: (start: number) => Promise>, + extractor: (batch: Response) => ResponseData, + max_items?: number +): Promise { + let batch: ResponseData; + const wholeResponse: ResponseData = { columns: [], data: [] }; + + do { + const response: Response = await getNextBatch(wholeResponse.data.length); + if (response === undefined) { + break; + } + if (response.issError) { + Logger.warn(response.issError, 'MoexService.readBatchedResponse'); + break; + } + + batch = extractor(response); + if (wholeResponse.columns.length === 0) { + wholeResponse.columns = batch.columns; + } + + wholeResponse.data = [...wholeResponse.data, ...batch.data]; + } while ( + batch.data.length > 0 && + (!max_items || wholeResponse.data.length <= max_items) + ); + + return wholeResponse; +} + +function getYahooSymbolFromMoex(symbol: string): string { + return `${symbol}.ME`; +} + +async function getAssetUrlFromYahoo( + yahooFinanceService: YahooFinanceService, + symbol: string +): Promise { + try { + const profile = await yahooFinanceService.getAssetProfile({ + symbol: getYahooSymbolFromMoex(symbol) + }); + return profile?.url; + } catch (e) { + Logger.warn(`Can't get url for symbol ${symbol} from YAHOO, error is ${e}`); + return null; + } +} + +async function getSecuritySpecification( + symbol: string, + dataSectionName: string, + key: string +): Promise>> { + const securitySpecificationResponse = + await moexClient.security.getSecuritySpecification({ security: symbol }); + const errorMessage = securitySpecificationResponse.issError; + if (errorMessage) { + Logger.warn(errorMessage, 'MoexService.getAssetProfile'); + return new Map>(); + } + + return response_data_to_map( + securitySpecificationResponse.data[dataSectionName], + key + ); +} + +interface SecurityTypeMap { + [key: string]: [$Enums.AssetClass?, $Enums.AssetSubClass?]; +} + +/// MOEX security types were obtained here: https://iss.moex.com/iss/index.json (add `?lang=en` for english) +/// | id | security_type_name | ghostfolio_assetclass | ghostfolio_assetsubclass | +/// | ---- | --------------------- | --------------------- | ------------------------ | +/// | 1 | preferred_share | EQUITY | STOCK | +/// | 2 | corporate_bond | FIXED_INCOME | BOND | +/// | 3 | common_share | EQUITY | STOCK | +/// | 4 | cb_bond | FIXED_INCOME | BOND | +/// | 5 | currency | LIQUIDITY | CASH | +/// | 6 | futures | COMMODITY | | +/// | 7 | public_ppif | EQUITY | MUTUALFUND | +/// | 8 | interval_ppif | EQUITY | MUTUALFUND | +/// | 9 | private_ppif | EQUITY | MUTUALFUND | +/// | 10 | state_bond | FIXED_INCOME | BOND | +/// | 41 | subfederal_bond | FIXED_INCOME | BOND | +/// | 42 | ifi_bond | FIXED_INCOME | BOND | +/// | 43 | exchange_bond | FIXED_INCOME | BOND | +/// | 44 | stock_index | | | +/// | 45 | municipal_bond | FIXED_INCOME | BOND | +/// | 51 | depositary_receipt | EQUITY | STOCK | +/// | 52 | option | COMMODITY | | +/// | 53 | rts_index | | | +/// | 54 | ofz_bond | FIXED_INCOME | BOND | +/// | 55 | etf_ppif | EQUITY | ETF | +/// | 57 | stock_mortgage | REAL_ESTATE | | +/// | 58 | gold_metal | LIQUIDITY | PRECIOUS_METAL | +/// | 59 | silver_metal | LIQUIDITY | PRECIOUS_METAL | +/// | 60 | euro_bond | FIXED_INCOME | BOND | +/// | 62 | currency_futures | LIQUIDITY | CASH | +/// | 63 | stock_deposit | LIQUIDITY | CASH | +/// | 73 | currency_fixing | LIQUIDITY | CASH | +/// | 74 | exchange_ppif | EQUITY | ETF | +/// | 75 | currency_index | LIQUIDITY | CASH | +/// | 76 | currency_wap | LIQUIDITY | CASH | +/// | 78 | non_exchange_bond | FIXED_INCOME | BOND | +/// | 84 | stock_index_eq | | | +/// | 85 | stock_index_fi | | | +/// | 86 | stock_index_mx | | | +/// | 87 | stock_index_ie | | | +/// | 88 | stock_index_if | | | +/// | 89 | stock_index_ci | | | +/// | 90 | stock_index_im | | | +/// | 1030 | stock_index_namex | | | +/// | 1031 | option_on_shares | EQUITY | STOCK | +/// | 1034 | stock_index_rusfar | | | +/// | 1155 | stock_index_pf | | | +/// | 1291 | option_on_currency | | | +/// | 1293 | option_on_indices | | | +/// | 1295 | option_on_commodities | COMMODITY | | +/// | 1337 | futures_spread | COMMODITY | | +/// | 1338 | futures_collateral | COMMODITY | | +/// | 1347 | currency_otcindices | LIQUIDITY | CASH | +/// | 1352 | other_metal | COMMODITY | PRECIOUS_METAL | +/// | 1403 | stock_index_ri | | | +const securityTypeMap: SecurityTypeMap = { + preferred_share: [$Enums.AssetClass.EQUITY, $Enums.AssetSubClass.STOCK], + corporate_bond: [$Enums.AssetClass.FIXED_INCOME, $Enums.AssetSubClass.BOND], + common_share: [$Enums.AssetClass.EQUITY, $Enums.AssetSubClass.STOCK], + cb_bond: [$Enums.AssetClass.FIXED_INCOME, $Enums.AssetSubClass.BOND], + currency: [$Enums.AssetClass.LIQUIDITY, $Enums.AssetSubClass.CASH], + futures: [$Enums.AssetClass.COMMODITY], + public_ppif: [$Enums.AssetClass.EQUITY, $Enums.AssetSubClass.MUTUALFUND], + interval_ppif: [$Enums.AssetClass.EQUITY, $Enums.AssetSubClass.MUTUALFUND], + private_ppif: [$Enums.AssetClass.EQUITY, $Enums.AssetSubClass.MUTUALFUND], + state_bond: [$Enums.AssetClass.FIXED_INCOME, $Enums.AssetSubClass.BOND], + subfederal_bond: [$Enums.AssetClass.FIXED_INCOME, $Enums.AssetSubClass.BOND], + ifi_bond: [$Enums.AssetClass.FIXED_INCOME, $Enums.AssetSubClass.BOND], + exchange_bond: [$Enums.AssetClass.FIXED_INCOME, $Enums.AssetSubClass.BOND], + municipal_bond: [$Enums.AssetClass.FIXED_INCOME, $Enums.AssetSubClass.BOND], + depositary_receipt: [$Enums.AssetClass.EQUITY, $Enums.AssetSubClass.STOCK], + option: [$Enums.AssetClass.COMMODITY], + etf_ppif: [$Enums.AssetClass.EQUITY, $Enums.AssetSubClass.ETF], + stock_mortgage: [$Enums.AssetClass.REAL_ESTATE], + gold_metal: [ + $Enums.AssetClass.LIQUIDITY, + $Enums.AssetSubClass.PRECIOUS_METAL + ], + silver_metal: [ + $Enums.AssetClass.LIQUIDITY, + $Enums.AssetSubClass.PRECIOUS_METAL + ], + currency_futures: [$Enums.AssetClass.LIQUIDITY, $Enums.AssetSubClass.CASH], + stock_deposit: [$Enums.AssetClass.LIQUIDITY, $Enums.AssetSubClass.CASH], + currency_fixing: [$Enums.AssetClass.LIQUIDITY, $Enums.AssetSubClass.CASH], + exchange_ppif: [$Enums.AssetClass.EQUITY, $Enums.AssetSubClass.ETF], + currency_index: [$Enums.AssetClass.LIQUIDITY, $Enums.AssetSubClass.CASH], + currency_wap: [$Enums.AssetClass.LIQUIDITY, $Enums.AssetSubClass.CASH], + non_exchange_bond: [ + $Enums.AssetClass.FIXED_INCOME, + $Enums.AssetSubClass.BOND + ], + option_on_shares: [$Enums.AssetClass.EQUITY, $Enums.AssetSubClass.STOCK], + option_on_commodities: [$Enums.AssetClass.COMMODITY], + futures_spread: [$Enums.AssetClass.COMMODITY], + futures_collateral: [$Enums.AssetClass.COMMODITY], + currency_otcindices: [$Enums.AssetClass.LIQUIDITY, $Enums.AssetSubClass.CASH], + other_metal: [ + $Enums.AssetClass.COMMODITY, + $Enums.AssetSubClass.PRECIOUS_METAL + ] +}; + +@Injectable() +export class MoexService implements DataProviderInterface { + public constructor( + private readonly yahooFinanceService: YahooFinanceService + ) {} + + canHandle(): boolean { + return true; + } + + async getAssetProfile({ + symbol + }: { + symbol: string; + }): Promise> { + const securitySpecification = await getSecuritySpecification( + symbol, + 'description', + 'name' + ); + + const issueDate = securitySpecification.get('ISSUEDATE'); + const faceunit = securitySpecification.get('FACEUNIT'); + const isin = securitySpecification.get('ISIN'); + const latname = securitySpecification.get('LATNAME'); + const shortname = securitySpecification.get('SHORTNAME'); + const name = securitySpecification.get('NAME'); + const secid = securitySpecification.get('SECID'); + const type = securitySpecification.get('TYPE'); + const [assetClass, assetSubClass] = + securityTypeMap[type.get('value').toString()] ?? []; + + let currency = faceunit + ? getCurrency(faceunit.get('value').toString().toUpperCase()) + : 'RUB'; + + if (!isISO4217CurrencyCode(currency)) { + currency = 'RUB'; + } + + return { + assetClass: assetClass, + assetSubClass: assetSubClass, + createdAt: issueDate ? new Date(issueDate.get('value')) : new Date(), + currency, + dataSource: this.getName(), + id: symbol, + isin: isin ? isin.get('value').toString() : null, + name: (latname ?? shortname ?? name ?? secid).get('value').toString(), + sectors: await getSectors(symbol), + symbol: symbol, + countries: isin + ? [ + { + code: isin.get('value').toString().substring(0, 2), + weight: 1 + } + ] + : null, + url: await getAssetUrlFromYahoo(this.yahooFinanceService, symbol) + }; + } + + getDataProviderInfo(): DataProviderInfo { + return { + isPremium: false + }; + } + + // MOEX endpoint for dividends isn't documented and sometimes doesn't return newer dividends. + // YAHOO endpoint for dividends sometimes doesn't respect date filters. + // So, we'll requests dividends for 2 years more from both providers and merge data. + // If dividends date from MOEX and YAHOO differs for 2 days or less, we'll assume it's the same payout given amount is the same. + // Payouts considered the same if they differ less that 1/100 of the currency. + async getDividends({ from, symbol, to }: GetDividendsParams): Promise<{ + [date: string]: DataProviderHistoricalResponse; + }> { + const twoYearsAgo = subYears(from, 2); + const twoYearsAhead = addYears(to, 2); + const [dividends, dividendsFromYahoo] = await Promise.all([ + getDividendsFromMoex({ from: twoYearsAgo, symbol, to: twoYearsAhead }), + this.yahooFinanceService.getDividends({ + from: twoYearsAgo, + symbol: getYahooSymbolFromMoex(symbol), + to: twoYearsAhead + }) + ]); + + const dateAlmostTheSame = (x: Date, y: Date) => + Math.abs(differenceInDays(x, y)) <= 2; + const payoutsAlmostTheSame = (x: number, y: number) => + 100 * Math.abs(x - y) < 1; + + for (const [yahooDateStr, yahooDividends] of Object.entries( + dividendsFromYahoo + )) { + const yahooDate = new Date(yahooDateStr); + const sameDividendIndex = Object.entries(dividends).findIndex( + (x) => + dateAlmostTheSame(new Date(x[0]), yahooDate) && + payoutsAlmostTheSame(x[1].marketPrice, yahooDividends.marketPrice) + ); + if (sameDividendIndex === -1) { + dividends[yahooDateStr] = yahooDividends; + } + } + + const result: { [date: string]: DataProviderHistoricalResponse } = {}; + + Object.entries(dividends) + .map( + ([dateStr, dividends]) => + [dateStr, dividends, new Date(dateStr)] as [ + string, + DataProviderHistoricalResponse, + Date + ] + ) + .filter(([, , date]) => date >= from) + .filter(([, , date]) => date <= to) + .sort(([, , a], [, , b]) => differenceInMilliseconds(a, b)) + .forEach(([dateStr, dividends]) => (result[dateStr] = dividends)); + + return result; + } + + async getHistorical({ from, symbol, to }: GetHistoricalParams): Promise<{ + [symbol: string]: { [date: string]: DataProviderHistoricalResponse }; + }> { + const securitySpecification = await getSecuritySpecification( + symbol, + 'boards', + 'is_primary' + ); + const primaryBoard = securitySpecification.get(1); + + const board_id = primaryBoard.get('boardid'); + const market = primaryBoard.get('market'); + const engine = primaryBoard.get('engine'); + + const params: Record = { + sort_order: 'desc', + marketprice_board: '1', + from: format(from, 'yyyy-MM-dd'), + to: format(to, 'yyyy-MM-dd') + }; + + const historyResponse = await readBatchedResponse<{ + history: ResponseData; + }>( + async (x) => { + params['start'] = x; + return await moexClient.request( + `history/engines/${engine}/markets/${market}/securities/${symbol}`, + params + ); + }, + (x) => { + return { + columns: x.data.history.columns, + data: x.data.history.data.filter((x) => x[0] === board_id) + }; + } + ); + + const history = response_data_to_map(historyResponse, 'TRADEDATE'); + + const result: { + [symbol: string]: { [date: string]: DataProviderHistoricalResponse }; + } = {}; + result[symbol] = {}; + + for (const [key, value] of history.entries()) { + const price = value.get('LEGALCLOSEPRICE') ?? value.get('CLOSE'); + if (typeof price === 'number') + result[symbol][key] = { marketPrice: price }; + else + Logger.error( + `We have quote, but can't get price. Symbol ${symbol}, columns are [${historyResponse.columns.join(', ')}]` + ); + } + + return result; + } + + getMaxNumberOfSymbolsPerRequest?(): number { + return 1; + } + + getName(): $Enums.DataSource { + return $Enums.DataSource.MOEX; + } + + async getQuotes({ + symbols + }: GetQuotesParams): Promise<{ [symbol: string]: DataProviderResponse }> { + const result: { [symbol: string]: DataProviderResponse } = {}; + + for (const symbol of symbols) { + const profile = await this.getAssetProfile({ symbol: symbol }); + + for ( + let date = startOfYesterday(); + !result[symbol]; + date = subDays(date, 1) + ) { + const history = await this.getHistorical({ + from: date, + to: date, + symbol: symbol + }); + + for (const [, v] of Object.entries(history[symbol])) { + result[symbol] = { + currency: profile.currency, + dataSource: profile.dataSource, + marketPrice: v.marketPrice, + marketState: 'closed' + }; + } + } + } + + return result; + } + + getTestSymbol(): string { + return 'SBER'; + } + + async search({ query }: GetSearchParams): Promise { + const MAX_SEARCH_ITEMS: number = 50; + // MOEX doesn't support search for queries less than 3 symbols + if (query.length < 3) { + return { items: [] }; + } + + const params: IGetSecuritiesParams & TGetSecuritiesParamsGroupBy = { + q: query + }; + + const searchResponse = await readBatchedResponse( + async (x) => { + params['start'] = x; + return await moexClient.security.getSecurities(params); + }, + (x) => x.data.securities, + MAX_SEARCH_ITEMS + ); + + const search = response_data_to_map(searchResponse, 'secid'); + + const result: LookupResponse = { items: [] }; + for (const k of search.keys()) { + if (typeof k != 'string') { + continue; + } + const profile = await this.getAssetProfile({ symbol: k }); + const lookedup_profile = { + assetClass: profile.assetClass, + assetSubClass: profile.assetSubClass, + currency: profile.currency, + dataProviderInfo: this.getDataProviderInfo(), + dataSource: this.getName(), + name: profile.name, + symbol: k + }; + + if (k === query) { + result.items.unshift(lookedup_profile); + } else { + result.items.push(lookedup_profile); + } + } + + return result; + } +} diff --git a/package-lock.json b/package-lock.json index f7371da13..223b17488 100644 --- a/package-lock.json +++ b/package-lock.json @@ -71,6 +71,7 @@ "jsonpath": "1.1.1", "lodash": "4.17.23", "marked": "17.0.2", + "moex-iss-api-client": "^0.4.2", "ms": "3.0.0-canary.1", "ng-extract-i18n-merge": "3.2.1", "ngx-device-detector": "11.0.0", @@ -7791,24 +7792,6 @@ "url": "https://github.com/chalk/chalk?sponsor=1" } }, - "node_modules/@nestjs/schematics/node_modules/chokidar": { - "version": "4.0.3", - "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-4.0.3.tgz", - "integrity": "sha512-Qgzu8kfBvo+cA4962jnP1KkS6Dop5NS6g7R5LFYJr4b8Ub94PPQXUksCw9PvXoeXPRRddRNC5C1JQUR2SMGtnA==", - "dev": true, - "license": "MIT", - "optional": true, - "peer": true, - "dependencies": { - "readdirp": "^4.0.1" - }, - "engines": { - "node": ">= 14.16.0" - }, - "funding": { - "url": "https://paulmillr.com/funding/" - } - }, "node_modules/@nestjs/schematics/node_modules/is-interactive": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/is-interactive/-/is-interactive-1.0.0.tgz", @@ -7843,22 +7826,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/@nestjs/schematics/node_modules/readdirp": { - "version": "4.1.2", - "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-4.1.2.tgz", - "integrity": "sha512-GDhwkLfywWL2s6vEjyhri+eXmfH6j1L7JE27WhqLeYzoh/A3DBaYGEj2H/HFZCn/kMfim73FXxEJTw06WtxQwg==", - "dev": true, - "license": "MIT", - "optional": true, - "peer": true, - "engines": { - "node": ">= 14.18.0" - }, - "funding": { - "type": "individual", - "url": "https://paulmillr.com/funding/" - } - }, "node_modules/@nestjs/serve-static": { "version": "5.0.4", "resolved": "https://registry.npmjs.org/@nestjs/serve-static/-/serve-static-5.0.4.tgz", @@ -14950,7 +14917,6 @@ "version": "0.4.0", "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==", - "dev": true, "license": "MIT" }, "node_modules/at-least-node": { @@ -16702,7 +16668,6 @@ "version": "1.0.8", "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", - "dev": true, "license": "MIT", "dependencies": { "delayed-stream": "~1.0.0" @@ -18409,7 +18374,6 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", - "dev": true, "license": "MIT", "engines": { "node": ">=0.4.0" @@ -19094,7 +19058,6 @@ "version": "2.1.0", "resolved": "https://registry.npmjs.org/es-set-tostringtag/-/es-set-tostringtag-2.1.0.tgz", "integrity": "sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==", - "dev": true, "license": "MIT", "dependencies": { "es-errors": "^1.3.0", @@ -19178,21 +19141,6 @@ "@esbuild/win32-x64": "0.27.2" } }, - "node_modules/esbuild-register": { - "version": "3.6.0", - "resolved": "https://registry.npmjs.org/esbuild-register/-/esbuild-register-3.6.0.tgz", - "integrity": "sha512-H2/S7Pm8a9CL1uhp9OvjwrBh5Pvx0H8qVOxNu8Wed9Y7qv56MPtq+GGM8RJpq6glYJn9Wspr8uw7l55uyinNeg==", - "dev": true, - "license": "MIT", - "optional": true, - "peer": true, - "dependencies": { - "debug": "^4.3.4" - }, - "peerDependencies": { - "esbuild": ">=0.12 <1" - } - }, "node_modules/esbuild-wasm": { "version": "0.27.2", "resolved": "https://registry.npmjs.org/esbuild-wasm/-/esbuild-wasm-0.27.2.tgz", @@ -20634,7 +20582,6 @@ "version": "4.0.5", "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.5.tgz", "integrity": "sha512-8RipRLol37bNs2bhoV67fiTEvdTrbMUYcFTiy3+wuuOnUog2QBHCZWXDRijWQfAkhBj2Uf5UnVaiWwA5vdd82w==", - "dev": true, "license": "MIT", "dependencies": { "asynckit": "^0.4.0", @@ -21458,7 +21405,6 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.2.tgz", "integrity": "sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==", - "dev": true, "license": "MIT", "dependencies": { "has-symbols": "^1.0.3" @@ -26032,6 +25978,26 @@ "pathe": "^2.0.1" } }, + "node_modules/moex-iss-api-client": { + "version": "0.4.2", + "resolved": "https://registry.npmjs.org/moex-iss-api-client/-/moex-iss-api-client-0.4.2.tgz", + "integrity": "sha512-gm0nI/d0aTprLhX5K8+3CD/S741juBSsG3ZlU1fkofOxPLyyGnIjrxxmHaCopmwoXG7OCeyP7Qm7QkImbKh3Ew==", + "license": "MIT", + "dependencies": { + "axios": "^0.28.0" + } + }, + "node_modules/moex-iss-api-client/node_modules/axios": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/axios/-/axios-0.28.1.tgz", + "integrity": "sha512-iUcGA5a7p0mVb4Gm/sy+FSECNkPFT4y7wt6OM/CDpO/OnNCvSs3PoMG8ibrC9jRoGYU0gUK5pXVC4NPXq6lHRQ==", + "license": "MIT", + "dependencies": { + "follow-redirects": "^1.15.0", + "form-data": "^4.0.0", + "proxy-from-env": "^1.1.0" + } + }, "node_modules/mrmime": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/mrmime/-/mrmime-2.0.1.tgz", @@ -29070,7 +29036,6 @@ "version": "1.1.0", "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==", - "dev": true, "license": "MIT" }, "node_modules/prr": { diff --git a/package.json b/package.json index 794a09ea7..de9f74ba8 100644 --- a/package.json +++ b/package.json @@ -116,6 +116,7 @@ "jsonpath": "1.1.1", "lodash": "4.17.23", "marked": "17.0.2", + "moex-iss-api-client": "^0.4.2", "ms": "3.0.0-canary.1", "ng-extract-i18n-merge": "3.2.1", "ngx-device-detector": "11.0.0", diff --git a/prisma/migrations/20240501071657_added_moex_to_data_sources/migration.sql b/prisma/migrations/20240501071657_added_moex_to_data_sources/migration.sql new file mode 100644 index 000000000..639e7aa9b --- /dev/null +++ b/prisma/migrations/20240501071657_added_moex_to_data_sources/migration.sql @@ -0,0 +1,2 @@ +-- AlterEnum +ALTER TYPE "DataSource" ADD VALUE 'MOEX'; diff --git a/prisma/schema.prisma b/prisma/schema.prisma index 232dde9ca..da522101c 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -324,6 +324,7 @@ enum DataSource { MANUAL RAPID_API YAHOO + MOEX } enum MarketDataState { From 59a3db2c088e0fa73b25503f6ad0097bd318b4c4 Mon Sep 17 00:00:00 2001 From: Anatoly Popov Date: Thu, 8 Jan 2026 00:18:16 +0300 Subject: [PATCH 4/4] chore(build): added script to update fork Signed-off-by: Anatoly Popov --- update-fork.sh | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100755 update-fork.sh diff --git a/update-fork.sh b/update-fork.sh new file mode 100755 index 000000000..8aa7a2ddf --- /dev/null +++ b/update-fork.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +set -eoux pipefail + +git fetch --all +git checkout main +git pull origin +git checkout feat/moex-service +git rebase main +git checkout --ours package-lock.json package.json && git add package-lock.json package.json +npm add moex-iss-api-client +git add package-lock.json package.json +git rebase --continue