From 978cc9c10c750f1d28ef808234490489def5166f Mon Sep 17 00:00:00 2001 From: Anatoly Popov Date: Sat, 7 Jun 2025 23:48:30 +0300 Subject: [PATCH 1/4] fix: don't fail whole request if one of providers fails, but log it Signed-off-by: Anatoly Popov --- .../data-providers/ghostfolio/ghostfolio.service.ts | 9 +++++++-- .../src/services/data-provider/data-provider.service.ts | 9 +++++++-- .../data-provider/yahoo-finance/yahoo-finance.service.ts | 1 + 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/apps/api/src/app/endpoints/data-providers/ghostfolio/ghostfolio.service.ts b/apps/api/src/app/endpoints/data-providers/ghostfolio/ghostfolio.service.ts index d088bf3ac..8c7e2ec3e 100644 --- a/apps/api/src/app/endpoints/data-providers/ghostfolio/ghostfolio.service.ts +++ b/apps/api/src/app/endpoints/data-providers/ghostfolio/ghostfolio.service.ts @@ -319,9 +319,14 @@ export class GhostfolioService { ); } - const searchResults = await Promise.all(promises); + const searchResults = await Promise.allSettled(promises); - for (const { items } of searchResults) { + for (const result of searchResults) { + if (result.status === 'rejected') { + Logger.warn(result.reason, 'GhostfolioService'); + continue; + } + const { items } = result.value; if (items?.length > 0) { lookupItems = lookupItems.concat(items); } diff --git a/apps/api/src/services/data-provider/data-provider.service.ts b/apps/api/src/services/data-provider/data-provider.service.ts index 5a088c0e4..4538cfe89 100644 --- a/apps/api/src/services/data-provider/data-provider.service.ts +++ b/apps/api/src/services/data-provider/data-provider.service.ts @@ -641,9 +641,14 @@ export class DataProviderService implements OnModuleInit { ); } - const searchResults = await Promise.all(promises); + const searchResults = await Promise.allSettled(promises); - for (const { items } of searchResults) { + for (const result of searchResults) { + if (result.status === 'rejected') { + Logger.warn(result.reason, 'DataProviderService'); + continue; + } + const { items } = result.value; if (items?.length > 0) { lookupItems = lookupItems.concat(items); } diff --git a/apps/api/src/services/data-provider/yahoo-finance/yahoo-finance.service.ts b/apps/api/src/services/data-provider/yahoo-finance/yahoo-finance.service.ts index de8807098..61feaa49a 100644 --- a/apps/api/src/services/data-provider/yahoo-finance/yahoo-finance.service.ts +++ b/apps/api/src/services/data-provider/yahoo-finance/yahoo-finance.service.ts @@ -335,6 +335,7 @@ export class YahooFinanceService implements DataProviderInterface { } } catch (error) { Logger.error(error, 'YahooFinanceService'); + return { items: [] }; } return { items }; From 6956d79dc16f115218dd6ba86f0aa668a9027baf Mon Sep 17 00:00:00 2001 From: Anatoly Popov Date: Sat, 7 Jun 2025 23:48:30 +0300 Subject: [PATCH 2/4] fix: if yahoo service returns zero quotes do not fail enhance request Signed-off-by: Anatoly Popov --- .../data-enhancer/yahoo-finance/yahoo-finance.service.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/apps/api/src/services/data-provider/data-enhancer/yahoo-finance/yahoo-finance.service.ts b/apps/api/src/services/data-provider/data-enhancer/yahoo-finance/yahoo-finance.service.ts index 65bcd6c06..84ba31c44 100644 --- a/apps/api/src/services/data-provider/data-enhancer/yahoo-finance/yahoo-finance.service.ts +++ b/apps/api/src/services/data-provider/data-enhancer/yahoo-finance/yahoo-finance.service.ts @@ -105,6 +105,9 @@ export class YahooFinanceDataEnhancerService implements DataEnhancerInterface { yahooSymbol = symbol; } else { const { quotes } = await this.yahooFinance.search(response.isin); + if (quotes.length === 0) { + return response; + } yahooSymbol = quotes[0].symbol as string; } From 158095c7c814b47a7e0c5db5c8672a615ca858bf Mon Sep 17 00:00:00 2001 From: Anatoly Popov Date: Sun, 18 May 2025 15:27:57 +0300 Subject: [PATCH 3/4] feat: support security from MOEX via MOEX api **Justification** My main portfolio is on MOEX, so I need a way to manage it. Ghostfolio looks cool but there's no way to proper manage assets on MOEX. So, first I've found `moex-iss-api` npm package. Turns out it needed some help, so couple of PR there. After that I was ready to work on ghostfolio. This commit implements full `DataProviderInterface` spec: - Disabled by default, so only those who need it would enabled it. - We `canHandle` all symbols - We aren't premium feature - `getQuotes`, `getHistorical` and `search` were pretty straightforward - `getTestSymbol` return `SBER` because if something happens to `SBER` stocks then MOEX would definetely doesn't matter anymore for sometime. - `getAssetProfile` and `getDividends` proved to be tricky to implement, so I'll cover them below separately. **getAssetProfile** **Currency** This is main method to get info about assets. Unfortunately, due to fall of USSR and ruble denomination in 1998, we have three currency tickers: 'RUR', 'SUR', 'RUB' For convenience we use 'RUB' instead of all of them. I don't see practical value to differentiate between them here, but I'm open to suggestions. Also, some of the tickers do not return currency in which they're listed on MOEX. Assumed that it's also 'RUB'. **Name** Every asset can have several things to identify it. And all of them are optional in MOEX API, except `secid` which is `Security ID`. So we use them for name in this order of preference: 1. Latin (usually English) name. 2. Latin short name. 3. Russian name. 4. Security ID. **Country** I try to detect country, parsing ISIN: first two letters should be country code. **Sectors** MOEX supports some industry related indices, so when we first encounter some symbol, I check whether it's in those indices and assign sectors accordingly. **AssetClass and AssetSubClass** At first, I was tempted to leave them empty, but finally decided to look into. I downloaded all asset types from MOEX and tried to best of my knowledge assign asset classes and subclasses. If I wasn't able to find proper relation, I left the cell empty. After that I took the table (you can check it in the comments in the code) and made `SecurityTypeMap` interface. **getDividends** MOEX API for dividends isn't documented at all (or probably I didn't find proper docs) and sometimes it doesn't return the newest dividends. Surprisingly, you can get dividends for MOEX-related assets from YAHOO, but the date can differ. So, there is heurestic implemented: if those date are no more than two days apart and payout is the same, then it's the exact same payout and we merge them. Signed-off-by: Anatoly Popov --- .../data-provider/data-provider.module.ts | 18 +- .../data-provider/moex/moex.service.ts | 604 ++++++++++++++++++ package-lock.json | 43 +- package.json | 1 + .../migration.sql | 2 + prisma/schema.prisma | 1 + 6 files changed, 643 insertions(+), 26 deletions(-) create mode 100644 apps/api/src/services/data-provider/moex/moex.service.ts create mode 100644 prisma/migrations/20240501071657_added_moex_to_data_sources/migration.sql diff --git a/apps/api/src/services/data-provider/data-provider.module.ts b/apps/api/src/services/data-provider/data-provider.module.ts index 71b54f01e..0881fd036 100644 --- a/apps/api/src/services/data-provider/data-provider.module.ts +++ b/apps/api/src/services/data-provider/data-provider.module.ts @@ -8,6 +8,7 @@ import { FinancialModelingPrepService } from '@ghostfolio/api/services/data-prov import { GhostfolioService } from '@ghostfolio/api/services/data-provider/ghostfolio/ghostfolio.service'; import { GoogleSheetsService } from '@ghostfolio/api/services/data-provider/google-sheets/google-sheets.service'; import { ManualService } from '@ghostfolio/api/services/data-provider/manual/manual.service'; +import { MoexService } from '@ghostfolio/api/services/data-provider/moex/moex.service'; import { RapidApiService } from '@ghostfolio/api/services/data-provider/rapid-api/rapid-api.service'; import { YahooFinanceService } from '@ghostfolio/api/services/data-provider/yahoo-finance/yahoo-finance.service'; import { MarketDataModule } from '@ghostfolio/api/services/market-data/market-data.module'; @@ -43,6 +44,7 @@ import { DataProviderService } from './data-provider.service'; ManualService, RapidApiService, YahooFinanceService, + MoexService, { inject: [ AlphaVantageService, @@ -53,7 +55,8 @@ import { DataProviderService } from './data-provider.service'; GoogleSheetsService, ManualService, RapidApiService, - YahooFinanceService + YahooFinanceService, + MoexService ], provide: 'DataProviderInterfaces', useFactory: ( @@ -65,7 +68,8 @@ import { DataProviderService } from './data-provider.service'; googleSheetsService, manualService, rapidApiService, - yahooFinanceService + yahooFinanceService, + moexService ) => [ alphaVantageService, coinGeckoService, @@ -75,11 +79,17 @@ import { DataProviderService } from './data-provider.service'; googleSheetsService, manualService, rapidApiService, - yahooFinanceService + yahooFinanceService, + moexService ] }, YahooFinanceDataEnhancerService ], - exports: [DataProviderService, ManualService, YahooFinanceService] + exports: [ + DataProviderService, + ManualService, + YahooFinanceService, + MoexService + ] }) export class DataProviderModule {} diff --git a/apps/api/src/services/data-provider/moex/moex.service.ts b/apps/api/src/services/data-provider/moex/moex.service.ts new file mode 100644 index 000000000..4587ebff5 --- /dev/null +++ b/apps/api/src/services/data-provider/moex/moex.service.ts @@ -0,0 +1,604 @@ +import { + DataProviderInterface, + GetDividendsParams, + GetHistoricalParams, + GetQuotesParams, + GetSearchParams +} from '@ghostfolio/api/services/data-provider/interfaces/data-provider.interface'; +import { YahooFinanceService } from '@ghostfolio/api/services/data-provider/yahoo-finance/yahoo-finance.service'; +import { + DataProviderHistoricalResponse, + DataProviderResponse, + DataProviderInfo, + LookupResponse +} from '@ghostfolio/common/interfaces'; + +import { Injectable, Logger } from '@nestjs/common'; +import { $Enums, SymbolProfile } from '@prisma/client'; +import { isISO4217CurrencyCode } from 'class-validator'; +import { + subYears, + format, + startOfYesterday, + subDays, + differenceInDays, + addYears, + differenceInMilliseconds +} from 'date-fns'; +import { createMoexCLient } from 'moex-iss-api-client'; +import { + IGetSecuritiesParams, + TGetSecuritiesParamsGroupBy +} from 'moex-iss-api-client/dist/client/security/requestTypes'; +import { ISecuritiesResponse } from 'moex-iss-api-client/dist/client/security/responseTypes'; + +const moexClient = createMoexCLient(); + +declare interface ResponseData { + columns: string[]; + data: (string | number | null)[][]; +} + +declare interface Response { + data: T; + issError: string; +} + +function response_data_to_map( + response: ResponseData, + keyColumnName: string +): Map> { + const result = new Map>(); + + response.data.forEach((x) => { + const item = new Map(); + response.columns.forEach((c, i) => item.set(c, x[i])); + result.set(item.get(keyColumnName), item); + }); + + return result; +} + +function getCurrency(currency: string): string { + if (currency === 'SUR' || currency === 'RUR') return 'RUB'; + + return currency; +} + +/// So, we try to guess sectors of security by looking into indexes, +/// in which this security was put by MOEX +const indexToSectorMapping = new Map([ + ['MOEXOG', ['Energy']], // MOEX Oil and Gas Index + ['MOEXEU', ['Utilities']], // MOEX Electric Utilities Index + ['MOEXTL', ['Communication Services']], // MOEX Telecommunication Index + ['MOEXMM', ['Basic Materials', 'Industrial']], // MOEX Metals and Mining Index + ['MOEXFN', ['Financial Services']], // MOEX Financials Index + ['MOEXCN', ['Consumer Defensive', 'Consumer Cyclical', 'Healthcare']], // MOEX Consumer Index + ['MOEXCH', ['Basic Materials']], // MOEX Chemicals Index + ['MOEXIT', ['Technology']], // MOEX Information Technologies Index + ['MOEXRE', ['Real Estate']], // MOEX Real Estate Index + ['MOEXTN', ['Consumer Cyclical', 'Industrial']] // MOEX Transportation Index +]); + +async function getSectors( + symbol: string +): Promise<{ name: string; weight: number }[]> { + const indicesResponse: Response<{ indices: ResponseData }> = + await moexClient.security.getSecurityIndexes({ security: symbol }); + const errorMessage = indicesResponse.issError; + if (errorMessage) { + Logger.warn(errorMessage, 'MoexService.getSectors'); + return []; + } + + const indices = response_data_to_map(indicesResponse.data.indices, 'SECID'); + + const sectorIncluded = new Set(); + const sectors = new Array<{ name: string; weight: number }>(); + + for (const [indexCode, indexSectors] of indexToSectorMapping.entries()) { + const index = indices.get(indexCode); + if (!index) { + continue; + } + + if (sectorIncluded.has(indexCode)) { + continue; + } + + sectorIncluded.add(indexCode); + indexSectors.forEach((x) => sectors.push({ name: x, weight: 1.0 })); + } + + return sectors; +} + +async function getDividendsFromMoex({ + from, + symbol, + to +}: GetDividendsParams): Promise<{ + [date: string]: DataProviderHistoricalResponse; +}> { + const response: Response<{ dividends: ResponseData }> = + await moexClient.request(`securities/${symbol}/dividends`); + if (response.issError) { + Logger.warn(response.issError, 'MoexService.getDividends'); + return {}; + } + + const dividends = response_data_to_map( + response.data.dividends, + 'registryclosedate' + ); + const result: { + [date: string]: DataProviderHistoricalResponse; + } = {}; + + for (const [key, value] of dividends.entries()) { + const date = new Date(key); + + if (date < from || date > to) { + continue; + } + + const price = value.get('value'); + if (typeof price === 'number') result[key] = { marketPrice: price }; + } + + return result; +} + +async function readBatchedResponse( + getNextBatch: (start: number) => Promise>, + extractor: (batch: Response) => ResponseData, + max_items?: number +): Promise { + let batch: ResponseData; + const wholeResponse: ResponseData = { columns: [], data: [] }; + + do { + const response: Response = await getNextBatch(wholeResponse.data.length); + if (response === undefined) { + break; + } + if (response.issError) { + Logger.warn(response.issError, 'MoexService.readBatchedResponse'); + break; + } + + batch = extractor(response); + if (wholeResponse.columns.length === 0) { + wholeResponse.columns = batch.columns; + } + + wholeResponse.data = [...wholeResponse.data, ...batch.data]; + } while ( + batch.data.length > 0 && + (!max_items || wholeResponse.data.length <= max_items) + ); + + return wholeResponse; +} + +function getYahooSymbolFromMoex(symbol: string): string { + return `${symbol}.ME`; +} + +async function getAssetUrlFromYahoo( + yahooFinanceService: YahooFinanceService, + symbol: string +): Promise { + try { + const profile = await yahooFinanceService.getAssetProfile({ + symbol: getYahooSymbolFromMoex(symbol) + }); + return profile?.url; + } catch (e) { + Logger.warn(`Can't get url for symbol ${symbol} from YAHOO, error is ${e}`); + return null; + } +} + +async function getSecuritySpecification( + symbol: string, + dataSectionName: string, + key: string +): Promise>> { + const securitySpecificationResponse = + await moexClient.security.getSecuritySpecification({ security: symbol }); + const errorMessage = securitySpecificationResponse.issError; + if (errorMessage) { + Logger.warn(errorMessage, 'MoexService.getAssetProfile'); + return new Map>(); + } + + return response_data_to_map( + securitySpecificationResponse.data[dataSectionName], + key + ); +} + +interface SecurityTypeMap { + [key: string]: [$Enums.AssetClass?, $Enums.AssetSubClass?]; +} + +/// MOEX security types were obtained here: https://iss.moex.com/iss/index.json (add `?lang=en` for english) +/// | id | security_type_name | ghostfolio_assetclass | ghostfolio_assetsubclass | +/// | ---- | --------------------- | --------------------- | ------------------------ | +/// | 1 | preferred_share | EQUITY | STOCK | +/// | 2 | corporate_bond | FIXED_INCOME | BOND | +/// | 3 | common_share | EQUITY | STOCK | +/// | 4 | cb_bond | FIXED_INCOME | BOND | +/// | 5 | currency | LIQUIDITY | CASH | +/// | 6 | futures | COMMODITY | | +/// | 7 | public_ppif | EQUITY | MUTUALFUND | +/// | 8 | interval_ppif | EQUITY | MUTUALFUND | +/// | 9 | private_ppif | EQUITY | MUTUALFUND | +/// | 10 | state_bond | FIXED_INCOME | BOND | +/// | 41 | subfederal_bond | FIXED_INCOME | BOND | +/// | 42 | ifi_bond | FIXED_INCOME | BOND | +/// | 43 | exchange_bond | FIXED_INCOME | BOND | +/// | 44 | stock_index | | | +/// | 45 | municipal_bond | FIXED_INCOME | BOND | +/// | 51 | depositary_receipt | EQUITY | STOCK | +/// | 52 | option | COMMODITY | | +/// | 53 | rts_index | | | +/// | 54 | ofz_bond | FIXED_INCOME | BOND | +/// | 55 | etf_ppif | EQUITY | ETF | +/// | 57 | stock_mortgage | REAL_ESTATE | | +/// | 58 | gold_metal | LIQUIDITY | PRECIOUS_METAL | +/// | 59 | silver_metal | LIQUIDITY | PRECIOUS_METAL | +/// | 60 | euro_bond | FIXED_INCOME | BOND | +/// | 62 | currency_futures | LIQUIDITY | CASH | +/// | 63 | stock_deposit | LIQUIDITY | CASH | +/// | 73 | currency_fixing | LIQUIDITY | CASH | +/// | 74 | exchange_ppif | EQUITY | ETF | +/// | 75 | currency_index | LIQUIDITY | CASH | +/// | 76 | currency_wap | LIQUIDITY | CASH | +/// | 78 | non_exchange_bond | FIXED_INCOME | BOND | +/// | 84 | stock_index_eq | | | +/// | 85 | stock_index_fi | | | +/// | 86 | stock_index_mx | | | +/// | 87 | stock_index_ie | | | +/// | 88 | stock_index_if | | | +/// | 89 | stock_index_ci | | | +/// | 90 | stock_index_im | | | +/// | 1030 | stock_index_namex | | | +/// | 1031 | option_on_shares | EQUITY | STOCK | +/// | 1034 | stock_index_rusfar | | | +/// | 1155 | stock_index_pf | | | +/// | 1291 | option_on_currency | | | +/// | 1293 | option_on_indices | | | +/// | 1295 | option_on_commodities | COMMODITY | | +/// | 1337 | futures_spread | COMMODITY | | +/// | 1338 | futures_collateral | COMMODITY | | +/// | 1347 | currency_otcindices | LIQUIDITY | CASH | +/// | 1352 | other_metal | COMMODITY | PRECIOUS_METAL | +/// | 1403 | stock_index_ri | | | +const securityTypeMap: SecurityTypeMap = { + preferred_share: [$Enums.AssetClass.EQUITY, $Enums.AssetSubClass.STOCK], + corporate_bond: [$Enums.AssetClass.FIXED_INCOME, $Enums.AssetSubClass.BOND], + common_share: [$Enums.AssetClass.EQUITY, $Enums.AssetSubClass.STOCK], + cb_bond: [$Enums.AssetClass.FIXED_INCOME, $Enums.AssetSubClass.BOND], + currency: [$Enums.AssetClass.LIQUIDITY, $Enums.AssetSubClass.CASH], + futures: [$Enums.AssetClass.COMMODITY], + public_ppif: [$Enums.AssetClass.EQUITY, $Enums.AssetSubClass.MUTUALFUND], + interval_ppif: [$Enums.AssetClass.EQUITY, $Enums.AssetSubClass.MUTUALFUND], + private_ppif: [$Enums.AssetClass.EQUITY, $Enums.AssetSubClass.MUTUALFUND], + state_bond: [$Enums.AssetClass.FIXED_INCOME, $Enums.AssetSubClass.BOND], + subfederal_bond: [$Enums.AssetClass.FIXED_INCOME, $Enums.AssetSubClass.BOND], + ifi_bond: [$Enums.AssetClass.FIXED_INCOME, $Enums.AssetSubClass.BOND], + exchange_bond: [$Enums.AssetClass.FIXED_INCOME, $Enums.AssetSubClass.BOND], + municipal_bond: [$Enums.AssetClass.FIXED_INCOME, $Enums.AssetSubClass.BOND], + depositary_receipt: [$Enums.AssetClass.EQUITY, $Enums.AssetSubClass.STOCK], + option: [$Enums.AssetClass.COMMODITY], + etf_ppif: [$Enums.AssetClass.EQUITY, $Enums.AssetSubClass.ETF], + stock_mortgage: [$Enums.AssetClass.REAL_ESTATE], + gold_metal: [ + $Enums.AssetClass.LIQUIDITY, + $Enums.AssetSubClass.PRECIOUS_METAL + ], + silver_metal: [ + $Enums.AssetClass.LIQUIDITY, + $Enums.AssetSubClass.PRECIOUS_METAL + ], + currency_futures: [$Enums.AssetClass.LIQUIDITY, $Enums.AssetSubClass.CASH], + stock_deposit: [$Enums.AssetClass.LIQUIDITY, $Enums.AssetSubClass.CASH], + currency_fixing: [$Enums.AssetClass.LIQUIDITY, $Enums.AssetSubClass.CASH], + exchange_ppif: [$Enums.AssetClass.EQUITY, $Enums.AssetSubClass.ETF], + currency_index: [$Enums.AssetClass.LIQUIDITY, $Enums.AssetSubClass.CASH], + currency_wap: [$Enums.AssetClass.LIQUIDITY, $Enums.AssetSubClass.CASH], + non_exchange_bond: [ + $Enums.AssetClass.FIXED_INCOME, + $Enums.AssetSubClass.BOND + ], + option_on_shares: [$Enums.AssetClass.EQUITY, $Enums.AssetSubClass.STOCK], + option_on_commodities: [$Enums.AssetClass.COMMODITY], + futures_spread: [$Enums.AssetClass.COMMODITY], + futures_collateral: [$Enums.AssetClass.COMMODITY], + currency_otcindices: [$Enums.AssetClass.LIQUIDITY, $Enums.AssetSubClass.CASH], + other_metal: [ + $Enums.AssetClass.COMMODITY, + $Enums.AssetSubClass.PRECIOUS_METAL + ] +}; + +@Injectable() +export class MoexService implements DataProviderInterface { + public constructor( + private readonly yahooFinanceService: YahooFinanceService + ) {} + + canHandle(): boolean { + return true; + } + + async getAssetProfile({ + symbol + }: { + symbol: string; + }): Promise> { + const securitySpecification = await getSecuritySpecification( + symbol, + 'description', + 'name' + ); + + const issueDate = securitySpecification.get('ISSUEDATE'); + const faceunit = securitySpecification.get('FACEUNIT'); + const isin = securitySpecification.get('ISIN'); + const latname = securitySpecification.get('LATNAME'); + const shortname = securitySpecification.get('SHORTNAME'); + const name = securitySpecification.get('NAME'); + const secid = securitySpecification.get('SECID'); + const type = securitySpecification.get('TYPE'); + const [assetClass, assetSubClass] = + securityTypeMap[type.get('value').toString()] ?? []; + + let currency = faceunit + ? getCurrency(faceunit.get('value').toString().toUpperCase()) + : 'RUB'; + + if (!isISO4217CurrencyCode(currency)) { + currency = 'RUB'; + } + + return { + assetClass: assetClass, + assetSubClass: assetSubClass, + createdAt: issueDate ? new Date(issueDate.get('value')) : new Date(), + currency, + dataSource: this.getName(), + id: symbol, + isin: isin ? isin.get('value').toString() : null, + name: (latname ?? shortname ?? name ?? secid).get('value').toString(), + sectors: await getSectors(symbol), + symbol: symbol, + countries: isin + ? [ + { + code: isin.get('value').toString().substring(0, 2), + weight: 1 + } + ] + : null, + url: await getAssetUrlFromYahoo(this.yahooFinanceService, symbol) + }; + } + + getDataProviderInfo(): DataProviderInfo { + return { + isPremium: false + }; + } + + // MOEX endpoint for dividends isn't documented and sometimes doesn't return newer dividends. + // YAHOO endpoint for dividends sometimes doesn't respect date filters. + // So, we'll requests dividends for 2 years more from both providers and merge data. + // If dividends date from MOEX and YAHOO differs for 2 days or less, we'll assume it's the same payout given amount is the same. + // Payouts considered the same if they differ less that 1/100 of the currency. + async getDividends({ from, symbol, to }: GetDividendsParams): Promise<{ + [date: string]: DataProviderHistoricalResponse; + }> { + const twoYearsAgo = subYears(from, 2); + const twoYearsAhead = addYears(to, 2); + const [dividends, dividendsFromYahoo] = await Promise.all([ + getDividendsFromMoex({ from: twoYearsAgo, symbol, to: twoYearsAhead }), + this.yahooFinanceService.getDividends({ + from: twoYearsAgo, + symbol: getYahooSymbolFromMoex(symbol), + to: twoYearsAhead + }) + ]); + + const dateAlmostTheSame = (x: Date, y: Date) => + Math.abs(differenceInDays(x, y)) <= 2; + const payoutsAlmostTheSame = (x: number, y: number) => + 100 * Math.abs(x - y) < 1; + + for (const [yahooDateStr, yahooDividends] of Object.entries( + dividendsFromYahoo + )) { + const yahooDate = new Date(yahooDateStr); + const sameDividendIndex = Object.entries(dividends).findIndex( + (x) => + dateAlmostTheSame(new Date(x[0]), yahooDate) && + payoutsAlmostTheSame(x[1].marketPrice, yahooDividends.marketPrice) + ); + if (sameDividendIndex === -1) { + dividends[yahooDateStr] = yahooDividends; + } + } + + const result: { [date: string]: DataProviderHistoricalResponse } = {}; + + Object.entries(dividends) + .map( + ([dateStr, dividends]) => + [dateStr, dividends, new Date(dateStr)] as [ + string, + DataProviderHistoricalResponse, + Date + ] + ) + .filter(([, , date]) => date >= from) + .filter(([, , date]) => date <= to) + .sort(([, , a], [, , b]) => differenceInMilliseconds(a, b)) + .forEach(([dateStr, dividends]) => (result[dateStr] = dividends)); + + return result; + } + + async getHistorical({ from, symbol, to }: GetHistoricalParams): Promise<{ + [symbol: string]: { [date: string]: DataProviderHistoricalResponse }; + }> { + const securitySpecification = await getSecuritySpecification( + symbol, + 'boards', + 'is_primary' + ); + const primaryBoard = securitySpecification.get(1); + + const board_id = primaryBoard.get('boardid'); + const market = primaryBoard.get('market'); + const engine = primaryBoard.get('engine'); + + const params: Record = { + sort_order: 'desc', + marketprice_board: '1', + from: format(from, 'yyyy-MM-dd'), + to: format(to, 'yyyy-MM-dd') + }; + + const historyResponse = await readBatchedResponse<{ + history: ResponseData; + }>( + async (x) => { + params['start'] = x; + return await moexClient.request( + `history/engines/${engine}/markets/${market}/securities/${symbol}`, + params + ); + }, + (x) => { + return { + columns: x.data.history.columns, + data: x.data.history.data.filter((x) => x[0] === board_id) + }; + } + ); + + const history = response_data_to_map(historyResponse, 'TRADEDATE'); + + const result: { + [symbol: string]: { [date: string]: DataProviderHistoricalResponse }; + } = {}; + result[symbol] = {}; + + for (const [key, value] of history.entries()) { + const price = value.get('LEGALCLOSEPRICE') ?? value.get('CLOSE'); + if (typeof price === 'number') + result[symbol][key] = { marketPrice: price }; + else + Logger.error( + `We have quote, but can't get price. Symbol ${symbol}, columns are [${historyResponse.columns.join(', ')}]` + ); + } + + return result; + } + + getMaxNumberOfSymbolsPerRequest?(): number { + return 1; + } + + getName(): $Enums.DataSource { + return $Enums.DataSource.MOEX; + } + + async getQuotes({ + symbols + }: GetQuotesParams): Promise<{ [symbol: string]: DataProviderResponse }> { + const result: { [symbol: string]: DataProviderResponse } = {}; + + for (const symbol of symbols) { + const profile = await this.getAssetProfile({ symbol: symbol }); + + for ( + let date = startOfYesterday(); + !result[symbol]; + date = subDays(date, 1) + ) { + const history = await this.getHistorical({ + from: date, + to: date, + symbol: symbol + }); + + for (const [, v] of Object.entries(history[symbol])) { + result[symbol] = { + currency: profile.currency, + dataSource: profile.dataSource, + marketPrice: v.marketPrice, + marketState: 'closed' + }; + } + } + } + + return result; + } + + getTestSymbol(): string { + return 'SBER'; + } + + async search({ query }: GetSearchParams): Promise { + const MAX_SEARCH_ITEMS: number = 50; + // MOEX doesn't support search for queries less than 3 symbols + if (query.length < 3) { + return { items: [] }; + } + + const params: IGetSecuritiesParams & TGetSecuritiesParamsGroupBy = { + q: query + }; + + const searchResponse = await readBatchedResponse( + async (x) => { + params['start'] = x; + return await moexClient.security.getSecurities(params); + }, + (x) => x.data.securities, + MAX_SEARCH_ITEMS + ); + + const search = response_data_to_map(searchResponse, 'secid'); + + const result: LookupResponse = { items: [] }; + for (const k of search.keys()) { + if (typeof k != 'string') { + continue; + } + const profile = await this.getAssetProfile({ symbol: k }); + const lookedup_profile = { + assetClass: profile.assetClass, + assetSubClass: profile.assetSubClass, + currency: profile.currency, + dataProviderInfo: this.getDataProviderInfo(), + dataSource: this.getName(), + name: profile.name, + symbol: k + }; + + if (k === query) { + result.items.unshift(lookedup_profile); + } else { + result.items.push(lookedup_profile); + } + } + + return result; + } +} diff --git a/package-lock.json b/package-lock.json index 95b71adf5..6c897f17d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -70,6 +70,7 @@ "jsonpath": "1.1.1", "lodash": "4.17.21", "marked": "17.0.1", + "moex-iss-api-client": "^0.4.2", "ms": "3.0.0-canary.1", "ng-extract-i18n-merge": "3.2.1", "ngx-device-detector": "11.0.0", @@ -14236,7 +14237,6 @@ "version": "0.4.0", "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==", - "dev": true, "license": "MIT" }, "node_modules/at-least-node": { @@ -15991,7 +15991,6 @@ "version": "1.0.8", "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", - "dev": true, "license": "MIT", "dependencies": { "delayed-stream": "~1.0.0" @@ -17690,7 +17689,6 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", - "dev": true, "license": "MIT", "engines": { "node": ">=0.4.0" @@ -18375,7 +18373,6 @@ "version": "2.1.0", "resolved": "https://registry.npmjs.org/es-set-tostringtag/-/es-set-tostringtag-2.1.0.tgz", "integrity": "sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==", - "dev": true, "license": "MIT", "dependencies": { "es-errors": "^1.3.0", @@ -18459,21 +18456,6 @@ "@esbuild/win32-x64": "0.26.0" } }, - "node_modules/esbuild-register": { - "version": "3.6.0", - "resolved": "https://registry.npmjs.org/esbuild-register/-/esbuild-register-3.6.0.tgz", - "integrity": "sha512-H2/S7Pm8a9CL1uhp9OvjwrBh5Pvx0H8qVOxNu8Wed9Y7qv56MPtq+GGM8RJpq6glYJn9Wspr8uw7l55uyinNeg==", - "dev": true, - "license": "MIT", - "optional": true, - "peer": true, - "dependencies": { - "debug": "^4.3.4" - }, - "peerDependencies": { - "esbuild": ">=0.12 <1" - } - }, "node_modules/esbuild-wasm": { "version": "0.26.0", "resolved": "https://registry.npmjs.org/esbuild-wasm/-/esbuild-wasm-0.26.0.tgz", @@ -19911,7 +19893,6 @@ "version": "4.0.4", "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.4.tgz", "integrity": "sha512-KrGhL9Q4zjj0kiUt5OO4Mr/A/jlI2jDYs5eHBpYHPcBEVSiipAvn2Ko2HnPe20rmcuuvMHNdZFp+4IlGTMF0Ow==", - "dev": true, "license": "MIT", "dependencies": { "asynckit": "^0.4.0", @@ -20735,7 +20716,6 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.2.tgz", "integrity": "sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==", - "dev": true, "license": "MIT", "dependencies": { "has-symbols": "^1.0.3" @@ -25322,6 +25302,26 @@ "pathe": "^2.0.1" } }, + "node_modules/moex-iss-api-client": { + "version": "0.4.2", + "resolved": "https://registry.npmjs.org/moex-iss-api-client/-/moex-iss-api-client-0.4.2.tgz", + "integrity": "sha512-gm0nI/d0aTprLhX5K8+3CD/S741juBSsG3ZlU1fkofOxPLyyGnIjrxxmHaCopmwoXG7OCeyP7Qm7QkImbKh3Ew==", + "license": "MIT", + "dependencies": { + "axios": "^0.28.0" + } + }, + "node_modules/moex-iss-api-client/node_modules/axios": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/axios/-/axios-0.28.1.tgz", + "integrity": "sha512-iUcGA5a7p0mVb4Gm/sy+FSECNkPFT4y7wt6OM/CDpO/OnNCvSs3PoMG8ibrC9jRoGYU0gUK5pXVC4NPXq6lHRQ==", + "license": "MIT", + "dependencies": { + "follow-redirects": "^1.15.0", + "form-data": "^4.0.0", + "proxy-from-env": "^1.1.0" + } + }, "node_modules/mrmime": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/mrmime/-/mrmime-2.0.1.tgz", @@ -28416,7 +28416,6 @@ "version": "1.1.0", "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==", - "dev": true, "license": "MIT" }, "node_modules/prr": { diff --git a/package.json b/package.json index 67eb101e1..e9b50e9d4 100644 --- a/package.json +++ b/package.json @@ -114,6 +114,7 @@ "jsonpath": "1.1.1", "lodash": "4.17.21", "marked": "17.0.1", + "moex-iss-api-client": "^0.4.2", "ms": "3.0.0-canary.1", "ng-extract-i18n-merge": "3.2.1", "ngx-device-detector": "11.0.0", diff --git a/prisma/migrations/20240501071657_added_moex_to_data_sources/migration.sql b/prisma/migrations/20240501071657_added_moex_to_data_sources/migration.sql new file mode 100644 index 000000000..639e7aa9b --- /dev/null +++ b/prisma/migrations/20240501071657_added_moex_to_data_sources/migration.sql @@ -0,0 +1,2 @@ +-- AlterEnum +ALTER TYPE "DataSource" ADD VALUE 'MOEX'; diff --git a/prisma/schema.prisma b/prisma/schema.prisma index 232dde9ca..da522101c 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -324,6 +324,7 @@ enum DataSource { MANUAL RAPID_API YAHOO + MOEX } enum MarketDataState { From 1cfc5106b27296530869cbc3878b982cf04abceb Mon Sep 17 00:00:00 2001 From: Anatoly Popov Date: Thu, 8 Jan 2026 00:18:16 +0300 Subject: [PATCH 4/4] chore(build): added script to update fork Signed-off-by: Anatoly Popov --- update-fork.sh | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100755 update-fork.sh diff --git a/update-fork.sh b/update-fork.sh new file mode 100755 index 000000000..8aa7a2ddf --- /dev/null +++ b/update-fork.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +set -eoux pipefail + +git fetch --all +git checkout main +git pull origin +git checkout feat/moex-service +git rebase main +git checkout --ours package-lock.json package.json && git add package-lock.json package.json +npm add moex-iss-api-client +git add package-lock.json package.json +git rebase --continue