Browse Source

use promise.all and address pr comments

pull/3723/head
Shaunak Das 12 months ago
parent
commit
f94a75c82d
  1. 34
      apps/api/src/services/data-provider/manual/manual.service.ts
  2. 2
      libs/common/src/lib/interfaces/scraper-configuration.interface.ts

34
apps/api/src/services/data-provider/manual/manual.service.ts

@ -166,15 +166,39 @@ export class ManualService implements DataProviderInterface {
} }
}); });
const profilesWithScraping = symbolProfiles.filter(
({ scraperConfiguration }) => {
return !!scraperConfiguration?.url;
}
);
const scrapingPromises = profilesWithScraping.map(
({ symbol, scraperConfiguration }) => {
return this.scrape(scraperConfiguration)
.then((marketPrice) => ({
symbol,
marketPrice
}))
.catch((error) => {
Logger.error(`Error scraping ${symbol}:`, error);
return { symbol, marketPrice: 0 };
});
}
);
// Wait for all scraping requests to complete concurrently
const scrapedResults = await Promise.all(scrapingPromises);
for (const { currency, symbol, scraperConfiguration } of symbolProfiles) { for (const { currency, symbol, scraperConfiguration } of symbolProfiles) {
const scrapedResult = scrapedResults.find(
(result) => result.symbol === symbol
);
let marketPrice = let marketPrice =
scrapedResult?.marketPrice ??
marketData.find((marketDataItem) => { marketData.find((marketDataItem) => {
return marketDataItem.symbol === symbol; return marketDataItem.symbol === symbol;
})?.marketPrice ?? 0; })?.marketPrice ??
0;
if (scraperConfiguration.mode === 'instant') {
marketPrice = await this.scrape(scraperConfiguration);
}
response[symbol] = { response[symbol] = {
currency, currency,

2
libs/common/src/lib/interfaces/scraper-configuration.interface.ts

@ -2,7 +2,7 @@ export interface ScraperConfiguration {
defaultMarketPrice?: number; defaultMarketPrice?: number;
headers?: { [key: string]: string }; headers?: { [key: string]: string };
locale?: string; locale?: string;
mode?: 'instant' | 'lazy';
selector: string; selector: string;
url: string; url: string;
mode?: 'instant' | 'lazy';
} }

Loading…
Cancel
Save