Ledger Nano X - The secure hardware wallet
SHARE
TWEET

Untitled

a guest Apr 10th, 2020 140 Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
  1. const fetch = require('node-fetch');
  2. const cheerio = require('cheerio');
  3. const fs = require('fs');
  4.  
  5. const FILENAME = 'items.json';
  6. const SITE_URL = 'https://www.dns-shop.ru';
  7. const PARSE_URL = 'https://www.dns-shop.ru/catalog/17a8a0f116404e77/elektronnye-knigi/';
  8. const STEP_SIZE = 3;
  9.  
  10. const Selectors = {
  11.     product: '.n-catalog-product',
  12.     productName: '.price-item-title',
  13.     price: '.current-price-value',
  14.     description: '.price-item-description > p',
  15.     link: '.product-info__title-link > a',
  16.     productCode: '.price-item-code > span',
  17.     paramsRows: '.table-params tbody tr',
  18.     tablePart: '.table-part',
  19.     paramName: '.dots > span',
  20.     paramValue: '> td:last-child'
  21. };
  22.  
  23. async function parseItem(url) {
  24.     try {
  25.         const response = await fetch(SITE_URL + url);
  26.         const body = await response.text();
  27.         const $ = cheerio.load(body);
  28.         const price = parseFloat($(Selectors.price).eq(0).text().replace(/\s/g, ''));
  29.         const code = parseInt($(Selectors.productCode).text());
  30.         const name = $(Selectors.productName).text();
  31.         const description = $(Selectors.description).text();
  32.         const rows = $(Selectors.paramsRows);
  33.  
  34.         const properties = {};
  35.         let currentPart = 'none';
  36.         rows.each((i, row) => {
  37.             const part = $(row).find(Selectors.tablePart).eq(0);
  38.  
  39.             if (part.length > 0) {
  40.                 currentPart = part.text();
  41.                 properties[currentPart] = {};
  42.             } else {
  43.                 const property = $(row).find(Selectors.paramName).text().trim();
  44.                 const value = $(row).find(Selectors.paramValue).text().trim();
  45.                 properties[currentPart][property] = value;
  46.             }
  47.         });
  48.  
  49.         return { name, price, code, description, properties };
  50.     } catch (error) {
  51.         console.error(error);
  52.         throw error;
  53.     }
  54. }
  55.  
  56. async function parsePage(page = 1, items = []) {
  57.     try {
  58.         const response = await fetch(`${PARSE_URL}?p=${page}`);
  59.         const body = await response.text();
  60.         const $ = cheerio.load(body);
  61.         const products = $(Selectors.product).toArray();
  62.  
  63.         if (!products.length) {
  64.             return items;
  65.         }
  66.  
  67.         console.log('Обрабатываю страницу:', page);
  68.  
  69.         const parseItemQueries = products.map(item => {
  70.             const link = $(item).find(Selectors.link).attr('href');
  71.             return parseItem(link);
  72.         });
  73.  
  74.         const parsedItems = await Promise.all(parseItemQueries);
  75.         const nextItems = items.concat(parsedItems);
  76.  
  77.         return parsePage(page + STEP_SIZE, nextItems);
  78.     } catch (error) {
  79.         console.error('Парсинг не удался.', error);
  80.     }
  81. }
  82.  
  83. async function main() {
  84.     try {
  85.         const parsers = new Array(STEP_SIZE).fill(null).map((_, i) => parsePage(i + 1));
  86.         const results = await Promise.all(parsers);
  87.         const items = results.flat();
  88.         console.log('Кол-во собранных товаров:', items.length);
  89.         fs.writeFileSync(FILENAME, JSON.stringify(items));
  90.         console.log(`Успешно записали результат в файл ${FILENAME}!`);
  91.     } catch (error) {
  92.         console.error(error);
  93.         process.exit(1);
  94.     }
  95. }
  96.  
  97. main();
RAW Paste Data
We use cookies for various purposes including analytics. By continuing to use Pastebin, you agree to our use of cookies as described in the Cookies Policy. OK, I Understand
Top