Advertisement
Guest User

Untitled

a guest
Feb 27th, 2019
148
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 1.22 KB | None | 0 0
  1. import Promise from "bluebird";
  2. import Selenium from "./Selenium";
  3. import Database from "./Database";
  4.  
  5. export default class ReviewScrape {
  6.  
  7. constructor(config = {
  8. database: {
  9. host: "localhost",
  10. user: "root",
  11. password: "root",
  12. database: "centraleyes_portal_bw"
  13. }
  14. }) {
  15. this._config = config;
  16. this._selenium = new Selenium();
  17. }
  18.  
  19. start() {
  20. return this._selenium.start()
  21. .then(() => Database.connect(this._config.database));
  22. }
  23.  
  24. execute(scrapers) {
  25. let count = 0;
  26. return Promise.map(scrapers, scraper => this._execute(scraper).then(() => count += scraper.getScrapedCount()), {concurrency: 1})
  27. .then(() => count);
  28. }
  29.  
  30. stop() {
  31. return this._selenium.stop()
  32. .then(() => Database.close());
  33. }
  34.  
  35. _execute(scraper) {
  36. console.log("--- " + scraper.getUri());
  37. const browser = this._selenium.getInstance();
  38. return browser.setViewportSize({ width: 1920, height: 1080 }, false)
  39. .then(() => browser.url(scraper.getUri()))
  40. .then(() => scraper.scrape(browser))
  41. .end();
  42. }
  43.  
  44. }
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement