Advertisement
Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- import Promise from "bluebird";
- import Selenium from "./Selenium";
- import Database from "./Database";
- export default class ReviewScrape {
- constructor(config = {
- database: {
- host: "localhost",
- user: "root",
- password: "root",
- database: "centraleyes_portal_bw"
- }
- }) {
- this._config = config;
- this._selenium = new Selenium();
- }
- start() {
- return this._selenium.start()
- .then(() => Database.connect(this._config.database));
- }
- execute(scrapers) {
- let count = 0;
- return Promise.map(scrapers, scraper => this._execute(scraper).then(() => count += scraper.getScrapedCount()), {concurrency: 1})
- .then(() => count);
- }
- stop() {
- return this._selenium.stop()
- .then(() => Database.close());
- }
- _execute(scraper) {
- console.log("--- " + scraper.getUri());
- const browser = this._selenium.getInstance();
- return browser.setViewportSize({ width: 1920, height: 1080 }, false)
- .then(() => browser.url(scraper.getUri()))
- .then(() => scraper.scrape(browser))
- .end();
- }
- }
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement