Advertisement
Guest User

Untitled

a guest
Jan 24th, 2019
147
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
  1. const Apify = require('apify');
  2. const { utils: { log } } = Apify;
  3.  
  4. // use process.env with environment variables instead
  5. const username = 'USERNAME'; // This can be saved in secret env variable
  6. const password = 'PASSWORD'; // This can be saved in secret env variable
  7.  
  8. const saveScreen = async (page, key = 'debug-screen') => {
  9.     const screenshotBuffer = await page.screenshot({ fullPage: true });
  10.     await Apify.setValue(key, screenshotBuffer, { contentType: 'image/png' });
  11. };
  12.  
  13. Apify.main(async () => {
  14.     const requestQueue = await Apify.openRequestQueue();
  15.     await requestQueue.addRequest({ url: 'https://staging-id.ogun.site/account/login'});
  16.     const pseudoUrls = [new Apify.PseudoUrl('https://staging-id.ogun.site/[.*]')];
  17.  
  18.     let cookies = null;
  19.  
  20.     const crawler = new Apify.PuppeteerCrawler({
  21.         requestQueue,
  22.         gotoFunction: async ({ request, page }) => {
  23.             // If we are already logged in set cookies for the new page
  24.             if (cookies) await page.setCookie(...cookies);
  25.             /*
  26.             // Debugging requests and responses
  27.             page.on('request', (request) => {
  28.                 if (request.method() === 'POST') {
  29.                     log.info(`POST: ${request.url()}`);
  30.                     log.info(request.postData());
  31.                 }
  32.             });
  33.             page.on('response', (response) => {
  34.                 log.info(`${response.status()} - ${response.statusText()}: ${response.url()}`);
  35.             });
  36.             */
  37.             // Goto to the url, wait until network is idle, timeout if network is not idle for 5 minutes
  38.             return page.goto(request.url, { waitUntil: 'networkidle2', timeout: 5 * 60 * 1000 });
  39.         },
  40.         handlePageFunction: async ({ request, page }) => {
  41.             // Handle login
  42.             if (request.url === 'https://staging-id.ogun.site/account/login') {
  43.                 log.info(`Logging in`);
  44.                 // await saveScreen(page, 'login.png');
  45.                 // Wait till login form loads
  46.                 await page.waitForSelector('.login-page', { timeout: 30 * 60000 });
  47.                 log.info(`Login page loaded`);
  48.                 // Type username with 10ms delay between key presses
  49.                 await page.type('#Username', username, { delay: 10 });
  50.                 log.info(`Username entered`);
  51.                 // await saveScreen(page, 'login-username.png');
  52.                 // Type username with 10ms delay between key presses
  53.                 await page.type('#Password', password, { delay: 10 });
  54.                 log.info(`Password entered`);
  55.                 // await saveScreen(page, 'login-password.png');
  56.                 // Click on remember me box
  57.                 await page.click('#RememberLogin');
  58.                 log.info(`Remember me checked`);
  59.                 // await saveScreen(page, 'login-remember-me.png');
  60.                 // Click on login buttton
  61.                 await page.click('.login-page button[type="submit"]');
  62.                 log.info(`Login button pressed`);
  63.  
  64.                 // Wait for settings page to load
  65.                 await page.waitForSelector('.settings-page', { timeout: 60 * 1000 });
  66.                 log.info('Settings page loaded');
  67.                 // await saveScreen(page, 'settings.png');
  68.                 // Save cookies
  69.                 cookies = await page.cookies();
  70.             }
  71.             // Handle other pages
  72.             const title = await page.title();
  73.             log.info(`Title of ${request.url}: ${title}`);
  74.             await Apify.pushData({ url: page.url(), title });
  75.             await Apify.utils.puppeteer.enqueueLinks({ page, selector: 'a', pseudoUrls, requestQueue });
  76.         }
  77.     });
  78.     await crawler.run();
  79. });
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement