SHOW:
|
|
- or go back to the newest paste.
1 | const Apify = require('apify'); | |
2 | const { utils: { log } } = Apify; | |
3 | ||
4 | - | await requestQueue.addRequest({ url: 'http://localhost:5007/'}); |
4 | + | // use process.env with environment variables instead |
5 | - | const pseudoUrls = [new Apify.PseudoUrl('http://localhost:3000/[.*]')]; |
5 | + | const username = 'USERNAME'; // This can be saved in secret env variable |
6 | const password = 'PASSWORD'; // This can be saved in secret env variable | |
7 | ||
8 | const saveScreen = async (page, key = 'debug-screen') => { | |
9 | const screenshotBuffer = await page.screenshot({ fullPage: true }); | |
10 | - | console.log(`Title of ${request.url}: ${title}`); |
10 | + | await Apify.setValue(key, screenshotBuffer, { contentType: 'image/png' }); |
11 | }; | |
12 | ||
13 | Apify.main(async () => { | |
14 | const requestQueue = await Apify.openRequestQueue(); | |
15 | await requestQueue.addRequest({ url: 'https://staging-id.ogun.site/account/login'}); | |
16 | const pseudoUrls = [new Apify.PseudoUrl('https://staging-id.ogun.site/[.*]')]; | |
17 | ||
18 | let cookies = null; | |
19 | ||
20 | const crawler = new Apify.PuppeteerCrawler({ | |
21 | requestQueue, | |
22 | gotoFunction: async ({ request, page }) => { | |
23 | // If we are already logged in set cookies for the new page | |
24 | if (cookies) await page.setCookie(...cookies); | |
25 | /* | |
26 | // Debugging requests and responses | |
27 | page.on('request', (request) => { | |
28 | if (request.method() === 'POST') { | |
29 | log.info(`POST: ${request.url()}`); | |
30 | log.info(request.postData()); | |
31 | } | |
32 | }); | |
33 | page.on('response', (response) => { | |
34 | log.info(`${response.status()} - ${response.statusText()}: ${response.url()}`); | |
35 | }); | |
36 | */ | |
37 | // Goto to the url, wait until network is idle, timeout if network is not idle for 5 minutes | |
38 | return page.goto(request.url, { waitUntil: 'networkidle2', timeout: 5 * 60 * 1000 }); | |
39 | }, | |
40 | handlePageFunction: async ({ request, page }) => { | |
41 | // Handle login | |
42 | if (request.url === 'https://staging-id.ogun.site/account/login') { | |
43 | log.info(`Logging in`); | |
44 | // await saveScreen(page, 'login.png'); | |
45 | // Wait till login form loads | |
46 | await page.waitForSelector('.login-page', { timeout: 30 * 60000 }); | |
47 | log.info(`Login page loaded`); | |
48 | // Type username with 10ms delay between key presses | |
49 | await page.type('#Username', username, { delay: 10 }); | |
50 | log.info(`Username entered`); | |
51 | // await saveScreen(page, 'login-username.png'); | |
52 | // Type username with 10ms delay between key presses | |
53 | await page.type('#Password', password, { delay: 10 }); | |
54 | log.info(`Password entered`); | |
55 | // await saveScreen(page, 'login-password.png'); | |
56 | // Click on remember me box | |
57 | await page.click('#RememberLogin'); | |
58 | log.info(`Remember me checked`); | |
59 | // await saveScreen(page, 'login-remember-me.png'); | |
60 | // Click on login buttton | |
61 | await page.click('.login-page button[type="submit"]'); | |
62 | log.info(`Login button pressed`); | |
63 | ||
64 | // Wait for settings page to load | |
65 | await page.waitForSelector('.settings-page', { timeout: 60 * 1000 }); | |
66 | log.info('Settings page loaded'); | |
67 | // await saveScreen(page, 'settings.png'); | |
68 | // Save cookies | |
69 | cookies = await page.cookies(); | |
70 | } | |
71 | // Handle other pages | |
72 | const title = await page.title(); | |
73 | log.info(`Title of ${request.url}: ${title}`); | |
74 | await Apify.pushData({ url: page.url(), title }); | |
75 | await Apify.utils.puppeteer.enqueueLinks({ page, selector: 'a', pseudoUrls, requestQueue }); | |
76 | } | |
77 | }); | |
78 | await crawler.run(); | |
79 | }); |