View difference between Paste ID: 0znaTXFG and ZQcG6hZE
SHOW: | | - or go back to the newest paste.
1
const Apify = require('apify');
2
const { utils: { log } } = Apify;
3
4-
    await requestQueue.addRequest({ url: 'http://localhost:5007/'});
4+
// use process.env with environment variables instead
5-
    const pseudoUrls = [new Apify.PseudoUrl('http://localhost:3000/[.*]')];
5+
const username = 'USERNAME'; // This can be saved in secret env variable
6
const password = 'PASSWORD'; // This can be saved in secret env variable
7
8
const saveScreen = async (page, key = 'debug-screen') => {
9
    const screenshotBuffer = await page.screenshot({ fullPage: true });
10-
            console.log(`Title of ${request.url}: ${title}`);
10+
    await Apify.setValue(key, screenshotBuffer, { contentType: 'image/png' });
11
};
12
13
Apify.main(async () => {
14
    const requestQueue = await Apify.openRequestQueue();
15
    await requestQueue.addRequest({ url: 'https://staging-id.ogun.site/account/login'});
16
    const pseudoUrls = [new Apify.PseudoUrl('https://staging-id.ogun.site/[.*]')];
17
18
    let cookies = null;
19
20
    const crawler = new Apify.PuppeteerCrawler({
21
        requestQueue,
22
        gotoFunction: async ({ request, page }) => {
23
            // If we are already logged in set cookies for the new page
24
            if (cookies) await page.setCookie(...cookies);
25
            /* 
26
            // Debugging requests and responses
27
            page.on('request', (request) => {
28
                if (request.method() === 'POST') {
29
                    log.info(`POST: ${request.url()}`);
30
                    log.info(request.postData());
31
                }
32
            });
33
            page.on('response', (response) => {
34
                log.info(`${response.status()} - ${response.statusText()}: ${response.url()}`);
35
            });
36
            */
37
            // Goto to the url, wait until network is idle, timeout if network is not idle for 5 minutes
38
            return page.goto(request.url, { waitUntil: 'networkidle2', timeout: 5 * 60 * 1000 });
39
        },
40
        handlePageFunction: async ({ request, page }) => {
41
            // Handle login
42
            if (request.url === 'https://staging-id.ogun.site/account/login') {
43
                log.info(`Logging in`);
44
                // await saveScreen(page, 'login.png');
45
                // Wait till login form loads
46
                await page.waitForSelector('.login-page', { timeout: 30 * 60000 });
47
                log.info(`Login page loaded`);
48
                // Type username with 10ms delay between key presses
49
                await page.type('#Username', username, { delay: 10 });
50
                log.info(`Username entered`);
51
                // await saveScreen(page, 'login-username.png');
52
                // Type username with 10ms delay between key presses
53
                await page.type('#Password', password, { delay: 10 });
54
                log.info(`Password entered`);
55
                // await saveScreen(page, 'login-password.png');
56
                // Click on remember me box
57
                await page.click('#RememberLogin');
58
                log.info(`Remember me checked`);
59
                // await saveScreen(page, 'login-remember-me.png');
60
                // Click on login buttton
61
                await page.click('.login-page button[type="submit"]');
62
                log.info(`Login button pressed`);
63
64
                // Wait for settings page to load
65
                await page.waitForSelector('.settings-page', { timeout: 60 * 1000 });
66
                log.info('Settings page loaded');
67
                // await saveScreen(page, 'settings.png');
68
                // Save cookies
69
                cookies = await page.cookies();
70
            }
71
            // Handle other pages
72
            const title = await page.title();
73
            log.info(`Title of ${request.url}: ${title}`);
74
            await Apify.pushData({ url: page.url(), title });
75
            await Apify.utils.puppeteer.enqueueLinks({ page, selector: 'a', pseudoUrls, requestQueue });
76
        }
77
    });
78
    await crawler.run();
79
});