ESC Scraper

var https = require('https');

var charList = 'qwertzuiopasdfghjklyxcvbnmQWERTZUIOPASDFGHJKLYXCVBNM1234567890';
const plainURL = "https://showcase-emp.ebu.ch/showcase/embedded.html?tag=dm9kOz";

//Unused atm
//const appendedLength = 6;
//var runningRequests = 0;
//var maxRequests = 1;

//Startvalues
//var a = 14, b = 5, c = 51, d = 38, e = 45, f = 45;
var a = 0, b = 0, c = 0, d = 0, e = 0, f = 0;


//example: ("https://showcase-emp.ebu.ch/showcase/embedded.html?tag=dm9kOzgzMDY1")
//example: ("https://showcase-emp.ebu.ch/showcase/embedded.html?tag=dm9kOzgzMDY0")



//Loop doesnt wor for now.
//Would be faster ;)
//for (i = 0; i < 30; i++) {

    checkNexturl();
//}


function checkNexturl() {

    var urlAppender = getNextUrlAppendix();

    if (urlAppender == "DONE")
        return;

    completeURL = plainURL + urlAppender;
    https.get(completeURL, function (res) {
        var pageData = "";

        //res.resume();
        res.on('data', function (chunk) {
            pageData += chunk;
        });

        res.on('end', function () {
            if (pageData.indexOf("Video not available") > -1) {
                console.log("NOK;" + completeURL);
            }
            else {
                //Prevent newline
                process.stdout.write("OK ;" + completeURL);

                var start = pageData.indexOf("<div class=\"desc_file\">");
                if (start > 0) {
                    var end = pageData.indexOf("</div>", start);
                    if (end > 0) {

                        process.stdout.write("; Title;" + pageData.substring(start + 23, end));
                    }
                }
                process.stdout.write("\n");
            }
            checkNexturl(); //call the next url to fetch
        });

    }).on('error', function (e) {
        console.log("Error: " + options.host + "\n" + e.message);
        checkNexturl(); //call the next url to fetch
    });

}


function getNextUrlAppendix() {
    var newUrl = "";

    newUrl = charList[a] + charList[b] + charList[c] + charList[d] + charList[e] + charList[f];

    f++;
    if (f >= charList.length) {
        f = 0;
        e++;
    }
    if (e >= charList.length) {
        e = 0;
        d++;
    }
    if (d >= charList.length) {
        d = 0;
        c++;
    }
    if (c >= charList.length) {
        c = 0;
        b++;
    }
    if (b >= charList.length) {
        b = 0;
        a++;
    }
    if (a >= charList.length) {
        return "DONE"
    }


    return newUrl;

}