var http = require("http");
var https = require("https");
var fs = require('fs');

var start = 116106;
var stop  = 117107;

// Câte requesturi să se facă maxim odată
var at_once = 200;
// Pauza de la momentul când a fost atins maximul
var delay = 2000;


var in_quee = 0;
var found = 0;
var checked = 0;
start = parseInt(start, 10);
stop = parseInt(stop, 10);
function process(url, id){
    //var url = 'http://www.internetculturale.it/jmms/iccuviewer/iccu.jsp?id=oai%3Awww.internetculturale.sbn.it%2FTeca%3A20%3ANT0000%3ACNMD\\0000117105';
    var req = (url.startsWith("https://")?https:http).get(url, function(res) {
        var resp = 0;
        in_quee--;
        checked++;
        res.on('data', function(chunk){resp+=chunk.length;});
        res.on('end', function(){
            //resp;
            if(resp > 50){
                //este valid
                found++;
                link = 'http://www.internetculturale.it/jmms/iccuviewer/iccu.jsp?id=oai%3Awww.internetculturale.sbn.it%2FTeca%3A20%3ANT0000%3ACNMD\\\\'+id+'&mode=all&teca=MagTeca+-+ICCU';
                fs.appendFile('valid_links.txt', link+"\r\n");
                console.log('['+id+'] Found document... ('+found+' found, '+checked+' checked)');
            }else{
                console.log('['+id+'] Invalid document... ('+found+' found, '+checked+' checked)');
            }
        });
    });
    req.on("error", function(err,data) {
      console.log("[Error]");
      setTimeout(function(){process(url, id);}, parseInt(Math.random()*8000));
    });
    req.setTimeout(parseInt(Math.random()*8000)+10000, function( ) {
      process(url, id);
    });
}
function repeater(i){
    if(i>stop) return;
    if(in_quee>=at_once){
        //console.log(checked+" checked documents / "+found+" valid documents found");
        setTimeout(function(){
            repeater(i);
        }, 2000);
    }else{
        in_quee++;
        var id = ("0000000000" + i).slice(-10);
        console.log("Requesting check for "+id+"...");
        process('http://www.internetculturale.it/jmms/magparser?teca=MagTeca+-+ICCU&mode=all&fulltext=0&id=oai%3Awww.internetculturale.sbn.it%2FTeca%3A20%3ANT0000%3ACNMD%5C%5C'+id, id);
        return repeater(i+1);
    }
}
repeater(start);
//for(var i=start;i<=stop;i++){


//}