const MongoClient = require("mongodb").MongoClient;
const fs = require("fs");
const XmlStream = require("xml-stream");

const client = new MongoClient("mongodb://localhost:27017", {
  useNewUrlParser: true,
  autoReconnect: true,
  poolSize: 3,
  reconnectTries: 60,
  reconnectInterval: 1000
});

client.connect(function(err) {
  if (err) {
    throw new Error("db connection error", err);
  }
  console.log("Connected successfully to server");

  let c = 0;
  let Errors = [];
  const db = client.db("opendata");
  const collection = db.collection("edrpou");

  function saveData(data) {
    if (data instanceof Object) {
      xml.pause();
      data["$"] ? delete data["$"] : null;
      collection.insertOne(data, (err, res) => {
        if (err) {
          console.error("insertOne: ", err);
          Errors.push(err);
        } else {
          console.clear();
          console.log(data);
          console.info("➕  ", c);
          xml.resume();
        }
      });
    }
  }

  let batchTemp = [];
  function saveDataBatch(data) {
    if (data instanceof Object) {
      xml.pause();
      data["$"] ? delete data["$"] : null;
      if (batchTemp.length < 100) {
        batchTemp.push(data);
      } else {
        batchTemp.push(data);
        collection.insertMany(batchTemp, (err, res) => {
          if (err) {
            console.error("insertMany: ", err);
            Errors.push(err);
          } else {
            console.clear();
            console.log(data);
            console.info("➕  ", c);
            batchTemp = [];
            xml.resume();
          }
        });
      }
    }
  }

  const streamWrite = fs.createWriteStream("./data/extracted/data.json");
  function saveToFile(data) {
    streamWrite.write(data);
  }

  const stream = fs.createReadStream("./data/extracted/edrpou.xml");
  const xml = new XmlStream(stream);

  xml.on("endElement: SUBJECT", function(item) {
    saveData(item);
    c++;
  });

  xml.on("end", function() {
    // client.close();
    console.error(JSON.stringify(Errors));
    console.log("🔥  END");
  });
});