// Database splitter and flattener.
// splits based on <fieldname>
// flattens arrays and objects

// usage: In CLI: 'mongo <dbname> < dbflatten.js'

// Specify collection to be flattened

col = db.<collectionname>

// Batches inserts in size 'batchSize' to conserve memory.
// Probably there's an optimal batchSize number.
function createCollection(x, batchSize = 20000){

  // recursive document flattener
  var newdoc = {}
  function parseDocKeys(doc, key){
    let keys = Object.keys(doc)
    for (let k in keys){
      let nextkey = key+'_'+keys[k]
      let nextdoc = doc[keys[k]]

      // for some unknown reason, we have to treat the timestamp and _id fields separately, even though
      //    they contain strings like many of the other fields
      if (keys[k] == 'timestamp'){
        newdoc[nextkey] = nextdoc
      } else if (keys[k] == '_id'){
        newdoc[nextkey] = nextdoc

      // Proceed with recursion if we find another Object
      } else if (nextdoc instanceof Object) {
        parseDocKeys(nextdoc, nextkey)

      // Default end of recursion
      } else {
        newdoc[nextkey] = nextdoc
      }
    }
    return
  };

	// get all docs for model
	var docs = col.find({"<fieldname>": {$eq : x}})

	// Create the collection, batch by batch
    var head_key = x
	var counter = 0
	var docContainer = []

	while (docs.hasNext()){
		while ((counter < batchSize) && (docs.hasNext()) ) {
			counter += 1
			doc = docs.next()
			parseDocKeys(doc, head_key)
			docContainer.push(newdoc)
			newdoc = {}
		}
		counter = 0

		// Insert the batch in a new collection defined by model
		// I don't know... insert or insertMany?
		db[model].insert(docContainer)
		docContainer = []
}

// ==========================
//
//  Main loop
//

//get splitter values from the db.<collectionname>
var xs = col.distinct("<fieldname>")
xsIterator = xs.entries()
for (let m of xsIterator){
	let xx = m[1]
	print(xx)
	createCollection(xx);

	// examine a sample flat document
	printjson(db[xx].findOne())
}