etl.py

#!/usr/bin/env python

import sys
import json
import os
def main():
 try:
    if len(sys.argv) >= 3:
        dirin = sys.argv[1]
        dirout = sys.argv[2]
    else:
        print 'Missing: required dir for input and dir for output'
        return 1 # indicates errorlessly exit, but not necessary
    count = 1

    for file in os.listdir(dirin):
        if file.endswith(".json"):
            with open(os.path.join(dirin, file), 'r') as f:
                data = json.load(f)
            data_new = {}
            data_new['id'] = str(count)
            data_new['published'] = data['published']
            data_new['text'] = data['text']
            data_new['title'] = data['title']
            data_new['author'] = data['author']
            data_new['url'] = data['url']
            with open(os.path.join(dirout, file), 'w') as f:
                json.dump(data_new, f)
            count = count+1

 except Exception as ex:
     print str(ex)
     return 1 # indicates error, but not necessary
 else:
    return 0 # indicates errorlessly exit, but not necessary
# this is the standard boilerplate that calls the main() function
if __name__ == '__main__':
    # sys.exit(main(sys.argv)) # used to give a better look to exists
    main()