etl

import sys
import json
import os
def main():
 try:
 if len(sys.argv) >= 3:
 dirin = sys.argv[1]
 dirout = sys.argv[2]
 else:
 print 'Missing: required dir for input and dir for output'
 return 1 # indicates errorlessly exit, but not necessary
 count = 1
for file in os.listdir(dirin):
 if file.endswith(".json"):
with open(os.path.join(dirin, file), 'r') as f:
 data = json.load(f)
 data_new = {}
data_new['id'] = str(count)
data_new['published'] = data['published']
data_new['text'] = data['text']
data_new['title'] = data['title']
data_new['author'] = data['author']
 data_new['url'] = data['url']
with open(os.path.join(dirout, file), 'w') as f:
 json.dump(data_new, f)
 count = count+1
 except Exception as ex:
 print ex.message
 return 1 # indicates error, but not necessary
 else:
 return 0 # indicates errorlessly exit, but not necessary
# this is the standard boilerplate that calls the main() function
if __name__ == '__main__':
 # sys.exit(main(sys.argv)) # used to give a better look to exists
 main()