Python Script used with NiFi

#!/usr/bin/env python

import datetime
import getopt
import httplib
import sys
import time
import urllib
import zlib
import ast
from collections import OrderedDict
import json
import os


# defaults

## URL stuff
URL             = "data.ip.net"
PORT            = "80"
API_ENDPOINT    = URL+":"+PORT


## POST body stuff
APIKEY          = "123456789abcdefghi"
FORMAT          = "json"
METHOD          = "delta"
TS              = ""


## other stuff
timestamp          = time.time()
timestamp_extended = datetime.datetime.fromtimestamp(timestamp).strftime('%Y%m%d_%H%M')
verbose            = ""
print_console      = False
morph_json         = True

# /defaults

headers = {"Content-type": "application/x-www-form-urlencoded"}

options, remainder = getopt.getopt(sys.argv[1:], 'a:f:m:r:c:t:v:o:j:gspVh', [
                                                        'apikey=',
                                                        'format=',
                                                        'method=',
                                                        'ts=',
                                                        'version=',
                                                        'output=',
                                                        ])

def usage():
    print ' -------------------------------------------------------------------------'
    print ' Typical usage:'
    print ' '
    print ' -a, --apikey          api key'
    print ' -f, --format          format [ csv, json, splunk ]'
    print ' -m, --method          method [ full, delta, docs, heartbeat ]'
    print ' -t, --ts              time slice  [ UNIX timestamp ]'
    print ' -j, --morph-json      turn format=json into a real json string [ singleline, multiline ]'
    print ' -p, --print           print output to console'
    print ' -V, --verbose         enable verbose output'
    print ' -h, --help            this help'

    print ' -------------------------------------------------------------------------'
    sys.exit(' ')

for opt, arg in options:
    if opt in ('-a', '--apikey'):
        APIKEY = arg
    elif opt in ('-f', '--format'):
        FORMAT = arg
    elif opt in ('-m', '--method'):
        METHOD = arg
    elif opt in ('-t', '--ts'):
        TS = arg
    elif opt in ('-v', '--version'):
        VERSION = arg
    elif opt in ('-j', '--morph-json'):
        morph_json = arg
        if FORMAT != "json":
            print "[!] json format required to use -j or --morph-json"
            sys.exit(1)
    elif opt in ('-V', '--verbose'):
        verbose = True
    elif opt in ('-h', '--help'):
        usage()

params = urllib.urlencode(
    {
        'apikey': APIKEY,
        'method': METHOD,
        'ts'    : TS,
        'v':      VERSION,
    }
)

if verbose:
    print 'posting data to   :', API_ENDPOINT
    for k, v in headers.items():
        print 'using headers     :', k, v
    print 'using post string :', params

conn = httplib.HTTPConnection(API_ENDPOINT)
conn.request("POST", "/", params, headers)

response = conn.getresponse()

if morph_json:

    data = response.read().splitlines()

    last_line = data.pop(-1)

    if last_line != "Version:2":
            print "something went wrong"
            sys.exit(1)

    header = data.pop(0)
    header = ast.literal_eval(header)
    entry_header = header[-1][0]
    records = []

    for line in data:
        line = line.replace("null", "\"\"")
        line = ast.literal_eval(line)
        out = OrderedDict(zip(header, line[0:-1]))
        entries = []
        for entry in line[-1]:
            entries.append(dict(zip(entry_header, entry)))

        out["entries"] = entries
        records.append(out)
        #print out
        sys.stdout.write(out)

conn.close()