import numpy
import shutils
import time

def fast():
    CHANNELS = 16
    SAMPLE_LENGTH = CHANNELS // 8
    BUFFER = 256
    PROCNUM = 4

    vals = []
    for i in range(2**CHANNELS):
        vals.append(tuple(b'1' if e == '1' else b'0' for e in list("{:016b}".format(i))))

    data = np.fromfile('largefile.dat', dtype=np.uint16)
    size = len(data)
    print("Data size: %d" % size)

    def work(data, vals, proc):
        size = len(data)

        print("Process %d data size: %d" % (proc, size))

        out_list = []
        for i in range(CHANNELS):
            out_file = open('dir/largefile_dump_%d_%d.dat' % (i, proc), 'wb')
            out_list.append(out_file)

        samples = []
        j = 0

        for sample in data:
            samples.append(vals[sample])
            if j >= BUFFER:
                samples = zip(*samples)
                for channel, bits in enumerate(samples):
                    s = b''.join(bits)
                    out_list[channel].write(s)
                samples = []
                j = 0
            j += 1

        if samples:
            samples = zip(*samples)
            for channel, bits in enumerate(samples):
                s = b''.join(bits)
                out_list[channel].write(s)

        for i in range(CHANNELS):
            out_list[i].close()

        print("Process %d is complete" % proc)

    samples_per_process = (size // SAMPLE_LENGTH) // PROCNUM
    procs = []

    for i in range(PROCNUM):
        d_size = SAMPLE_LENGTH*samples_per_process
        if i != (PROCNUM - 1):
            d = data[i*d_size:(i+1)*d_size]
        else:
            d = data[i*d_size:]
        p = Process(target=work, args=(d, vals, i))
        procs.append(p)
        p.start()

    while True:
        if not any(Process.is_alive(proc) for proc in procs):
            break
        time.sleep(0.5)

    print("STARTING TO MERGE FILES!")

    for i in range(CHANNELS):
        files = ['dir/%s' % f for f in os.listdir('dir') if '_%d_' % i in f]
        with open("dir/channel_%d.dat" % i, 'wb') as w:
            for f in files:
                with open(f, 'rb') as r:
                    shutil.copyfileobj(r, w, 10*1024*1024)

fast()