Download Kraken trades

#!/usr/bin/env python3

import krakenex
import os
import h5py
import numpy as np
import time
import datetime

k = krakenex.API()
c = krakenex.Connection()

until = np.uint64(1525564800000000000) # 2018-05-06 00:00:00 GMT

def download_trades(pair):
  path = "download/%s.h5" % pair

  if os.path.exists(path):
    print("%s already exists, appending" % path)
  else:
    print("Creating %s" % path)

  h5 = h5py.File(path, "a")

  last = np.uint64(0)

  if "last" in h5.attrs.keys():
    last = h5.attrs.get("last")
  else:
    h5.attrs.create("last", last, dtype=np.uint64)

  size = 0
  if ("time_price_vol" in h5.keys()):
    size = h5["time_price_vol"].shape[0]

  h5.close()

  # main download loop
  while (last < until):
    while True:
      try:
        q = k.query_public("Trades", { "pair": pair, "since": last }, c)
        if ("error" in q.keys()):
          if (len(q["error"]) == 0):
            if ("result" in q.keys()):
              break
          else:
            print("Error: ", q["error"])
            time.sleep(5)
      except ValueError as err:
        print("Exception: ", err)

      time.sleep(1)

    res = q["result"]
    ts = res[pair]

    n = len(ts)

    h5 = h5py.File(path, "a")

    time_price_vol = h5.require_dataset("time_price_vol", shape=(size, 3),
        dtype=np.float64, chunks=(1000, 3), maxshape=(None, 3),
        compression="gzip", compression_opts=7)

    buy_market = h5.require_dataset("buy_market", shape=(size, 2),
        dtype=np.bool_, chunks=(1000, 2), maxshape=(None, 2),
        compression="gzip", compression_opts=7)

    time_price_vol.resize((size + n, 3))
    buy_market.resize((size + n, 2))

    time_price_vol[size:,0] = [float(t[2]) for t in ts]
    time_price_vol[size:,1] = [float(t[0]) for t in ts]
    time_price_vol[size:,2] = [float(t[1]) for t in ts]
    buy_market[size:,0] = [t[3] == "b" for t in ts]
    buy_market[size:,1] = [t[4] == "m" for t in ts]

    last = np.uint64(res["last"])
    h5.attrs.modify("last", last)

    h5.close()

    datestr = datetime.datetime.utcfromtimestamp(last/1e9).strftime('%Y-%m-%d %H:%M:%S GMT')
    print("Downloaded %s up to %s (%lu)" % (pair, datestr, last))
    size = size + n

f = open("pairs", "r")
pairs = [l.strip() for l in f.readlines()]

for pair in pairs:
  download_trades(pair)