sharding.py

#!/usr/bin/env python
# encoding: utf-8

import os
import sys
import hashlib

def parse_and_shard(srr_releases_fn, action):
    with open(srr_releases_fn) as fileobject:
        for line in fileobject:
            action(line.strip())

def shard_file(rls_name):
    #print(repr(rls_name))
    count = 0
    sha1 = hashlib.sha1(rls_name.lower().encode('utf8')).hexdigest()
    srr = rls_name + '.srr'
    if os.path.exists(srr):
        count += 1
        destination = os.path.join(sha1[0], sha1[1:3], srr)
        os.renames(srr, destination)
        if count % 10000 == 0:
            print("%d - %s" % (count, destination))

if len(sys.argv) == 1:
    print("Script works on the current directory!")
    print("Do the initial sorting based on list of release names:")
    print("  sharding.py sharding.txt")
    print("Sort all other srrs:")
    print("  sharding.py --")
elif (len(sys.argv) == 2 and sys.argv[1] == "--"):
    # by listing all files
    for f in os.listdir(os.curdir):
        if f.endswith(".srr"):
            shard_file(os.path.basename(f)[:-4])
    print("Done.")
elif len(sys.argv) == 2:
    # use text file with known release names to speed up listing all files
    parse_and_shard(sys.argv[1], shard_file)
    print("Done.")
else:
    print("Parameters not understood")