msp

#!/usr/bin/python
#===========================================================================

#  This script parses "partition.xml" and creates numerous output files
#  specifically, partition.bin, rawprogram.xml

# REFERENCES

#  $Header: //source/qcom/qct/core/storage/tools/emmc/src/jsdcc/partition_load_pt/msp.py#18 $
#  $DateTime: 2011/08/09 20:02:16 $
#  $Author: ahughes $

# when          who     what, where, why
# --------      ---     -------------------------------------------------------
# 2011-08-09    ah      -t option specifies output directory for files
# 2011-08-01    ah      Bug fix whhen patching DISK
# 2011-07-26    ah      Much better error messages
# 2011-07-21    ah      Major changes, now using getopt(), patching/writing can be all one step
# 2011-07-13    ah      Better error messages for user experience
# 2011-06-01    ah      Improved singleimage support for GPT
# 2011-05-26    ah      Handles sparse files and "zeroout" tag
# 2011-05-17    ah      Removed auto-patching MBR - must manually patch just like in GPT
# 2011-05-02    ah      No longer querying for /proc/partitions by default (user must ask now)
# 2011-03-22    ah      Enforcing that msp can't patch/write to partitions other than USER
# 2011-03-22    ah      Fixed memory bug with large file writing, fixed bug with SINGLEIMAGE creation (i.e. append)
# 2011-03-16    ah      Fixed memory bug with large file patching and read/write permissions for
#                       SCons and singleimage.bin, Now need user input when patching DISK, "sync" command for linux
# 2011-03-09    ah      Replaces msp.exe - compatible with Linux and Windows
#                       does not need cygwin, cleaner code, more error checking
# 2011-02-14    ah      Changed 'dd' to 'cat' to avoid error happening on Win7
# 2010-10-26    ah      Now "patches" partition table on the fly, adds warnings before nuking drive
# 2010-10-22    ah      Used to replace msp.exe - can program cards and singleimages.bin

# Copyright (c) 2007-2010
# Qualcomm Incorporated.
# All Rights Reserved.
# Qualcomm Confidential and Proprietary
# ===========================================================================*/

import struct, os, sys, getopt
import math
import re
import codecs
from types import *
import time
import subprocess as sub

from xml.etree import ElementTree as ET
#from elementtree.ElementTree import ElementTree
from xml.etree.ElementTree import Element, SubElement, Comment, tostring
from xml.dom import minidom

DiskSectors         =      0
SingleImageSize     =      0

BLOCK_SIZE          =      0x200
TABLE_ENTRY_0       =      0x1BE
TABLE_ENTRY_1       =      0x1CE
TABLE_ENTRY_2       =      0x1DE
TABLE_ENTRY_3       =      0x1EE

EMMCBLD_MIN_DISK_SIZE   = (4*1024*1024) # 4MB

MAX_FILE_SIZE_BEFORE_SPLIT = (10*1024*1024)
#MAX_FILE_SIZE_BEFORE_SPLIT = (2048) # testing purposes

ExtendedPartitionBegins = 0
bytes_read              = 0
FileNotFoundShowWarning = 0

print "\nmsp.py is running from CWD: ", os.getcwd(), "\n"


SECTOR_SIZE = 512
disk_size = None

def ReadSectors(opfile,NumSectors):
    try:
        return opfile.read(NumSectors*512)
    except:
        print "ERROR: Could not complete the read"
        sys.exit()

def EnsureDirectoryExists(filename):
    dir = os.path.dirname(filename)

    try:
        os.stat(dir)
    except:
        os.makedirs(dir)

def PrintBigError(sz):
    print "\t _________________ ___________ "
    print "\t|  ___| ___ \\ ___ \\  _  | ___ \\"
    print "\t| |__ | |_/ / |_/ / | | | |_/ /"
    print "\t|  __||    /|    /| | | |    / "
    print "\t| |___| |\\ \\| |\\ \\\\ \\_/ / |\\ \\ "
    print "\t\\____/\\_| \\_\\_| \\_|\\___/\\_| \\_|\n"

    if len(sz)>0:
        print sz
        sys.exit(1)

def PrettyPrintArray(bytes_read):
    Bytes = struct.unpack("%dB" % len(bytes_read),bytes_read)

    for k in range(len(Bytes)/512):
        print "-"*78
        for j in range(32):
            for i in range(16):
                sys.stdout.write("%.2X " % Bytes[i+j*16])

            sys.stdout.write("\t")

            for i in range(16):
                sys.stdout.write("%c" % Bytes[i+j*16])
            print " "
    print " "

def Zerout(disk_name,start_sector,num_sectors):
    global verbose


    verbose = True

    if verbose:
        print "\nIn Zeroout"
        print "Attempting to open %s" % disk_name
    try:
        opfile = open(disk_name, "r+b")
    except:
        print "ERROR: (Zerout) Could not open %s, cwd=%s" % (disk_name,os.getcwd() )

        if sys.platform.startswith("linux"):
            print "Did you remember to use SUDO?\n"
        else:
            print "\t  ___      _           _       _     _             _                ___  "
            print "\t / _ \\    | |         (_)     (_)   | |           | |              |__ \\ "
            print "\t/ /_\\ \\ __| |_ __ ___  _ _ __  _ ___| |_ _ __ __ _| |_  ___  _ __     ) |"
            print "\t|  _  |/ _` | '_ ` _ \\| | '_ \\| / __| __| '__/ _` | __|/ _ \\| '__|   / / "
            print "\t| | | | (_| | | | | | | | | | | \\__ \\ |_| | | (_| | |_| (_) | |     |_|  "
            print "\t\\_| |_/\\__,_|_| |_| |_|_|_| |_|_|___/\\__|_|  \\__,_|\\__|\\___/|_|     (_)  \n\n"

            print "Did you remember to be in Administrator Mode for Win7?\n"
        sys.exit()


    if start_sector<0:
        # Means do this NUM_DISK_SECTORS - start_sector
        start_sector = (AvailablePartitions[disk_name]/512) + start_sector # remember start_sector is negative, so add it

    if verbose:
        print "Moving to sector %i" % start_sector
    try:
        opfile.seek(int(start_sector*512))
    except:
        print "ERROR: Could not move to sector %d in %s" % (start_sector,disk_name)
        sys.exit()

    temp = [0]*(512*num_sectors)
    bytes_read = struct.pack("%iB"%(512*num_sectors),*temp)
    if verbose:
        print "Writing %i zeros" % len(temp)
    try:
        opfile.write(bytes_read)
    except:
        print "ERROR: Could not write at sector %d in %s" % (Zeroout['start_sector'],disk_name)
        sys.exit()

    try:
        opfile.close()
    except:
        print "\tWARNING: Could not close %s" % disk_name
        pass


def external_call(command, capture_output=True):
    errors = None
    output = None
    try:
        if capture_output:
            p = sub.Popen(command, stdout=sub.PIPE, stderr=sub.PIPE)
            output, errors = p.communicate()
        else:
            os.system(command)
    except Exception, e:
        print "Error executing command %s. %s" % (str(command), e)
        #clean_up()
        sys.exit(1)
    finally:
        #if not output is None:
        #    print "Result: %s" % output
        if (not errors is None) and (not errors == ""):
            print "Process stderr: %s" % errors
    return output

def ParseXML(xml_filename):
    global PartitionArray, PatchArray, ZeroOutArray,MinDiskSize

    root = ET.parse( xml_filename )
    #Create an iterator
    iter = root.getiterator()
    for element in iter:
        #print "\nElement:" , element.tag
        # Parse out include files
        if element.tag=="program":
            #print "\nFound a partition to program"

            if element.keys():
                ##print "\tAttributes:"
                Partition = {}
                for name, value in element.items():
                    #print "\t\tName: '%s'=>'%s' " % (name,value)
                    Partition[name]=value

            ##print "Partition['start_sector']=",Partition['start_sector']

            m = re.search("NUM_DISK_SECTORS-(\d+)", Partition['start_sector'])
            if type(m) is not NoneType:
                if DiskSize > 0 :
                    Partition['start_sector']           = int((DiskSize/512)-int(m.group(1)))
                else:
                    Partition['start_sector']           = int((EMMCBLD_MIN_DISK_SIZE/512)-int(m.group(1)))
            else:
                Partition['start_sector']           = int(Partition['start_sector'])

            if 'num_sectors' in Partition:
                Partition['num_sectors']            = int(Partition['num_sectors'])
            else:
                Partition['num_sectors']            = int(Partition['num_partition_sectors'])

            if 'offset' in Partition:
                Partition['offset']                 = int(Partition['offset'])
            else:
                Partition['offset']                 = int(Partition['file_sector_offset'])

            Partition['physical_partition_number']  = int(Partition['physical_partition_number'])

            PartitionArray.append( Partition )

        elif element.tag=="patch":
            #print "\nFound a PATCH to program"

            if element.keys():
                #print "\n\tAttributes:"
                Patch = {}
                for name, value in element.items():
                    #print "\t\tName: '%s'=>'%s' " % (name,value)
                    Patch[name]=value

                Patch['function']           = "none"
                Patch['arg0']               = 0
                Patch['arg1']               = 0


                m = re.search("NUM_DISK_SECTORS-(\d+)", Patch['start_sector'])
                if type(m) is not NoneType:
                    Patch['start_sector'] = Patch['start_sector'].replace("NUM_DISK_SECTORS-%s"%m.group(1),str(int((DiskSize/512)-int(m.group(1)))))

                m = re.search("NUM_DISK_SECTORS-(\d+)", Patch['value'])
                if type(m) is not NoneType:
                    # if we are patching, then at this point NUM_DISK_SECTORS could have been zero
                    # In that case, set to EMMCBLD_MIN_DISK_SIZE
                    if DiskSize>0:
                        Patch['value'] = Patch['value'].replace("NUM_DISK_SECTORS-%s"%m.group(1),str(int((DiskSize/512)-int(m.group(1)))))
                    else:
                        Patch['value'] = Patch['value'].replace("NUM_DISK_SECTORS-%s"%m.group(1),str(int(EMMCBLD_MIN_DISK_SIZE/512)-int(m.group(1))))

                Patch['start_sector']   = int(float(Patch['start_sector']))

                m = re.search("CRC32\((\d+).?,(\d+).?\)", Patch['value'])
                if type(m) is not NoneType:
                    Patch['value']          = 0
                    Patch['function']       = "CRC32"
                    Patch['arg0']           = int(float(m.group(1)))   # start_sector
                    Patch['arg1']           = int(float(m.group(2)))   # len_in_bytes

                ##print Patch['value']

                Patch['value']              = int(float(Patch['value']))
                Patch['byte_offset']        = int(float(Patch['byte_offset']))
                Patch['size_in_bytes']      = int(float(Patch['size_in_bytes']))
                Patch['byte_offset']        = int(float(Patch['byte_offset']))

                Patch['physical_partition_number']  = int(float(Patch['physical_partition_number']))

                if Patching=="DISK":
                    if Patch['filename']=="DISK":
                        PatchArray.append( Patch )
                if Patching=="FILES":
                    if Patch['filename']!="DISK":
                        PatchArray.append( Patch )

        elif element.tag=="zeroout":
            #print "\nFound a zeroout to program"

            if element.keys():
                #print "\n\tAttributes:"
                Zeroout = {}
                for name, value in element.items():
                    #print "\t\tName: '%s'=>'%s' " % (name,value)
                    Zeroout[name]=value

                    # <zeroout label="Wipe_GPT_Backup" num_partition_sectors="1" physical_partition_number="0" start_sector="NUM_DISK_SECTORS-1."/>

                m = re.search("NUM_DISK_SECTORS-(\d+)", Zeroout['start_sector'])
                if type(m) is not NoneType:
                    Zeroout['start_sector'] = Zeroout['start_sector'].replace("NUM_DISK_SECTORS-%s"%m.group(1),str(int((DiskSize/512)-int(m.group(1)))))

                Zeroout['start_sector']   = int(float(Zeroout['start_sector']))
                Zeroout['physical_partition_number']  = int(float(Zeroout['physical_partition_number']))

                ZeroOutArray.append( Zeroout )
                #print Zeroout
                #print "\n"


def ReturnArrayFromCommaSeparatedList(sz):
    temp = re.sub("\s+|\n"," ",sz)
    temp = re.sub("^\s+","",temp)
    temp = re.sub("\s+$","",temp)
    return temp.split(',')

def find_file(filename, search_paths):
    print "\n\n\tLooking for",filename
    print "\t"+"-"*40
    for x in search_paths:
        print "\tSearching ",x
        temp = os.path.join(x, filename)
        if os.path.exists(temp):
            print "\n\t**Found %s (%i bytes)" % (temp,os.path.getsize(temp))
            return temp

    ## search cwd last
    print "\tSearching ",os.getcwd()
    if os.path.exists(filename):
        print "\n\t**Found %s (%i bytes)" % (filename,os.path.getsize(filename))
        return filename

    print "\tCound't find file\n"
    return None


def DoubleCheckDiskSize():

    if os.path.basename(Filename)=="singleimage.bin":
        return

    if sys.platform.startswith("win"):
        print "\n\nTesting of OS detected disk size correctly...\n"

        Size = AvailablePartitions[Filename]

        TrueSize = Size

        count = 0
        # Windows workaround to get the correct number of sectors
        fp = open(Filename, 'rb')
        fp.seek(int(Size))
        try:
            while True:
                fp.read(SECTOR_SIZE)

                if count % 128 == 0:
                    sys.stdout.write(".")

                count += 1

        except:
            TrueSize = fp.tell()
        fp.close()

        if TrueSize != Size and Size<=(64*1024*1024*1024):
            print "\n\t                          _             "
            print "\t                         (_)            "
            print "\t__      ____ _ _ __ _ __  _ _ __   __ _ "
            print "\t\\ \\ /\\ / / _` | '__| '_ \\| | '_ \\ / _` |"
            print "\t \\ V  V / (_| | |  | | | | | | | | (_| |"
            print "\t  \\_/\\_/ \\__,_|_|  |_| |_|_|_| |_|\\__, |"
            print "\t                                   __/ |"
            print "\t                                  |___/ \n\n"
            print "NOTE: This OS has *not* detected the correct size of the disk"
            print "\nSECTORS: Size=%i, TrueSize=%i, Difference=%i sectors (%s)" % (Size/512,TrueSize/512,(TrueSize-Size)/512,ReturnSizeString(TrueSize-Size))
            print "This means the backup GPT header will *not* be located at the true last sector"
            print "This is only an issue if you care :) It will be off by %s" % ReturnSizeString(TrueSize-Size)
            print "\nNOTE: This program *can't* write to the end of the disk, OS limitation"
        else:
            print "\n\nAll is well\n"



def PerformWrite():
    global PartitionSorted, LoadSubsetOfFiles, search_paths, interactive, SingleImageSize, Filename

    print "\t                                                     _            "
    print "\t                                                    (_)            "
    print "\t _ __  _ __ ___   __ _ _ __ __ _ _ __ ___  _ __ ___  _ _ __   __ _ "
    print "\t| '_ \\| '__/ _ \\ / _` | '__/ _` | '_ ` _ \\| '_ ` _ \\| | '_ \\ / _` |"
    print "\t| |_) | | | (_) | (_| | | | (_| | | | | | | | | | | | | | | | (_| |"
    print "\t| .__/|_|  \\___/ \\__, |_|  \\__,_|_| |_| |_|_| |_| |_|_|_| |_|\\__, |"
    print "\t| |               __/ |                                       __/ |"
    print "\t|_|              |___/                                       |___/ "

    for Partition in PartitionSorted:   # Here Partition is a *sorted* entry from rawprogram.xml
        if Partition['physical_partition_number']!=0:   ## msp tool can only write to PHY partition 0
            print "Skipping '%s' for physical_partition_number=%d (only 0 is accessible)" % (Partition['filename'],Partition['physical_partition_number'])
        else:

            if len(Partition['filename'])==0:
                continue

            if LoadSubsetOfFiles is True:
                # To be here means user only wants some of the files loaded from rawprogram0.xml
                if Partition['filename'] in file_list:
                    #print "LOAD: '%s' was specified to be programmed" % Partition['filename']
                    pass
                else:
                    #print "SKIPPING: '%s', it was not specified to be programmed" % Partition['filename']
                    continue


            print "\n"+"="*78
            print "="*78
            FileWithPath = find_file(Partition['filename'], search_paths)

            size=0
            if FileWithPath is not None:
                size = os.path.getsize(FileWithPath)


            # to be here the rawprogram.xml file had to have a "filename" entry
            print "\n'%s' (%s) to partition '%s' at sector %d (at %s)\n" % (Partition['filename'],ReturnSizeString(size),Partition['label'],Partition['start_sector'],ReturnSizeString(Partition['start_sector']*512))

            if interactive is True:
                loadfile = raw_input("Do you want to load this file? (Y|n|q)")
                if loadfile=='Y' or loadfile=='y' or loadfile=='':
                    pass
                elif loadfile=='q' or loadfile=='Q':
                    sys.exit()
                else:
                    continue

            while FileWithPath is None:
                FileNotFoundShowWarning = 1

                print "\t______      _   _        ___  "
                print "\t| ___ \\    | | | |      |__ \\ "
                print "\t| |_/ /__ _| |_| |__       ) |"
                print "\t|  __// _` | __| '_ \\     / / "
                print "\t| |  | (_| | |_| | | |   |_|  "
                print "\t\\_|   \\__,_|\\__|_| |_|   (_)  \n\n"

                print "ERROR: '%s' listed in '%s' not found\n" % (Partition['filename'],rawprogram_filename)

                print "Please provide a path for this file"
                print "Ex. \\\\somepath\\folder OR c:\\somepath\\folder\n"
                temppath = raw_input("Enter PATH or Q to quit? ")
                if temppath=='Q' or temppath=='q' or temppath=='':
                    sys.exit()

                print "\n"

                FileWithPath = find_file(Partition['filename'], [temppath])

                size=0
                if FileWithPath is not None:
                    size = os.path.getsize(FileWithPath)

                    temp = raw_input("\nShould this path be used to find other files? (Y|n|q)")
                    if temp=='Q' or temp=='q':
                        sys.exit()
                    elif temp=='Y' or temp=='y' or temp=='':
                        search_paths.append(temppath)
                    print "\n"

            if size==0:
                loadfile = raw_input("WARNING: This file is 0 bytes, do you want to load this file? (y|N|q)")
                if loadfile=='N' or loadfile=='n' or loadfile=='':
                    continue
                elif loadfile=='q' or loadfile=='Q':
                    sys.exit()
                else:
                    pass


            if Partition['num_sectors']==0:
                Partition['num_sectors'] = int(size/512)
                if size%512 != 0:
                    Partition['num_sectors']+=1 # not an even multiple of 512, so ++

            ##print "At start_sector %i (%.2fKB) write %i sectors" % (Partition['start_sector'],Partition['start_sector']/2.0,Partition['num_sectors'])
            ##print "\tsize of \"%s\" is %i bytes" % (Partition['filename'],size)
            ##print "\tsize of partition listed in in \"%s\" is %i bytes" % (rawprogram_filename,Partition['num_sectors']*512)

            ## This below happens on files like partition0.bin, where they hold the entire partition table,
            ## but, only MBR is meant to be written, thus partition0.bin is 9 sectors but MBR is only 1 sector

            if size > (Partition['num_sectors']*512):
                size = Partition['num_sectors']*512

            ##print "\tAttempting to read %i bytes from \n\t\"%s\" at offset %i" % (size,Partition['filename'],Partition['offset'])
            #os.getcwd()+"\\"+

            try:
                ipfile = open(FileWithPath, "rb")
            except:
                print "ERROR: Could not open FileWithPath=%s, cwd=%s" % (Partition['filename'], os.getcwd() )
                sys.exit()


            print "\tAttempting to move to sector %i (file offset) in %s" % (Partition['offset'],Partition['filename'])
            try:
                ipfile.seek(int(Partition['offset']*512))
            except:
                print "ERROR: Could not move to sector %d in %s" % (Partition['offset'],Partition['filename'])
                sys.exit()

            print "\tAttempting to read %i bytes" % (size)
            try:
                if size<MAX_FILE_SIZE_BEFORE_SPLIT:
                    bytes_read = ipfile.read(size)
                else:
                    print "File is too large to read all at once, must be broken up"
            except:
                print "ERROR: Could not read %d bytes in %s" % (size,Partition['filename'])
                sys.exit()

            if size<MAX_FILE_SIZE_BEFORE_SPLIT:
                ipfile.close()

            if size<MAX_FILE_SIZE_BEFORE_SPLIT:
                print "\tSuccessfully read %d bytes of %d bytes and closed %s" % (len(bytes_read),size,Partition['filename'])

                Remainder = len(bytes_read)%512

                if Remainder != 0:
                    print "\tbytes_read is not a multiple of 512, appending zeros"

                    Bytes = struct.unpack("%dB" % len(bytes_read),bytes_read)    ## unpack returns list, so get index 0

                    Temp         = list(Bytes) + [0x00]*(512-Remainder)   # concat
                    TotalSize = len(Temp)

                    bytes_read = struct.pack("%dB" % TotalSize, *Temp)
                    print "\tNow len(bytes_read)=%d" % len(bytes_read)

                # At this point bytes_read is a multiple of 512

            #print "\t\tNeed to move to location %i (%i bytes) in %s" % (Partition['start_sector'],(Partition['start_sector']*512),Filename)
            #print "\t\tPartition['start_sector']*512 = %i" % (Partition['start_sector']*512)

            if (2*DiskSize)<Partition['start_sector'] and os.path.basename(Filename)!="singleimage.bin":
                print "2*DiskSize=%d" % (2*DiskSize)
                print "Partition['start_sector']=",Partition['start_sector']
                print "\n\nERROR: Attempting to move to sector %i (%.2f MB) and only %i sectors (%.2f MB) exist (%i difference (%.2f MB) )" % (Partition['start_sector'],Partition['start_sector']/2048.0,2*DiskSize,DiskSize/1024.0,Partition['start_sector']-(2*DiskSize),(Partition['start_sector']-(2*DiskSize))/2048.0)
                sys.exit()

            try:
                if os.path.basename(Filename)=="singleimage.bin":
                    Filename = OutputFolder+os.path.basename(Filename)

                opfile = open(Filename, "r+b")  ## Filename = '\\.\PHYSICALDRIVE1'
                #print "Opened '%s', cwd=%s" % (Filename, os.getcwd() )
            except:
                print "ERROR: Could not open Filename=%s, cwd=%s" % (Filename, os.getcwd() )

                if sys.platform.startswith("linux"):
                    print "\tDon't forget you need SUDO with this program"
                    print "\tsudo python msp.py partition.xml /dev/sdx (where x is the device node)"
                else:
                    print "\t  ___      _           _       _     _             _                ___  "
                    print "\t / _ \\    | |         (_)     (_)   | |           | |              |__ \\ "
                    print "\t/ /_\\ \\ __| |_ __ ___  _ _ __  _ ___| |_ _ __ __ _| |_  ___  _ __     ) |"
                    print "\t|  _  |/ _` | '_ ` _ \\| | '_ \\| / __| __| '__/ _` | __|/ _ \\| '__|   / / "
                    print "\t| | | | (_| | | | | | | | | | | \\__ \\ |_| | | (_| | |_| (_) | |     |_|  "
                    print "\t\\_| |_/\\__,_|_| |_| |_|_|_| |_|_|___/\\__|_|  \\__,_|\\__|\\___/|_|     (_)  \n\n"

                    print "\n"+"-"*78
                    print "\tThis program needs to be run as Administrator!!"
                    print "-"*78+"\n"
                    print "-"*78
                    print "\tTo fix, you must open a CMD prompt with \"Run as administrator\""
                    print "-"*78+"\n"

                sys.exit()

            print "\n\tOpened %s" % Filename


            if Partition['start_sector'] < 0:
                print "start sector is less than 0 - skipping this instruction, most likely for GPT HACK"
                continue

            if Partition['start_sector'] > int(DiskSize/512):
                print "\nERROR: Start sector is BIGGER than the disk"
                print "\nERROR: Your device is TOO SMALL to handle this partition info"
                sys.exit(1)

            try:
                opfile.seek(int(Partition['start_sector']*512))
            except:
                print "ERROR: Could not move to sector %d on %s" % (Partition['start_sector'],Filename)
                sys.exit()

            print "\tMoved to sector %d on %s" % (Partition['start_sector'],Filename)

            ##print "size=",size
            ##print "MAX_FILE_SIZE_BEFORE_SPLIT=",MAX_FILE_SIZE_BEFORE_SPLIT

            if size<MAX_FILE_SIZE_BEFORE_SPLIT:
                print "\tFile can be written completely."
                try:
                    opfile.write(bytes_read)
                except:
                    print "ERROR: Could not write to %s" % (Filename)
                    print "\nPlease try removing the medium and re-inserting it"
                    print "Strangly this helps sometimes after writing *new* partition tables\n\n"
                    sys.exit()

            else:
                ## To be here means I need to break up the file
                ##print "I need to break up this file"

                TempSize = size
                NumLoop = int(TempSize/MAX_FILE_SIZE_BEFORE_SPLIT)
                Remainder = size%MAX_FILE_SIZE_BEFORE_SPLIT
                if Remainder>0:
                    Remainder=1

                print "\nNeed to break up this file, will loop %d times writing %i bytes each time" % (NumLoop+Remainder,MAX_FILE_SIZE_BEFORE_SPLIT)
                TempSize=0
                for a in range(NumLoop):
                    #print "read %i bytes" % MAX_FILE_SIZE_BEFORE_SPLIT
                    try:
                        bytes_read = ipfile.read(MAX_FILE_SIZE_BEFORE_SPLIT)
                    except:
                        print "ERROR: Could not read from %s" % (FileWithPath)
                        sys.exit()

                    ##print "\n\t%i) Packing %i Bytes [%i:%i]" % (a+1,MAX_FILE_SIZE_BEFORE_SPLIT,TempSize,TempSize+MAX_FILE_SIZE_BEFORE_SPLIT)
                    ##bytes_read = struct.pack("%dB" % MAX_FILE_SIZE_BEFORE_SPLIT, *Bytes[TempSize:(TempSize+MAX_FILE_SIZE_BEFORE_SPLIT)])
                    print "\t%.2i) Writing %i Bytes [%i:%i]" % (a+1,MAX_FILE_SIZE_BEFORE_SPLIT,TempSize,TempSize+MAX_FILE_SIZE_BEFORE_SPLIT)
                    try:
                        opfile.write(bytes_read)
                    except:
                        print "ERROR: Could not write to %s" % (Filename)
                        sys.exit()

                    TempSize += MAX_FILE_SIZE_BEFORE_SPLIT
                ##print "Out of loop"
                a+=1

                if Remainder == 1:
                    # Need to PAD the file to be a multiple of 512 bytes too

                    #print "\n\t%i) Packing %i Bytes [%i:%i]" % (a,(len(Bytes)-TempSize),TempSize,len(Bytes))
                    #bytes_read = struct.pack("%dB" % (len(Bytes)-TempSize), *Bytes[TempSize:len(Bytes)])

                    print "\t%.2i) Writing %i Bytes [%i:%i]" % (a+1,(size-TempSize),TempSize,size)
                    try:
                        bytes_read = ipfile.read(size-TempSize)
                    except:
                        print "ERROR: Could not read from %s" % (FileWithPath)
                        sys.exit()

                    ##print "len(bytes_read)=",len(bytes_read)

                    Remainder = len(bytes_read)%512

                    if Remainder != 0:
                        print "\tbytes_read is not a multiple of 512, appending zeros"

                        Bytes = struct.unpack("%dB" % len(bytes_read),bytes_read)    ## unpack returns list, so get index 0

                        Temp         = list(Bytes) + [0x00]*(512-Remainder)   # concat
                        TotalSize = len(Temp)

                        bytes_read = struct.pack("%dB" % TotalSize, *Temp)
                        print "\tNow len(bytes_read)=%d" % len(bytes_read)

                    # At this point bytes_read is a multiple of 512

                    #print "This is the final write"
                    try:
                        opfile.write(bytes_read)
                    except:
                        print "ERROR: Could not write to %s" % (Filename)
                        sys.exit()

                ipfile.close()

            if os.path.basename(Filename)=="singleimage.bin":
                print "\tSingleImageSize before = ", SingleImageSize
                SingleImageSize+=int(len(bytes_read)/512)
                print "\tSingleImageSize after = ", SingleImageSize

            #print "\tWrote %d bytes at sector %d on %s" % (len(bytes_read),Partition['start_sector'],Filename)

            try:
                ##print opfile
                opfile.close()
            except:
                print "\tWARNING: Can't close the file?"
                #sys.exit()
                pass


            print "\n\tWritten with"
            print "\tpython dd.py --if=%s --bs=512 --count=%i --seek=%i --of=%s" % (FileWithPath,int((size-1)/SECTOR_SIZE)+1,Partition['start_sector'],Filename)
            print "\n\tVerify with"
            print "\tpython dd.py --if=%s --bs=512 --count=%i --skip=%i --of=dump.bin" % (Filename,int((size-1)/SECTOR_SIZE)+1,Partition['start_sector'])

            print "\n\tSuccessfully wrote \"%s\" (%s payload) to %s" % (Partition['filename'],ReturnSizeString(size),Filename)

            #raw_input("Enter something: ")

            #if Partition['filename']=="sbl3.mbn":
            #    sys.exit()
    print "\nDone Writing Files\n"

def GetPartitions():
    global Devices,AvailablePartitions
    if sys.platform.startswith("linux"):
        print "This is a linux system since sys.platform='%s'" % sys.platform

        print "-"*78
        print "\tRemember - DON'T FORGET SUDO"
        print "\tRemember - DON'T FORGET SUDO"
        print "\tRemember - DON'T FORGET SUDO"
        print "-"*78+"\n"

        os.system("cat /proc/partitions > temp_partitions.txt")
        IN = open("temp_partitions.txt")
        output = IN.readlines()
        for line in output:
            #print line

            m = re.search("(\d+) (sd[a-z])$", line)
            if type(m) is not NoneType:
                Size    = int(m.group(1))
                Device  = "/dev/"+m.group(2)
                #print "%s\tSize=%d,%.1fMB (%.2fGB) (%iKB)" % (Device,Size,int(Size)/1024.0,int(Size)/(1024.0*1024.0),int(Size))
                AvailablePartitions[Device] = Size*1024.0    # linux reports in terms of 1024,


    else:
        ##print "This is a windows system since sys.platform='%s'" % sys.platform

        print "\t  ___      _           _       _     _             _                ___  "
        print "\t / _ \\    | |         (_)     (_)   | |           | |              |__ \\ "
        print "\t/ /_\\ \\ __| |_ __ ___  _ _ __  _ ___| |_ _ __ __ _| |_  ___  _ __     ) |"
        print "\t|  _  |/ _` | '_ ` _ \\| | '_ \\| / __| __| '__/ _` | __|/ _ \\| '__|   / / "
        print "\t| | | | (_| | | | | | | | | | | \\__ \\ |_| | | (_| | |_| (_) | |     |_|  "
        print "\t\\_| |_/\\__,_|_| |_| |_|_|_| |_|_|___/\\__|_|  \\__,_|\\__|\\___/|_|     (_)  \n\n"

        print "-"*78
        print "\tRemember - Under Win7 you must run this as Administrator"
        print "\tRemember - Under Win7 you must run this as Administrator"
        print "\tRemember - Under Win7 you must run this as Administrator"
        print "-"*78

        response = external_call('wmic DISKDRIVE get DeviceID, MediaType, Model, Size')
        m = re.search("Access is denied", response)
        if type(m) is not NoneType:
            print "\nERROR: This computer does not have correct privileges, you need administrator group privilege\n"
            sys.exit(1)

        response = response.replace('\r', '').strip("\n").split("\n")[1:]
        for line in response:
            m = re.search("(PHYSICALDRIVE\d+).+ (\d+) ", line)
            if type(m) is not NoneType:
                Size    = int(m.group(2))       # size in sectors
                Device  = "\\\\.\\"+m.group(1)  # \\.\PHYSICALDRIVE1

                AvailablePartitions[Device] = Size


    Devices = AvailablePartitions.keys()
    Devices.sort()

    print "\n\n"+"-"*78
    print "--------------------------------Partitions Detected--------------------------------------"
    print "-"*78+"\n"
    for device in Devices:
        value = AvailablePartitions[device]

        if value/(1024.0*1024.0*1024.0) > 31.0:
            print "%s   %s\tsectors:%i <--- Not likely an SD card, careful!" % (device,ReturnSizeString(value),value/512)
        else:
            print "%s   %s\tsectors:%i" % (device,ReturnSizeString(value),value/512)

    print "\n"+"-"*78+"\n"

def PerformPatching():
    global PatchArray,Patching

    print "\t             _        _     _             "
    print "\t            | |      | |   (_)            "
    print "\t _ __   __ _| |_  ___| |__  _ _ __   __ _ "
    print "\t| '_ \\ / _` | __|/ __| '_ \\| | '_ \\ / _` |"
    print "\t| |_) | (_| | |_| (__| | | | | | | | (_| |"
    print "\t| .__/ \\__,_|\\__|\\___|_| |_|_|_| |_|\\__, |"
    print "\t| |                                  __/ |"
    print "\t|_|                                 |___/ "

    var = 'Y'
    if Patching == "DISK":
        var = 'N'           ## user must authorize this

    ## PATCHING HAPPENS HERE - PATCHING HAPPENS HERE - PATCHING HAPPENS HERE

    for Patch in PatchArray:

        if Patch['physical_partition_number']!=0:   ## msp tool can only write to PHY partition 0
            print "Skipping '%s' for physical_partition_number=%d (only 0 is accessible)" % (Patch['filename'],Patch['physical_partition_number'])
        else:

            print "\n" + "-"*78
            print "PATCH: (%s) %s" % (Patch['filename'],Patch['what'])

            FileToOpen =  Patch['filename']

            if nopromt is True:
                # means don't bug them, i.e. automation
                pass
            else:
                if var=='N' or var=='n':
                    var = raw_input("\nWARNING: Are you sure you want to PATCH to '%s' of size %s (y|N) " % (Filename,ReturnSizeString(DiskSize)))
                    if var=='Y' or var=='y':
                        pass
                    else:
                        sys.exit(1)

            if Patching=="DISK":
                FileWithPath = Filename
            else:
                FileWithPath = find_file(FileToOpen, search_paths)

            while FileWithPath is None:
                FileNotFoundShowWarning = 1

                print "\t______      _   _        ___  "
                print "\t| ___ \\    | | | |      |__ \\ "
                print "\t| |_/ /__ _| |_| |__       ) |"
                print "\t|  __// _` | __| '_ \\     / / "
                print "\t| |  | (_| | |_| | | |   |_|  "
                print "\t\\_|   \\__,_|\\__|_| |_|   (_)  \n\n"

                if rawprogram_filename is None:
                    print "ERROR: '%s' listed in '%s' not found\n" % (FileToOpen,patch_filename)
                else:
                    print "ERROR: '%s' listed in '%s' not found\n" % (FileToOpen,rawprogram_filename)

                print "Please provide a path for this file"
                print "Ex. \\\\somepath\\folder OR c:\\somepath\\folder\n"
                temppath = raw_input("Enter PATH or Q to quit? ")
                if temppath=='Q' or temppath=='q' or temppath=='':
                    sys.exit()

                print "\n"

                FileWithPath = find_file(Partition['filename'], [temppath])

                size=0
                if FileWithPath is not None:
                    size = os.path.getsize(FileWithPath)

                    temp = raw_input("\nShould this path be used to find other files? (Y|n|q)")
                    if temp=='Q' or temp=='q':
                        sys.exit()
                    elif temp=='Y' or temp=='y' or temp=='':
                        search_paths.append(temppath)
                    print "\n"

            try:
                opfile = open(FileWithPath, "r+b")
                print "Opened %s, cwd=%s" % (FileWithPath,os.getcwd() )
            except:
                print "ERROR: Could not open %s, cwd=%s" % (FileWithPath,os.getcwd() )
                sys.exit()

            if Patch['function']=="CRC32":
                print "\t%s(%u,%u) requested " % (Patch['function'],Patch['arg0'],Patch['arg1'])

                PStartSector = int(Patch['arg0'])
                PNumSectors  = int(Patch['arg1']/512)
                if PNumSectors==0:
                    PNumSectors+=1

                try:
                    if Patch['arg0']>64:
                        print "\tPatch['arg0']=%d" % Patch['arg0']
                        print "\tPatch['arg1']=%d" % Patch['arg1']

                        print "\tmoving to sector %d-%d (%d)" % (PStartSector,(64-PNumSectors),PStartSector-(64-PNumSectors))
                        opfile.seek( int( (PStartSector-(64-PNumSectors) )*512 ))
                    else:
                        print "moving to sector %d (byte location %d)" % (Patch['arg0'],Patch['arg0']*512)
                        opfile.seek( int(Patch['arg0']*512))

                except:
                    print "ERROR: Could not complete move in %s" % (FileWithPath)
                    sys.exit()

                print "\tMove Successful "

                try:
                    if Patch['arg0']>64:
                        print "\tTrying to read %d bytes in %s" % (64*512,FileWithPath)
                        bytes_read = opfile.read(64*512)
                    else:
                        print "\tTrying to read %d bytes in %s" % (Patch['arg1'],FileWithPath)
                        bytes_read = opfile.read(Patch['arg1'])
                except:
                    print "ERROR: Could not read in %s" % (FileWithPath)
                    sys.exit()

                print "\tlen(bytes_read)=",len(bytes_read)

                if Patch['arg0']>64:
                    Bytes = struct.unpack("%dB" % (64*512),bytes_read)    ## unpack returns list, so get index 0
                else:
                    Bytes = struct.unpack("%dB" % Patch['arg1'],bytes_read)    ## unpack returns list, so get index 0



                if Patch['arg0']>64:
                    #PStartSector
                    #PNumSectors
                    StartByte = (64-PNumSectors)*512
                    print "\tStartByte=",StartByte

                    Patch['value'] = CalcCRC32(Bytes[StartByte:],Patch['arg1'])
                else:
                    Patch['value'] = CalcCRC32(Bytes,Patch['arg1'])

                print "\tCRC32=0x%.8X" % Patch['value']

            # ELSE - patch is not a CRC
            try:
                if Patch['start_sector']>64:
                    #print "moving to %d-63=%d" % (Patch['start_sector'],Patch['start_sector']-63)
                    opfile.seek( int((Patch['start_sector']-63)*512))
                else:
                    #print "moving to sector %d (byte location %d)" % (Patch['start_sector'],Patch['start_sector']*512)
                    opfile.seek(int(Patch['start_sector']*512))
            except:
                print "ERROR: Could not move to sector %d in %s" % (Patch['start_sector'],FileWithPath)
                sys.exit()

            try:
                if Patch['start_sector']>64:
                    bytes_read = opfile.read(64*512)
                    if len(bytes_read) != (64*512):
                        PrintBigError("Didn't get the read size 64*512");
                else:
                    bytes_read = opfile.read(512)
                    if len(bytes_read) != (512):
                        PrintBigError("Didn't get the read size 512");
            except:
                print "ERROR: Could not read sector %d in %s" % (Patch['start_sector'],FileWithPath)
                sys.exit()


            print "success was able to read len(bytes_read)=%d" % (len(bytes_read))

            # Move back to do the write
            try:
                if Patch['start_sector']>64:
                    #print "moving to %d-63=%d" % (Patch['start_sector'],Patch['start_sector']-63)
                    opfile.seek(int((Patch['start_sector']-63)*512))
                else:
                    #print "moving to sector %d (byte location %d)" % (Patch['start_sector'],Patch['start_sector']*512)
                    opfile.seek(int(Patch['start_sector']*512))
            except:
                print "ERROR: Could not move to sector %d in %s" % (Patch['start_sector'],FileWithPath)
                sys.exit()


            if Patch['value'] < 0:
                print "ERROR: Patch value was negative. This means your DISK size is too small"
                sys.exit()

            print "At sector %d (0x%X) offset %d (0x%X) in %s with %d (0x%X)" % (Patch['start_sector'],Patch['start_sector']*512,Patch['byte_offset'],Patch['byte_offset'],FileWithPath,Patch['value'],Patch['value'])

            #Patch['byte_offset']
            if Patch['start_sector']>64:
                ValueList = list(struct.unpack("32768B",bytes_read))
            else:
                ValueList = list(struct.unpack("512B",bytes_read))

            #print "\nBefore"
            #j=0
            #sys.stdout.write("%.4X\t "%j)
            #for b in ValueList[-512:]:
            #    sys.stdout.write("%.2X "%b)
            #    j+=1
            #    #if j>64:
            #    #    break
            #    if j%16==0:
            #        print " "
            #        sys.stdout.write("%.4X\t "%j)


            print "Patch value:",Patch['value'],"\t"

            for j in range(Patch['size_in_bytes']):
                if Patch['start_sector']>64:
                    #print "Applying patch at %d+%d=%d value=0x%.2X" % (63*512,Patch['byte_offset']+j,63*512+j+Patch['byte_offset'],(Patch['value']>>(j*8))&0xFF)
                    ValueList[63*512+Patch['byte_offset']+j ] = (Patch['value']>>(j*8))&0xFF
                    sys.stdout.write("%.2X " % (int(Patch['value']>>(j*8)) & 0xFF))
                else:
                    ValueList[Patch['byte_offset']+j ] = (Patch['value']>>(j*8))&0xFF
                    #import pdb; pdb.set_trace()
                    sys.stdout.write("%.2X " % (int(Patch['value']>>(j*8)) & 0xFF))


            #for b in ValueList:
            #    sys.stdout.write("%.2X "%b)


            #print "\nAfter"
            #j=0
            #sys.stdout.write("%.4X\t "%j)
            #for b in ValueList[-512:]:
            #    sys.stdout.write("%.2X "%b)
            #    j+=1
            #    #if j>64:
            #    #    break
            #    if j%16==0:
            #        print " "
            #        sys.stdout.write("%.4X\t "%j)

            #print " "

            if Patch['start_sector']>64:
                bytes_read = struct.pack("32768B",*ValueList)
            else:
                bytes_read = struct.pack("512B",*ValueList)

            print "(little endian)"
            print "committing patch"
            try:
                opfile.write(bytes_read)
            except:
                print "ERROR: Could not write %d bytes to %s" % (len(bytes_read),FileWithPath)
                sys.exit()

                #WriteValue(fd, RawProgramInfo.start_sector, RawProgramInfo.byte_offset, RawProgramInfo.value, RawProgramInfo.size_in_bytes);

            try:
                opfile.close()
            except:
                #print "\tWARNING: Could not close %s" % FileWithPath
                pass


            if sys.platform.startswith("linux"):
                os.system("sync")

            print "PATCH:" , Patch['what']
            print "DONE\n"

            #if Patch['what']== "Update Backup Header with Partition Array Location.":
            #    sys.exit(0)

    print "Done patching"

def Usage():
    print "\nUsage: Mass Storage Programmer - destructively writes data to disks!!\n"

    sudo = ""
    drive= ""
    if sys.platform.startswith("linux"):
        sudo = "sudo"
        drive= "/dev/sdb"
    else:
        print "\tYou must run as Administrator under Win7"
        print "\tYou must run as Administrator under Win7"
        print "\tYou must run as Administrator under Win7\n"
        drive="\\\\.\\PHYSICALDRIVE1"

    print "%-40s\t python msp.py" % "Display this info"
    print "%-40s\t%s python msp.py -w -d %s" % ("Wipe partition info (-w)",sudo,drive)
    print "%-40s\t%s python msp.py -e -d %s" % ("Erase entire card (-e)",sudo,drive)
    print "\n%-40s\t%s python msp.py -r rawprogram0.xml -d %s" % ("Program a device (-r)",sudo,drive)
    print "%-40s\t%s python msp.py -r rawprogram0.xml -d 0" % ("Create a singleimage.bin (-r)",sudo)
    print "%-40s\t%s python msp.py -r rawprogram0.xml -d 0 -t c:\\temp" % ("singleimage.bin stored to c:\\temp (-r)",sudo)
    print "%-40s\t%s python msp.py -r rawprogram0.xml -d 8192" % ("Create an 4MB singleimage.bin (-r)",sudo)
    print "%-40s\t%s python msp.py -r rawprogram0.xml -d 16777216" % ("Create an 8GB singleimage.bin (-r)",sudo)
    print "\n%-40s\t%s python msp.py -n -r rawprogram.xml -d %s" % ("no prompts (-n) i.e. automation",sudo,drive)
    print "%-40s\t%s python msp.py -i -r rawprogram.xml -d %s " % ("Interactively choose files (-i)",sudo,drive)
    print "\n%-40s\t%s python msp.py -f sbl1.mbn -f sbl2.mbn -r rawprogram.xml -d %s " % ("Specify files from rawprogram.xml (-f)",sudo,drive)
    print "%-40s\t%s python msp.py -f sbl1.mbn,sbl2.mbn -r rawprogram.xml -d %s " % ("Specify files from rawprogram.xml (-f)",sudo,drive)
    print "\n%-40s\t%s python msp.py -s c:\windows -s d:\ -r rawprogram.xml -d %s " % ("Search this path for files (-s)",sudo,drive)
    print "%-40s\t%s python msp.py -s c:\windows,d:\ -r rawprogram.xml -d %s " % ("Search this path for files (-s)",sudo,drive)
    print "\n%-40s\t%s python msp.py -p patch0.xml -d %s" % ("Patch a device (-p)",sudo,drive)
    print "%-40s\t%s python msp.py -p patch0.xml -d 0" % ("Patch files to default size (-p)",sudo)
    print "%-40s\t%s python msp.py -p patch0.xml -d 16384" % ("Patch files to 8MB disk size (-p)",sudo)
    print "\n%-40s\t%s python msp.py -r rawprogram0.xml -p patch0.xml -d %s" % ("ALL IN ONE STEP",sudo,drive)



def ReturnSizeString(size):
    if size>(1024*1024*1024):
        return "%.2f GB" % (size/(1024.0*1024.0*1024.0))
    elif size>(1024*1024):
        return "%.2f MB" % (size/(1024.0*1024.0))
    elif size>(1024):
        return "%.2f KB" % (size/(1024.0))
    else:
        return "%i B" % (size)


# A8h reflected is 15h, i.e. 10101000 <--> 00010101
def reflect(data,nBits):

    reflection = 0x00000000
    bit = 0

    for bit in range(nBits):
        if(data & 0x01):
            reflection |= (1 << ((nBits - 1) - bit))
        data = (data >> 1);

    return reflection


def CalcCRC32(array,len):
   k        = 8;            # length of unit (i.e. byte)
   MSB      = 0;
   gx       = 0x04C11DB7;   # IEEE 32bit polynomial
   regs     = 0xFFFFFFFF;   # init to all ones
   regsMask = 0xFFFFFFFF;   # ensure only 32 bit answer

   for i in range(len):
      DataByte = array[i]
      DataByte = reflect( DataByte, 8 );

      for j in range(k):
        MSB  = DataByte>>(k-1)  ## get MSB
        MSB &= 1                ## ensure just 1 bit

        regsMSB = (regs>>31) & 1

        regs = regs<<1          ## shift regs for CRC-CCITT

        if regsMSB ^ MSB:       ## MSB is a 1
            regs = regs ^ gx    ## XOR with generator poly

        regs = regs & regsMask; ## Mask off excess upper bits

        DataByte <<= 1          ## get to next bit


   regs          = regs & regsMask ## Mask off excess upper bits
   ReflectedRegs = reflect(regs,32) ^ 0xFFFFFFFF;

   return ReflectedRegs


## ==============================================================================================
## ==============================================================================================
## ==============================================================================================
## =====main()===================================================================================
## ==============================================================================================
## ==============================================================================================
## ==============================================================================================

AvailablePartitions = {}

try:
    opts, args = getopt.getopt(sys.argv[1:], "r:p:d:iewns:f:t:v", ["rawprogram=", "patch=", "dest=","nopromt=", "interactive=", "erase=",  "wipe=", "search_path=","file=","location=","verbose="])
except getopt.GetoptError, err:
    # print help information and exit:
    print str(err) # will print something like "option -a not recognized"
    Usage()
    sys.exit(1)

check_sparse            = None
dd                      = None
xml_filename            = None
rawprogram_filename     = None
patch_filename          = None
disk_name               = None
search_paths            = []
file_list               = []
LoadSubsetOfFiles       = False
interactive             = False
erase_device            = False
wipe_device             = False
nopromt                 = False
verbose                 = False

Operation               = 0
OPERATION_PROGRAM       = 2
OPERATION_PATCH         = 4
OPERATION_ERASE         = 8
OPERATION_WIPE          = 16
Patching                = "FILES"
OutputFolder            = ""

Filename                = "singleimage.bin"
DiskSize                = 0
MinDiskSize             = 0

GetPartitions()
Usage()

for o, a in opts:
    if o in ("-r", "--rawprogram"):
        rawprogram_filename = a
        Operation |= OPERATION_PROGRAM
    elif o in ("-t", "--location"):
        OutputFolder = a
        OutputFolder = re.sub(r"\\$","",OutputFolder)    # remove final slash if it exists
        OutputFolder = re.sub(r"/$","",OutputFolder)     # remove final slash if it exists

        OutputFolder += "/"     # slashes will all be corrected below

        if sys.platform.startswith("linux"):
            OutputFolder = re.sub(r"\\","/",OutputFolder)   # correct slashes
        else:
            OutputFolder = re.sub(r"/","\\\\",OutputFolder) # correct slashes

        print "OutputFolder=",OutputFolder
        EnsureDirectoryExists(OutputFolder) # only need to call once

        Filename = OutputFolder + Filename

    elif o in ("-p", "--patch"):
        patch_filename = a
        Operation |= OPERATION_PATCH
    elif o in ("-d", "--dest"):
        disk_name = a

        # if a number was specified, it's size in sectoras and we're making a singleimage.bin
        # otherwise, a drive /dev/sdb or something was specified

        m = re.search("^(\d+)$", disk_name)
        if type(m) is not NoneType:

            ## they specified a number must be making a single image or patching files
            Patching = "FILES"
            Filename = OutputFolder + "singleimage.bin"
            DiskSize = int(int(m.group(1))*512) # comes in as sectors

        else:

            ## to be here means they didn't specify a number, thus a device
            Filename        = disk_name
            Patching        = "DISK"
            ValidDiskName   = False

            # User can also specify the singleimage.bin name, and I can treat that like it's the disk

            if sys.platform.startswith("linux"):
                m = re.search("/dev/sd[a-z]", disk_name)
                if type(m) is not NoneType:
                    ValidDiskName   = True
            else:
                m = re.search("(PHYSICALDRIVE\d+)", disk_name)
                if type(m) is not NoneType:
                    ValidDiskName   = True
                    Filename = "\\\\.\\"+m.group(1)

            if ValidDiskName:
                if Filename not in Devices:
                    print "ERROR: %s does not exist" % Filename
                    Usage()
                    sys.exit()
                DiskSize = AvailablePartitions[Filename]
            else:
                # To be here means user did this possibly "-d singleimage.bin"
                try:
                    DiskSize = os.path.getsize(disk_name)
                    if DiskSize < EMMCBLD_MIN_DISK_SIZE:
                        DiskSize = EMMCBLD_MIN_DISK_SIZE
                except:
                    print "ERROR: Can't get size of %s" % Filename
                    Usage()
                    sys.exit()

    elif o in ("-i", "--interactive"):
        interactive = True
    elif o in ("-e", "--erase"):
        Operation = OPERATION_ERASE
        erase_device = True
    elif o in ("-n", "--nopromt"):
        nopromt = True
    elif o in ("-v", "--verbose"):
        verbose = True
    elif o in ("-w", "--wipe"):
        Operation = OPERATION_WIPE
        wipe_device = True
    elif o in ("-s", "--search_path"):
        ## also allow seperating commas
        for x in a.strip("\n").split(","):
            search_paths.append(x)
    elif o in ("-f", "--file_list"):
        ## also allow seperating files by commas, i.e. -f sbl1.mbn,slb2.mbn,etc
        LoadSubsetOfFiles = True
        for x in a.strip("\n").split(","):
            file_list.append(x)
    else:
        assert False, "unhandled option"

if Operation==0:
    sys.exit(1)



print "\nSearch Paths"
print search_paths
print "\nFiles"
print file_list

if (Operation & OPERATION_PROGRAM) > 0:
    if rawprogram_filename is None:
        print "Error: You must specify an \"rawprogram\" XML file for option -r"
        sys.exit(1)
    else:
        rawprogram_filename = find_file(rawprogram_filename, search_paths)
        if rawprogram_filename is None:
            print "Error: You must specify an \"rawprogram\" XML file for option -r"
            sys.exit(1)

if (Operation & OPERATION_PATCH) > 0:
    if patch_filename is None:
        print "Error: You must specify an \"patch\" XML file for option -p"
        sys.exit(1)
    else:
        patch_filename = find_file(patch_filename, search_paths)
        if patch_filename is None:
            print "Error: You must specify an \"patch\" XML file for option -p"
            sys.exit(1)

elif Operation==OPERATION_ERASE or Operation==OPERATION_WIPE:
    if disk_name is None:
        print "Error: You must specify a DISK, option -d"
        sys.exit(1)


if (Operation & OPERATION_WIPE) > 0:
    if nopromt is True:
        # means don't bug them, i.e. automation
        pass
    else:
        print "\n\nWARNING: Destructive Operation Requested"
        print "\n\nWARNING: Wiping out the MBR/GPT information means you must re-partition and format after!!"
        var = raw_input("\nAre you sure you want to ZERO OUT the MBR/GPT information for %s of %s (y|N) " % (disk_name,ReturnSizeString(AvailablePartitions[disk_name])))
        if var=='Y' or var=='y':
            pass
        else:
            sys.exit()

    print "\n\nZero out first 2 sectors on card"
    Zerout(disk_name,0 ,2)  ## Sector 0, wipe out 2 sectors

    print "\n\nZero out last sector on card"
    Zerout(disk_name,-1,1)  ## Last Sector, wipe out 1 sector

    print "\n'%s' no longer has partition information\n" % disk_name
    sys.exit(1)



if (Operation & OPERATION_PROGRAM) > 0:

    # Can we test for sparse images?
    check_sparse = find_file("checksparse.py", search_paths)

    if check_sparse is not None:
        sz = "python %s -i %s " % (check_sparse,rawprogram_filename)
        for path in search_paths:
            sz += "-s %s " % path

        print "\n"+"-"*78
        print "-"*78

        print "Checking for sparse files"
        print "-"*78
        print "-"*78
        print sz
        os.system(sz)

    if os.path.basename(Filename)=="singleimage.bin":
        ## Wipe out any old singleimage
        SingleImageSize = 0
        opfile = open(Filename, "wb")
        opfile.close()

        print "\nProgramming %s of size %s" % (Filename,ReturnSizeString(DiskSize))

        if nopromt is False:
            if (DiskSize/(1024.0*1024.0))>100:
                var = raw_input("\nThis will be a LARGE singleimage.bin, it will take a long time, Do you want to continue? (Y|n)")
                if var=='Y' or var=='y' or var=='':
                    pass
                else:
                    sys.exit()
    else:
        if nopromt is True:
            # means don't bug them, i.e. automation
            pass
        else:
            var = raw_input("\nWARNING: Are you sure you want to write to '%s' of size %s (y|N) " % (Filename,ReturnSizeString(DiskSize)))
            if var=='Y' or var=='y':
                pass
            else:
                sys.exit()


NumPhyPartitions        = 0
PartitionArray          = []
PatchArray              = []
ZeroOutArray            = []
PhyPartition            = {}       # Main HASH that holds all the partition info

if rawprogram_filename is not None:
    ParseXML(rawprogram_filename)

if patch_filename is not None:
    ParseXML(patch_filename)

if DiskSize>0:
    if DiskSize<int(MinDiskSize):
        print "\nERROR: Current eMMC/SD card is too small to program these partitions"
        print "       Need at least %s" % ReturnSizeString(int(MinDiskSize))
        sys.exit(1)

if DiskSize==0:
    DiskSize = int(MinDiskSize)
    if DiskSize == 0:
        DiskSize = EMMCBLD_MIN_DISK_SIZE
    print "\nDiskSize was set to 0, DiskSize will be %s (%d sectors)" % (ReturnSizeString(DiskSize),int(DiskSize/512))

#print PartitionArray

## rawprogram0.xml is random in terms of start_sector. It's important for
## making the single image that we first write at sector 0, then 100, then 200 etc
## i.e. writing 200, then going back to write 100 is more complicated, so SORT

## Step 1. Sort all the start sectors where there's a file to program
PartitionStartSector = []

for Partition in PartitionArray:

    if Partition['filename']=="":
        #print "no filename for label ",Partition['label']
        continue

    if 'sparse' in Partition:
        m = re.search("true", Partition['sparse'],re.IGNORECASE)
        if type(m) is not NoneType:
            print """
            \nERROR: Found sparse image, can't continue\n
            You must first run \"python checksparse.py -i rawprogram0.xml -s C:\path1 -s C:\path2\"
            This will modify rawprogram0.xml to be able to handle the sparse image(s)\n
            Also, you *must* specify the various paths to the sparse images and they need to be
            opened and parsed. Then you can run msp.py again to program.
            """
            sys.exit(1)

    PartitionStartSector.append( Partition["start_sector"] )

PartitionStartSector.sort()

## Step 2. Create a "PartitionSorted" based on PartitionStartSector
PartitionSorted = []
Count = 0
while 1 and (Operation & OPERATION_PROGRAM) > 0:
    for Partition in PartitionArray:
        ##print "---------------- with start_sector = %i and PartitionStartSector[%i]=%i" % (Partition["start_sector"],Count,PartitionStartSector[Count])

        if Partition['filename']=="":
            continue

        if Partition["start_sector"] == PartitionStartSector[Count]:
            ##print "FOUND, len(PartitionStartSector)=%d and len(PartitionSorted)=%d" % (len(PartitionStartSector),len(PartitionSorted))
            # To be here means I found the *next* start_sector in order, i.e. 0,100,200 etc
            PartitionSorted.append( Partition )
            Count += 1  # now go to next sorted entry
            if len(PartitionSorted) == len(PartitionStartSector):
                break   # we're done, both arrays are the same size

    if len(PartitionSorted) == len(PartitionStartSector):
        break   # we're done, both arrays are the same size


#for Partition in PartitionSorted:
#    print "----------------------"
#    print Partition

# Possibilities,
# 1. programming DISK, then patching DISK
# 2. patching FILES, programming singleimage
# 3. programming DISK
# 4. programming singleimage
# 5. patching DISK
# 6. patching FILES

if (Operation & OPERATION_PROGRAM) > 0 and (Operation & OPERATION_PATCH) > 0:
    ## Have info to do both write and patch

    if Filename!="singleimage.bin":
        # This is a DISK and we need to zero out sectors already there

        print "\n\nZero out first 34 sectors on card"
        Zerout(Filename,0,34)           # First 34 sectors wipes MBR/GPT

        print "\n\nZero out last 33 sectors on card, DiskSize=",DiskSize
        Zerout(Filename,(DiskSize/512)-34,33) # Last 33 wipes backup GPT

        # User could have specified more zero out ranges
        for Zeroout in ZeroOutArray:
            print "\n======================================================================"
            print "ZERO-OUT: '%s'" % Zeroout['label']
            Zerout(Filename,Zeroout['start_sector'],int(Zeroout['num_partition_sectors']))

        PerformWrite()
        PerformPatching()
        DoubleCheckDiskSize()
    else:
        PerformPatching()
        PerformWrite()
        DoubleCheckDiskSize()

elif (Operation & OPERATION_PROGRAM) > 0:
    PerformWrite()
    print "\n"+"-"*78
    print "If you wrote any partition table information (MBR0.bin, gpt_main0.bin, etc)"
    print " "
    print "         _             _ _      __                     _   "
    print "        | |           ( ) |    / _|                   | |  "
    print "      __| | ___  _ __ |/| |_  | |_ ___  _ __ __ _  ___| |_ "
    print "     / _` |/ _ \\| '_ \\  | __| |  _/ _ \\| '__/ _` |/ _ \\ __|"
    print "    | (_| | (_) | | | | | |_  | || (_) | | | (_| |  __/ |_ "
    print "     \\__,_|\\___/|_| |_|  \\__| |_| \\___/|_|  \\__, |\\___|\\__|"
    print "                                             __/ |         "
    print "                                            |___/          "
    print "             _                       _        _     "
    print "            | |                     | |      | |    "
    print "            | |_  ___    _ __   __ _| |_  ___| |__  "
    print "            | __|/ _ \\  | '_ \\ / _` | __|/ __| '_ \\ "
    print "            | |_| (_) | | |_) | (_| | |_| (__| | | |"
    print "             \\__|\\___/  | .__/ \\__,_|\\__|\\___|_| |_|"
    print "                        | |                         "
    print "                        |_|                         "
    print "\n"

    sudo=""
    if sys.platform.startswith("linux"):
        sudo="sudo "

    print "\tEx: %spython msp.py -p patch0.xml -d %s " % (sudo,Filename)
    print "\n\tOr, do it all in one step\n"
    print "\tEx: %spython msp.py -r rawprogram.xml -d %s -p patch0.xml" % (sudo,Filename)
    print "\n"+"-"*78

    DoubleCheckDiskSize()

elif (Operation & OPERATION_PATCH) > 0:
    PerformPatching()
    DoubleCheckDiskSize()

if (Operation & OPERATION_PROGRAM) > 0:

    if os.path.basename(Filename)=="singleimage.bin":
        print "\nNOTE: This program does *not* pad the last partition, therefore"
        print "      singleimage.bin might be smaller than %d sectors " % int(DiskSize/512)

        print "\n\nSUCCESS - %s created" % Filename
        print "SUCCESS - %s created" % Filename
        print "SUCCESS - %s created\n" % Filename

        if FileNotFoundShowWarning==1:
            print "\nWARNING: 1 or more files were *not* found, your singleimage.bin is *NOT* complete"
            print "\nWARNING: 1 or more files were *not* found, your singleimage.bin is *NOT* complete"
            print "\nWARNING: 1 or more files were *not* found, your singleimage.bin is *NOT* complete\n\n"



print "\n"