This is MiscFunctions.py:

path = r"/home/activ/utils/DataComparisonScript"


if not os.path.exists("log"):
    os.makedirs("log")
    print("done!")

def sqliteConnection(path):
    conn = sqlite3.connect(path+"/TempLocalDatabase/DB_Comparisons.db")
    c = conn.cursor()
    c2 = conn.cursor()
    return c, c2, conn

def GetDate():
    """return the date in YYY-MM-DD"""
    return datetime.datetime.today().strftime('%Y-%m-%d')

def GetTableName(host,table):
    table_name = "{0}_Table{1}_{2}".format(host, table, GetDate())
    return table_name


def CleanupLogs(days): # runs at startup
    """Cleans up logs that are a specified days old or older"""
    folder_path = path+"\log"
    print(folder_path)
    file_ends_with = ".log"
    how_many_days_old_logs_to_remove = days
    now = time.time()
    only_files = []
    for file in os.listdir(folder_path):
        file_full_path = os.path.join(folder_path, file)
        if os.path.isfile(file_full_path) and file.endswith(file_ends_with:
            # Delete files older than x days
            if os.stat(file_full_path).st_mtime < now - how_many_days_old_logs_to_remove * 86400:
                os.remove(file_full_path)


def CleanupDatabase(): # runs at startup
    """Deletes the LOCAL sqlite3 database on startup removing old data as these are stored temporarily due to disk space"""
    try:
        os.remove(path+"/TempLocalDatabase/DB_Comparisons.db")
    except OSError:
        pass






this is

import sqlite3
from ComparisonConfig import username, password, path, InsertChunk
from MiscFunctions import GetDate,CleanupDatabase
import shlex
import socket
import os
import subprocess
from MiscFunctions import logger, sqliteConnection
from datetime import datetime
hostname = socket.gethostname()

#CleanupDatabase()

if not os.path.exists(path+"/TempLocalDatabase/"):
    os.makedirs(path+"/TempLocalDatabase")

c,c2,conn = sqliteConnection(path)

# This file contains the function that runs AdminDataCapture and captures it's output into a sqlite3 database, one table per output. This is a local database stored in CWD/TempLocalDatabase/
# See readme.txt for more information on the purpose of some functions if needed





def TupleForInserts(values):
    vtuple = tuple(['?' for i in values]) # values is line.split("|")[7::2] for these files e.g all the pipe separated values
    vtuple = str(vtuple)
    vtuple = vtuple.replace("'", "")
    vtuple = vtuple.replace(" ?)", " ?, ?, ?, ?)")# add an extra ? for inserting Symbol later on in query
    return vtuple




def InitialiseTable(table_name,data):
    global vtuple # make vutple variable global as it is needed outside this function later on for usage in inserts
    DropTable = "DROP TABLE IF EXISTS '{}'".format(table_name)
    c.execute(DropTable)
    CreateTable = "CREATE TABLE IF NOT EXISTS '{}' (Symbol VARCHAR(255), ExchangeCode VARCHAR(255), PermID VARCHAR(255), PRIMARY KEY(Symbol))".format(table_name) # initialise table with the columns that are ALWAYS present
    c.execute(CreateTable)

    columns = data[6::2]
    values = data[7::2]
    vtuple = TupleForInserts(values) # Generate (?,?,...,?) tuple for inserts
    for column in columns:
        Query = "ALTER TABLE '{0}' ADD '{1}' VARCHAR(255)".format(table_name,column)
        c.execute(Query)
    conn.commit()


def CaptureIntoSQL(host,table, InitialConfigDone=False):
    """Starts AdminDataCapture and redirects output to a table in a local SQL database"""
    DataListToInsert = []

    logger.info(host+" capture starting")
    command =path+"/AdminDataCapture_x86-64_rhel6_gcc48_cxx11-vstring_mds -u {0} -p {1} -h {2} -t {3} --static -c".format(username,password,host,table)
    print(command)

    table_name ="{0}_Table{1}_{2}".format(host,table,GetDate())
    process = subprocess.Popen(shlex.split(command), stdout=subprocess.PIPE)
    logger.info("process started for {}".format(table_name))
    #logger.error("process did not start for {}".format(table_name))
    capture_begin = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
    logger.info("Capture begins {}".format(capture_begin))
    while True:
        cmdoutput = process.stdout.readline().decode("utf-8")
        if cmdoutput == '' and process.poll() is not None:
            InsertQuery = "INSERT INTO '{0}' VALUES{1}".format(table_name, vtuple)
            c.executemany(InsertQuery,DataListToInsert)
            conn.commit()
            break
        if cmdoutput:
            data = cmdoutput.split("|")
            try:
                Symbol = data[5]
                if Symbol != "<Symbol>" and InitialConfigDone == False: # Run InitialiseTable() function to create the sqlite3 table, if it hasn't been done
                    try:
                        InitialiseTable(table_name,data)
                        InitialConfigDone = True
                        logger.info("Table created for {0}".format(table_name))
                    except:
                        logger.info("InitialiseTable(table_name,data) failed:\n table_name: {0} \n data: {1}".format(table_name,data))


                elif Symbol != "<Symbol>" and InitialConfigDone == True: # If we've already ran InitialiseTable() begin inserting data.
                    Permission = data[3]
                    exchangeCode = data[5].split(".")[-1]
                    if exchangeCode == "":
                        exchangeCode = "."
                    else:
                        pass
                    valuelist = data[7::2]
                    valuelist[0:0] = [Symbol,exchangeCode,Permission]
                    DataListToInsert.append(valuelist)
                    if len(DataListToInsert) == InsertChunk:
                        InsertQuery = "INSERT INTO '{0}' VALUES{1}".format(table_name, vtuple)
                        try:
                            c.executemany(InsertQuery,DataListToInsert)
                            conn.commit()

                        except:
                            logger.info("FAILED to insert")
                            logger.error("Failed to insert data \n Values: {0} \n table_name : {1} \n InsertQuery : {2}".format(valuelist,table,InsertQuery))
                        DataListToInsert = []
                    else:
                        pass

            except:
                pass
    capture_end_time =datetime.now().strftime('%Y-%m-%d %H:%M:%S')
    logger.info("capture end time is {}".format(capture_end_time))
    conn.commit()