Plutonergy

Compile and Copy

May 16th, 2021
775
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
  1. from PyQt5        import QtWidgets, uic
  2. from PyQt5.Qt     import QObject, QRunnable, QThreadPool
  3. from PyQt5.QtCore import pyqtSignal, pyqtSlot
  4. from functools    import partial
  5. from sqlite3      import Error
  6. import concurrent.futures
  7. import hashlib
  8. import os
  9. import pathlib
  10. import shutil
  11. import sqlite3
  12. import subprocess
  13. import sys
  14. import tempfile
  15. import time
  16. import traceback
  17.  
  18. sqliteconnection = sqlite3.connect('/home/plutonergy/Documents/sneaky_compiler_v2.sqlite')
  19. sqlitecursor = sqliteconnection.cursor()
  20.  
  21. white_extensions = ['.ui', '.py', '.so', '.pyx', '.zip', '.txt', '.ini']
  22. white_dirs = ['gui', 'img', 'mkm_expansion_data']
  23. black_dirs = ['__pycache__']
  24.  
  25. global techdict
  26. techdict = {}
  27.  
  28. class WorkerSignals(QObject):
  29.     finished = pyqtSignal()
  30.     error = pyqtSignal(tuple)
  31.     result = pyqtSignal(object)
  32.     progress = pyqtSignal(int)
  33.  
  34. class Worker(QRunnable):
  35.     def __init__(self, function):
  36.         super(Worker, self).__init__()
  37.         self.fn = function
  38.         self.signals = WorkerSignals()
  39.  
  40.     @pyqtSlot()
  41.     def run(self):
  42.         try:
  43.             result = self.fn()
  44.         except:
  45.             traceback.print_exc()
  46.             exctype, value = sys.exc_info()[:2]
  47.             self.signals.error.emit((exctype, value, traceback.format_exc()))
  48.         else:
  49.             self.signals.result.emit(result)  # Return the result of the processing
  50.         finally:
  51.             self.signals.finished.emit()  # Done
  52.  
  53. def sqlite_superfunction(connection, cursor, table, column, type):
  54.     global techdict
  55.  
  56.     try: return techdict[connection][table][column]
  57.     except KeyError:
  58.         if connection not in techdict:
  59.             techdict.update({connection: {}})
  60.         if table not in techdict[connection]:
  61.             techdict[connection].update({table : {}})
  62.  
  63.     query_one = 'select * from ' + table
  64.     try: cursor.execute(query_one)
  65.     except Error:
  66.  
  67.         with connection:
  68.             query_two = 'create table ' + table + ' (id INTEGER PRIMARY KEY AUTOINCREMENT)'
  69.             cursor.execute(query_two)
  70.  
  71.             if table == 'settings':
  72.                 query_three = 'insert into settings values(?)'
  73.                 sqlitecursor.execute(query_three, (None,))
  74.  
  75.     cursor.execute(query_one)
  76.  
  77.     col_names = cursor.description
  78.  
  79.     for count, row in enumerate(col_names):
  80.         if row[0] not in techdict[connection][table]:
  81.             techdict[connection][table].update({row[0] : count})
  82.  
  83.     try: return techdict[connection][table][column]
  84.     except KeyError:
  85.         with connection:
  86.             query = 'alter table ' +  table + ' add column ' + column + ' ' + type.upper()
  87.             cursor.execute(query)
  88.  
  89.     return len(col_names)
  90.  
  91. def db_sqlite(table, column, type='text'):
  92.     return sqlite_superfunction(sqliteconnection, sqlitecursor, table, column, type)
  93.  
  94. class DB: # local database
  95.  
  96.     location = db_sqlite('local_files', 'location')
  97.     md5 = db_sqlite('local_files', 'md5')
  98.  
  99.     copy_pyx_files = db_sqlite('settings', 'copy_pyx_files', 'integer')
  100.     copy_ini_files = db_sqlite('settings', 'copy_ini_files', 'integer')
  101.     force_update = db_sqlite('settings', 'force_update', 'integer')
  102.     program_name = db_sqlite('settings', 'program_name')
  103.     presets = db_sqlite('settings', 'presets')
  104.     recent_job = db_sqlite('settings', 'recent_job')
  105.  
  106. class tech:
  107.     global techdict
  108.     @staticmethod
  109.     def md5_hash(local_path):
  110.         hash_md5 = hashlib.md5()
  111.         with open(local_path, "rb") as f:
  112.             for chunk in iter(lambda: f.read(4096), b""):
  113.                 hash_md5.update(chunk)
  114.         return hash_md5.hexdigest()
  115.     @staticmethod
  116.     def save_setting(setting_column, object):
  117.         """
  118.        updates sqlite settings table with the column and bool or other value from object
  119.        if object is string, that works!
  120.        :param setting_column: string
  121.        :param object: any Qt-object or string
  122.        """
  123.         def update(column, value):
  124.             with sqliteconnection:
  125.                 query = f'update settings set {column} = (?) where id is 1'
  126.                 sqlitecursor.execute(query, (value,))
  127.  
  128.         if type(object) == str:
  129.             update(setting_column, object)
  130.  
  131.         try:
  132.             if object.isChecked():
  133.                 update(setting_column, object.isChecked())
  134.             else:
  135.                 update(setting_column, object.isChecked())
  136.         except:
  137.             pass
  138.  
  139.         try:
  140.             if object.currentText():
  141.                 update(setting_column, object.currentText())
  142.         except:
  143.             pass
  144.  
  145.         try:
  146.             if object.toPlainText():
  147.                 update(setting_column, object.toPlainText())
  148.         except:
  149.             pass
  150.  
  151.     @staticmethod
  152.     def retrieve_setting(index):
  153.         """
  154.        :param index: integer
  155.        :return: column
  156.        """
  157.         sqlitecursor.execute('select * from settings where id is 1')
  158.         data = sqlitecursor.fetchone()
  159.         if data:
  160.             return data[index]
  161.  
  162.     @staticmethod
  163.     def empty_insert_query(cursor, table):
  164.         cursor.execute('PRAGMA table_info("{}")'.format(table,))
  165.         tables = cursor.fetchall()
  166.         query_part1 = "insert into " + table + " values"
  167.         query_part2 = "(" + ','.join(['?']*len(tables)) + ")"
  168.         values = [None] * len(tables)
  169.         return query_part1 + query_part2, values
  170.  
  171. def create_shared_object(path):
  172.     subprocess.run(['cythonize', '-i', '-3', path])
  173.  
  174. class main(QtWidgets.QMainWindow):
  175.     def __init__(self):
  176.         super(main, self).__init__()
  177.         uic.loadUi('main_program_v1.ui', self)
  178.         self.move(1200,150)
  179.         self.setStyleSheet('background-color: gray; color: black')
  180.         self.threadpool_main = QThreadPool()
  181.         self.threadpool_status = QThreadPool()
  182.         self.status_bar = self.statusBar()
  183.         self.preset_settings()
  184.  
  185.         # TRIGGERS >
  186.         self.btn_kill.clicked.connect(self.delete_preset)
  187.         self.btn_save_preset.clicked.connect(self.save_current_job)
  188.         self.btn_start_compile.clicked.connect(self.start_compiling)
  189.         self.check_copy_pyx.clicked.connect(partial(tech.save_setting, 'copy_pyx_files', self.check_copy_pyx))
  190.         self.check_copy_ini.clicked.connect(partial(tech.save_setting, 'copy_ini_files', self.check_copy_ini))
  191.         self.check_force.clicked.connect(partial(tech.save_setting, 'force_update', self.check_force))
  192.         self.combo_name.currentIndexChanged.connect(self.load_preset)
  193.         # TRIGGER <
  194.  
  195.         self.show()
  196.  
  197.     def mousePressEvent(self, QMouseEvent):
  198.         self.load_preset()
  199.  
  200.     def delete_preset(self):
  201.         """
  202.        deletes the current preset from presets
  203.        """
  204.         name = self.combo_name.currentText()
  205.         data = tech.retrieve_setting(DB.presets)
  206.  
  207.         if data:
  208.             presets = data.split('\n')
  209.             for c in range(len(presets)-1,-1,-1):
  210.                 if presets[c].find(name) > -1 and presets[c][0:len(name) +1] == f'{name},':
  211.                     presets.pop(c)
  212.                     break
  213.  
  214.             values = '\n'.join(presets)
  215.             with sqliteconnection:
  216.                 sqlitecursor.execute('update settings set presets = (?) where id is 1', (values,))
  217.  
  218.     def preset_settings(self):
  219.         """
  220.        set gui according to sqlite settings table
  221.        :return:
  222.        """
  223.         cycle = {
  224.             self.check_force: 'force_update',
  225.             self.combo_name: 'program_name',
  226.             self.check_copy_pyx: 'copy_pyx_files',
  227.             self.check_copy_ini: 'copy_ini_files'
  228.         }
  229.  
  230.         for widget, string in cycle.items():
  231.             rv = tech.retrieve_setting(getattr(DB, string))
  232.             if rv:
  233.                 if rv == True or rv == False:
  234.                     widget.setChecked(rv)
  235.  
  236.                 elif type(rv) == str:
  237.                     try:
  238.                         widget.setCurrentText(rv)
  239.                         continue
  240.                     except:
  241.                         pass
  242.  
  243.                     try:
  244.                         widget.setPlainText(rv)
  245.                         continue
  246.                     except:
  247.                         pass
  248.  
  249.         data = tech.retrieve_setting(DB.presets)
  250.         if data:
  251.             presets = data.split('\n')
  252.             saves = []
  253.             for i in presets:
  254.                 j = i.split(',')
  255.                 saves.append(j[0])
  256.             saves.sort()
  257.             self.combo_name.clear()
  258.             for ii in saves:
  259.                 self.combo_name.addItem(ii)
  260.  
  261.             recent_job = tech.retrieve_setting(DB.recent_job)
  262.             if recent_job:
  263.                 for count, ii in enumerate(saves):
  264.                     if recent_job == ii:
  265.                         self.combo_name.setCurrentIndex(count)
  266.                         self.load_preset()
  267.  
  268.     def load_preset(self):
  269.         """
  270.        looks in sqlitedatabase for the name thats in the combobox and sets the
  271.        data accordingly, its a CSV 0: name, 1: source, 2: destination
  272.        """
  273.         name = self.combo_name.currentText()
  274.         data = tech.retrieve_setting(DB.presets)
  275.         if data:
  276.             presets = data.split('\n')
  277.             for c in range(len(presets)-1,-1,-1):
  278.                 if presets[c].find(name) > -1 and presets[c][0:len(name) +1] == f'{name},':
  279.                     this_job = presets[c].split(',')
  280.                     self.te_source.setPlainText(this_job[1])
  281.                     self.te_dest.setPlainText(this_job[2])
  282.                     tech.save_setting('recent_job', name)
  283.                     break
  284.  
  285.     def delete_and_fresh_copy(self, source_path, new_path):
  286.         """
  287.        deletes tmporary working directory and copies
  288.        the entire tree from source directory here
  289.        :param source_path: string
  290.        :param new_path: string
  291.        """
  292.         if os.path.exists(new_path):
  293.             shutil.rmtree(new_path)
  294.         shutil.copytree(source_path, new_path)
  295.  
  296.     def find_files_of_interest(self, tmp_dir):
  297.         """
  298.        returns a list of files that was hit from your white_extensions and white_dirs
  299.        meaning all trash files are excluded from the list
  300.        :param tmp_dir: folder as string
  301.        :return: list of files
  302.        """
  303.         save_files = []
  304.         for walk in os.walk(tmp_dir):
  305.             for file in walk[2]:
  306.  
  307.                 current_file = f'{walk[0]}/{file}'
  308.                 top_dir = walk[0][walk[0].rfind('/') +1:]
  309.  
  310.  
  311.                 for ext in white_extensions:
  312.                     if file.lower().find(ext) > -1 and file[-len(ext):len(file)].lower() == ext:
  313.                         save_files.append(current_file)
  314.  
  315.                 for folder in white_dirs:
  316.                     if top_dir == folder and current_file not in save_files:
  317.                         save_files.append(current_file)
  318.  
  319.         return save_files
  320.  
  321.     def check_if_file_is_interesting(self, file_or_list_of_files):
  322.         """
  323.        if file_or_list_of_files == str, returns True if file not exists or md5 is changed
  324.        if file_or_list_of_files == list, pops files from the list that is SAME as database
  325.        :param file_or_list_of_files: file as string or list of files
  326.        :return: bool or list
  327.        """
  328.         def quick(file):
  329.             sqlitecursor.execute('select * from local_files where location = (?)', (file,))
  330.             data = sqlitecursor.fetchone()
  331.             if not data:
  332.                 return True
  333.             else:
  334.                 hash = tech.md5_hash(file)
  335.                 if data[DB.md5] != hash:
  336.                     return True
  337.  
  338.         if type(file_or_list_of_files) == str:
  339.  
  340.             if self.check_force.isChecked(): # force recompile
  341.                 return False
  342.  
  343.             rv = quick(file_or_list_of_files)
  344.             return rv
  345.  
  346.         elif type(file_or_list_of_files) == list:
  347.  
  348.             if self.check_force.isChecked():  # force recompile
  349.                 return file_or_list_of_files
  350.  
  351.             for c in range(len(file_or_list_of_files) - 1, -1, -1):
  352.                 if not quick(file_or_list_of_files[c]):
  353.                     file_or_list_of_files.pop(c)
  354.  
  355.         return file_or_list_of_files
  356.  
  357.     def save_md5_hashes(self, file_list):
  358.         """
  359.        save all hashes for the current job
  360.        :param file_list:
  361.        """
  362.         query, values = tech.empty_insert_query(sqlitecursor, 'local_files')
  363.         for file in file_list:
  364.             values[DB.location] = file
  365.             values[DB.md5] = tech.md5_hash(file)
  366.  
  367.             with sqliteconnection:
  368.                 sqlitecursor.execute('delete from local_files where location = (?)', (file,))
  369.                 sqlitecursor.execute(query, values)
  370.  
  371.     def determine_which_files_to_be_compiled(self, list_of_files):
  372.         """
  373.        :param list_of_files:
  374.        :return: a list of pyx files
  375.        """
  376.         pyx_files = []
  377.         for i in list_of_files:
  378.             if len(i) > len('.pyx') and i[-len('.pyx'):len(i)].lower() == '.pyx':
  379.                 pyx_files.append(i)
  380.         return pyx_files
  381.  
  382.     def compile_list_of_pyxfiles(self, single_or_list):
  383.         with concurrent.futures.ProcessPoolExecutor() as executor:
  384.             for _, _ in zip(single_or_list, executor.map(create_shared_object, single_or_list)):
  385.                 pass
  386.  
  387.     def delete_unwanted_files_from_tmp_dir(self, all_files, tmp_dir):
  388.         """
  389.        removes all unwanted files first, then removes all unwanted dirs
  390.        :param all_files: list
  391.        :param tmp_dir: string
  392.        """
  393.         for walk in os.walk(tmp_dir):
  394.             for file in walk[2]:
  395.                 current_file = f'{walk[0]}/{file}'
  396.                 if current_file not in all_files:
  397.                     os.remove(current_file)
  398.  
  399.         for walk in os.walk(tmp_dir):
  400.             for dir in walk[1]:
  401.                 iterdir = f'{walk[0]}/{dir}'
  402.                 for walkwalk in os.walk(iterdir):
  403.                     if walkwalk[1] == [] and walkwalk[2] == []:
  404.                         shutil.rmtree(walkwalk[0])
  405.  
  406.     def remove_ext_files_before_final(self, all_files, ext):
  407.         """
  408.        parameter needed, but asks the gui checkboxes if we keep them
  409.        :param all_files: list
  410.        :param ext: string
  411.        :return: list
  412.        """
  413.         def remover(all_files, ext):
  414.             for c in range(len(all_files)-1,-1,-1):
  415.                 if all_files[c].find(ext) > -1 and all_files[c][-len(ext):len(all_files[c])] == ext:
  416.                     all_files.pop(c)
  417.  
  418.         if not self.check_copy_pyx.isChecked() and ext == '.pyx':
  419.             remover(all_files, ext)
  420.  
  421.         if not self.check_copy_ini.isChecked() and ext == '.ini':
  422.             remover(all_files, ext)
  423.  
  424.         return all_files
  425.  
  426.     def copy_tree(self, source_dir, destination_dir):
  427.         shutil.copytree(source_dir, destination_dir, dirs_exist_ok=True)
  428.  
  429.     def pre_checking(self, source_path, destination_path):
  430.         """
  431.        checks that all paths seems ok
  432.        :param source_path: string
  433.        :param destination_path: string
  434.        :return: bool
  435.        """
  436.         if not os.path.exists(source_path):
  437.             self.status_bar.showMessage('Source path has covid!')
  438.             return False
  439.  
  440.         if not os.path.exists(destination_path):
  441.             if len(destination_path) < 5:
  442.                 self.status_bar.showMessage('Destination path short!')
  443.                 return False
  444.  
  445.             try:
  446.                 pathlib.Path(destination_path).mkdir(parents=True)
  447.                 if not os.path.exists(destination_path):
  448.                     self.status_bar.showMessage('Destination path shit on floor!')
  449.                     return False
  450.  
  451.             except:
  452.                 self.status_bar.showMessage('Destination path has rabies!')
  453.                 return False
  454.  
  455.         if self.combo_name.currentText().replace(" ", "") == "":
  456.             self.combo_name.setCurrentText('SNEAKY_TMP_FOLDER')
  457.  
  458.         return True
  459.  
  460.     def save_current_job(self):
  461.         """
  462.        saves in sqlite a string that later is split('\n') and then split again(',')
  463.        0: name, 1: source, 2: destination
  464.        """
  465.         name = self.combo_name.currentText()
  466.         source_path = self.te_source.toPlainText()
  467.         destination_path = self.te_dest.toPlainText()
  468.         presets = []
  469.  
  470.         data = tech.retrieve_setting(DB.presets)
  471.  
  472.         if data:
  473.             presets = data.split('\n')
  474.             for c in range(len(presets)-1,-1,-1):
  475.                 if presets[c].find(name) > -1 and presets[c][0:len(name) +1] == f'{name},':
  476.                     presets.pop(c)
  477.                     break
  478.  
  479.         presets.append(f'{name},{source_path},{destination_path}')
  480.         values = '\n'.join(presets)
  481.         with sqliteconnection:
  482.             sqlitecursor.execute('update settings set presets = (?) where id is 1', (values,))
  483.  
  484.     def start_compiling(self):
  485.         """
  486.        everything starts here
  487.        """
  488.         source_path = self.te_source.toPlainText()
  489.         destination_path = self.te_dest.toPlainText()
  490.  
  491.         if not self.pre_checking(source_path, destination_path):
  492.             return
  493.  
  494.         if os.path.exists('/mnt/ramdisk'):
  495.             tmp_dir = f'/mnt/ramdisk/{self.combo_name.currentText()}'
  496.         else:
  497.             tmp_dir = f'{tempfile.gettempdir()}/{self.combo_name.currentText()}'
  498.  
  499.         def finished(self):
  500.             complete_files = self.find_files_of_interest(tmp_dir)
  501.             complete_files = self.remove_ext_files_before_final(complete_files, '.pyx')
  502.             complete_files = self.remove_ext_files_before_final(complete_files, '.ini')
  503.             self.delete_unwanted_files_from_tmp_dir(complete_files, tmp_dir)
  504.             self.copy_tree(tmp_dir, destination_path)
  505.  
  506.             end_time = time.time() - self.start_time
  507.             message = f'Completed in: {round(end_time, 2)}s... {len(complete_files)} has been updated!'
  508.             self.status_bar.showMessage(message)
  509.             self.save_current_job()
  510.  
  511.         self.start_time = time.time()
  512.  
  513.         self.delete_and_fresh_copy(source_path, tmp_dir)
  514.         save_files = self.find_files_of_interest(tmp_dir)
  515.         save_files = self.check_if_file_is_interesting(save_files)
  516.         pyx_files = self.determine_which_files_to_be_compiled(save_files)
  517.         self.save_md5_hashes(save_files)
  518.  
  519.         thread = Worker(partial(self.compile_list_of_pyxfiles, pyx_files))
  520.         thread.signals.finished.connect(partial(finished, self))
  521.         self.threadpool_main.start(thread)
  522.  
  523.  
  524. app = QtWidgets.QApplication(sys.argv)
  525. window = main()
  526. app.exec_()
RAW Paste Data