import random import time from sortedcontainers import SortedDict # These functions are used to populate the database with # values that are kinda-sorta like the ones in my app. def xdi(B,N,M): ''' eXponential Distribution of Integers, XDI Return a list of length N of integers with max value M, from an exponential distribution with beta B. ''' M = int(M) # just in case # pre-allocate the list rather than building it with .append() r = [0]*N for j in range(N) : r[j] = int( random.expovariate(B) ) % M return r RWTALPH = 'aeiouåáïbcdfghjklmnpqrstvwxyz_BCDFGHJKLMNPQRSTVWZ' def rwt(N): ''' Random Word-like Token, RWT Return a word-like token comprising N Latin-1 letters with an emphasis on vowels. ''' ii = xdi( 0.1, N, len(RWTALPH)-1 ) return ''.join( [ RWTALPH[i] for i in ii ] ) def rft(): ''' Random Flag Token Return an 8-char token composed mostly of dashes with a sprinkle of Xs, e.g. "---X--X-" or "XX------". ''' ii = xdi( 2, 8, 2 ) return ''.join( ['-X'[i] for i in ii ] ) # # Define the database. All this code is out here at the module level -- # instead of inside the Table Model class -- because in the real app, the # database is managed in a separate module entirely. # # The database comprises a single SortedDict, DATABASE, in which the keys are # rwt(8) "word" tokens, and the values are tuples comprising ( word, int in # 0-99, rft() flag string). (Yes this is redundant in having the word as both # key and value but it simplifies coding the data() method of the model.) # DATABASE = SortedDict() NUMBER_OF_ROWS = 10000 # how many rows to create # # The contents are accessed via a valuesView object, VALUES_BY_ROW. This can # be indexed, so VALUES_BY_ROW[J] is the values for the Jth entry in the # ascending order of the dict keys. VALUES_BY_ROW[J][2] is the column-2 data # from row J (the flag string column). # VALUES_BY_ROW = None # valuesview here # # The database is ordered using the Python native default sort order, which # is almost certainly not what the data view wants. In order to sort it # correctly (i.e. locale-aware and with or without respecting case) the view # will impose a sort order using get_column_vector() below. # # In order to provide the indirection required for sorting, access to # VALUES_BY_ROW is by way of a sort vector of indices. As many as 6 such # vectors, one for each column for each sort order, are stored in a two # lists, SORT_UP_VECTORS and SORT_DOWN_VECTORS. When the database is # initialized, none of these exist. Thus the view must call get_column_vector # after initializing the database. # SORT_UP_DICTS = [ None, None, None ] SORT_UP_VECTORS = [ None, None, None ] SORT_DOWN_VECTORS = [ None, None, None ] # Clear the database, called when the model initializes and before Refresh def clear_the_db() : global DATABASE, VALUES_BY_ROW, SORT_UP_VECTORS, SORT_DOWN_VECTORS DATABASE.clear() VALUES_BY_ROW = None SORT_UP_DICTS = [ None, None, None ] SORT_UP_VECTORS = [ None, None, None ] SORT_DOWN_VECTORS = [ None, None, None ] # Rebuild the database by repopulating it with phony data. This emulates what # worddata.py does when it scans the book and tabulates its vocabulary. def rebuild_the_db(): global DATABASE, VALUES_BY_ROW, NUMBER_OF_ROWS # Throw everything away. In the real app, worddata doesn't throw everything away, # it only clears the counts in column 1. clear_the_db() # Create values for column 1, small random integers. Use the length # of column 1 to control the loop. column_1 = xdi( 0.05, NUMBER_OF_ROWS, 100 ) # Populate the database with rows of stuff for k in column_1 : w = rwt(8) # a random "word" f = rft() # a random "flag" string DATABASE[ w ] = ( w, k, f ) # Create the indexable view of the data VALUES_BY_ROW = DATABASE.values() # # On request from the table model, return a sort vector for a given column # (0, 1 or 2) based on a given sort order and key function. # # 1. If we have a vector for that column and sort order, just return it. # # 2. If we do not have a vector for ascending order on this column, build a # dict for accessing the values of DATABASE in ascending sequence over that # column. Place the sortedDict in SORT_UP_DICTS[col]. # # 3. If the request is for an ascending order, return that. # # 4. Make a vector for descending order by applying reversed() to the # ascending order dict and place it in SORT_DOWN_VECTORS[col], and return it. # # We build the ascending vector as follows. Make a SortedDict in which each # key is the concatenation of the column value for a row (which may have # dups) and the primary key for the same row. Use the key_func passed by the # caller. The combination is unique and sorts properly. # # The value associated to each key is the index of that row in DATABASE. # Because this is a SortedDict, the values taken in sequence produce the # DATABASE rows in column-value order. # # This list of functions extracts the key for a vector for each column. # Column 0, just the "word"; col 1, nnnnnPRIMARYKEY; col 2 XXXXXXXXPRIMARYKEY # KEY_GETTERS = [ lambda j : VALUES_BY_ROW[j][0], lambda j : '{:05}{}'.format( VALUES_BY_ROW[j][1], VALUES_BY_ROW[j][0] ), lambda j : '{}{}'.format( VALUES_BY_ROW[j][2], VALUES_BY_ROW[j][0] ) ] def get_column_vector( col, order, key_func = None, filter_func = None ): global DATABASE, VALUES_BY_ROW, SORT_UP_DICTS, SORT_UP_VECTORS, SORT_DOWN_VECTORS if filter_func is None : # return a sort vector for the whole database, cached if we have one. vector = SORT_UP_VECTORS[ col ] if order == Qt.AscendingOrder \ else SORT_DOWN_VECTORS[ col ] if vector is None : # we need to make one if SORT_UP_VECTORS[ col ] is None : # we have no up-vector getter_func = KEY_GETTERS[ col ] sort_dict = SortedDict( key_func ) for j in range( len(DATABASE) ) : k = getter_func( j ) sort_dict[ k ] = j # Cache the whole-database dict for later. SORT_UP_DICTS[ col ] = sort_dict # Cache the values-view of it for use by the view SORT_UP_VECTORS[ col ] = sort_dict.values() # We have a sort-up vector, or was it sort-down we need? vector = SORT_UP_VECTORS[ col ] # be optimistic if order == Qt.DescendingOrder : # ok, make the reverse SORT_DOWN_VECTORS[ col ] = [ j for j in reversed( SORT_UP_VECTORS[ col ] ) ] vector = SORT_DOWN_VECTORS [ col ] else : # Build a filtered vector for the requested column, order and filter. getter_func = KEY_GETTERS[ col ] sort_dict = SortedDict( key_func ) for j in range( len(DATABASE) ) : if filter_func( *VALUES_BY_ROW[j] ) : k = getter_func( j ) sort_dict[ k ] = j vector = sort_dict.values() if order == Qt.DescendingOrder : # convert the up-vector to a down one vector = [ j for j in reversed( vector ) ] return vector # Define the Table Model, instrumented to count certain calls from PyQt5.QtCore import Qt, QAbstractTableModel, pyqtSignal import natsort class Model( QAbstractTableModel ): tableSorted = pyqtSignal() def __init__ ( self, parent=None ) : super().__init__( parent ) clear_the_db() self.access_counts = [0, 0, 0] self.sort_order = Qt.AscendingOrder self.sort_column = 0 #self.sort_key_func = natsort.natsort_keygen( alg = (natsort.ns.LOCALE | natsort.ns.IGNORECASE ) ) self.sort_key_func = natsort.natsort_keygen( alg = (natsort.ns.LOCALE ) ) self.sort_vector = [] self.filter_func = None self.sort_time = 0.0 # TIMING def rowCount( self, index ) : global DATABASE if index.isValid() : return 0 return len(self.sort_vector) # 0 until Refresh is called def columnCount( self, index ) : if index.isValid() : return 0 return 3 def data(self, index, role ) : global VALUES_BY_ROW, DXX if role != Qt.DisplayRole : return None row = index.row() col = index.column() self.access_counts[col] += 1 sort_row = self.sort_vector[ row ] return VALUES_BY_ROW[ sort_row ][ col ] def clear_counts( self ) : self.access_counts = [0, 0, 0] def counts( self ) : return list( self.access_counts ) # method to let external code initiate a sort without knowing about # how we store the sort column and order. def do_sort( self ) : self.sort( self.sort_column, self.sort_order ) # reimplement QAbstractItemModel.sort() which receives two arguments, # the column number to sort, and the sort order, Qt.AscendingOrder # or Qt.DescendingOrder. Then emit layoutChanged to repaint the view. def sort( self, col, order ) : t0 = time.process_time() # TIMING self.sort_column = col self.sort_order = order self.sort_vector = get_column_vector( col, order, self.sort_key_func, self.filter_func ) self.layoutChanged.emit() self.sort_time = time.process_time() - t0 # TIMING self.tableSorted.emit() # TIMING # Override endResetModel() to make sure to refresh the sort # when there is new data. def endResetModel( self ) : super().endResetModel() self.sort( self.sort_column, self.sort_order ) # Define the Table View, which at this point is quite minimal. # The View is instantiated from MainWindow, which also connects # the model to it. from PyQt5.QtWidgets import QTableView class View( QTableView ): def __init__ ( self, parent=None ) : super().__init__( parent ) self.setSortingEnabled( True ) self.sortByColumn( 0, Qt.AscendingOrder ) # Define the main window which is the visual face of this app. from PyQt5.QtWidgets import ( QComboBox, QLabel, QMainWindow, QPushButton, QVBoxLayout, QHBoxLayout, QWidget ) class Main( QMainWindow ) : def __init__ ( self ) : super().__init__( ) self.times = [0, 0, 0] clear_the_db() # make sure to start with 0 rows self._uic() # all the layout stuff out of line self.refresh_button.clicked.connect( self.do_refresh ) self.table_model.tableSorted.connect( self.update_sort_time ) self.popup.activated.connect(self.set_filter) # Slot called when Refresh is clicked: # * clear the counts # * start model reset # * rebuild the DATABASE, timing it # * end the model reset, timing that # * update the labels displaying call counts and times def do_refresh( self ) : self.table_model.clear_counts() self.table_model.beginResetModel() self.times[0] = time.process_time() rebuild_the_db() self.times[1] = time.process_time() self.table_model.endResetModel() QApplication.processEvents() self.times[2] = time.process_time() self.update_labels() # Slot called when the popup combobox is activated. Input is # the index of the chosen entry, 0=All. Set the filter and # force a sort with the current column and order. filters = [ None, lambda w, c, f: f[0]=='X', lambda w, c, f: f[1]=='X', lambda w, c, f: f[2]=='X' ] def set_filter(self, choice) : self.table_model.filter_func = self.filters[ choice ] self.table_model.do_sort() # Slot called when a sort happens. Get the process time from # the table model and put it in the sort_time_label. def update_sort_time( self ) : self.sort_time_label.setText ('{:02.5f}'.format( self.table_model.sort_time ) ) self.row_count.setText(str(len(self.table_model.sort_vector)) ) def update_labels(self) : [c0, c1, c2] = self.table_model.counts() [t0, t1, t2] = self.times self.c0_label.setText( str(c0) ) self.c1_label.setText( str(c1) ) self.c2_label.setText( str(c2) ) self.td_label.setText( '{:02.5f}'.format(t1-t0) ) self.tr_label.setText( '{:02.5f}'.format(t2-t1) ) def _make_label( self, text='0' ) : # just make a right-aligned label out of line L = QLabel(text) L.setAlignment( Qt.AlignRight | Qt.AlignVCenter ) return L def _uic( self ) : # create the table self.table_view = View( parent=self ) self.table_model = Model( parent=self ) self.table_view.setModel( self.table_model ) # create the refresh button, put it in an hbox by with the sort time self.refresh_button = QPushButton( "Refresh" ) hb0 = QHBoxLayout() hb0.addWidget(self.refresh_button, 0) # create a combobox with 4 entries self.popup = QComboBox() self.popup.addItems( ['All','X..','.X.','..X'] ) hb0.addWidget( self.popup ) hb0.addStretch(1) hb0.addWidget( self._make_label( 'sort time:'), 0 ) self.sort_time_label = self._make_label() hb0.addWidget( self.sort_time_label ) hb0.addWidget( self._make_label( 'rows:'), 0 ) self.row_count = self._make_label() hb0.addWidget( self.row_count ) # create a set of labels to display counts of # entry to the model.data() method and times self.c0_label = self._make_label() # display role calls to column 0 self.c1_label = self._make_label() # 1 self.c2_label = self._make_label() # 2 self.tr_label = self._make_label() # time to reset the model self.td_label = self._make_label() # time to rebuild the db # build the row of call numbers hb1 = QHBoxLayout() hb1.addStretch(1) # push this row to the right hb1.addWidget( self._make_label( 'Display role calls col 0:' ) ) hb1.addWidget( self.c0_label ) hb1.addStretch(0) hb1.addWidget( self._make_label( 'col 1:' ) ) hb1.addWidget( self.c1_label ) hb1.addStretch(0) hb1.addWidget( self._make_label( 'col 2:' ) ) hb1.addWidget( self.c2_label ) # build the row of times hb2 = QHBoxLayout() hb2.addStretch(1) hb2.addWidget( self._make_label( 'Seconds to build DB:' ) ) hb2.addWidget( self.td_label ) hb2.addStretch(0) hb2.addWidget( self._make_label( 'to reset model:' ) ) hb2.addWidget( self.tr_label ) # stack up the central layout vb = QVBoxLayout() vb.addLayout( hb0, 0 ) vb.addWidget( self.table_view, 1 ) vb.addLayout( hb1, 0 ) vb.addLayout( hb2, 0 ) # put all that in a widget and make the widget our central layout wij = QWidget() wij.setLayout( vb ) wij.setMinimumSize( 500, 500 ) self.setCentralWidget( wij ) if __name__ == '__main__' : import sys print(sys.argv) try : NUMBER_OF_ROWS = int( sys.argv[1] ) except Exception as whatever : NUMBER_OF_ROWS = 10000 from PyQt5.QtWidgets import QApplication the_app = QApplication(sys.argv) main_window = Main() main_window.show() the_app.exec_()