I am developing an application using PyQt5 (5.7.1) with Python 3.5. I use a QTableView to display a long list of record (more than 10,000). I want to be able to sort and filter this list on several columns at the same time.
I tried using a QAbstractTableModel with a QSortFilterProxyModel, reimplementing QSortFilterProxyModel.filterAcceptsRow() to have multicolumn filtering (see this blog post: http://www.dayofthenewdan.com/2013/02/09/Qt_QSortFilterProxyModel.html). but as this method is called for every row, filtering is very slow when there are a large number of rows.
I thought using Pandas for filtering could improve performance. So I created the following PandasTableModel class, which can indeed perform multicolumn filtering very quickly even with a large number of rows, as well as sorting:
import pandas as pd
from PyQt5 import QtCore, QtWidgets
class PandasTableModel(QtCore.QAbstractTableModel):
def __init__(self, parent=None, *args):
super(PandasTableModel, self).__init__(parent, *args)
self._filters = {}
self._sortBy = []
self._sortDirection = []
self._dfSource = pd.DataFrame()
self._dfDisplay = pd.DataFrame()
def rowCount(self, parent=QtCore.QModelIndex()):
if parent.isValid():
return 0
return self._dfDisplay.shape[0]
def columnCount(self, parent=QtCore.QModelIndex()):
if parent.isValid():
return 0
return self._dfDisplay.shape[1]
def data(self, index, role):
if index.isValid() and role == QtCore.Qt.DisplayRole:
return QtCore.QVariant(self._dfDisplay.values[index.row()][index.column()])
return QtCore.QVariant()
def headerData(self, col, orientation=QtCore.Qt.Horizontal, role=QtCore.Qt.DisplayRole):
if orientation == QtCore.Qt.Horizontal and role == QtCore.Qt.DisplayRole:
return QtCore.QVariant(str(self._dfDisplay.columns[col]))
return QtCore.QVariant()
def setupModel(self, header, data):
self._dfSource = pd.DataFrame(data, columns=header)
self._sortBy = []
self._sortDirection = []
self.setFilters({})
def setFilters(self, filters):
self.modelAboutToBeReset.emit()
self._filters = filters
self.updateDisplay()
self.modelReset.emit()
def sort(self, col, order=QtCore.Qt.AscendingOrder):
#self.layoutAboutToBeChanged.emit()
column = self._dfDisplay.columns[col]
ascending = (order == QtCore.Qt.AscendingOrder)
if column in self._sortBy:
i = self._sortBy.index(column)
self._sortBy.pop(i)
self._sortDirection.pop(i)
self._sortBy.insert(0, column)
self._sortDirection.insert(0, ascending)
self.updateDisplay()
#self.layoutChanged.emit()
self.dataChanged.emit(QtCore.QModelIndex(), QtCore.QModelIndex())
def updateDisplay(self):
dfDisplay = self._dfSource.copy()
# Filtering
cond = pd.Series(True, index = dfDisplay.index)
for column, value in self._filters.items():
cond = cond & \
(dfDisplay[column].str.lower().str.find(str(value).lower()) >= 0)
dfDisplay = dfDisplay[cond]
# Sorting
if len(self._sortBy) != 0:
dfDisplay.sort_values(by=self._sortBy,
ascending=self._sortDirection,
inplace=True)
# Updating
self._dfDisplay = dfDisplay
This class replicates the behaviour of a QSortFilterProxyModel, except for one aspect. If an item in the table is selected in the QTableView, sorting the table will not affect the selection (e.g. if the first row is selected before sorting, the first row will still be selected after sorting, not the same one as before.
I think the problem is related to the signals which are emitted. For filtering, I used modelAboutToBeReset() and modelReset(), but these signals cancel selection in the QTableView, so they are not suited for sorting. I read there (How to update QAbstractTableModel and QTableView after sorting the data source?) that layoutAboutToBeChanged() and layoutChanged() should be emitted. However, QTableView doesn't update if I use these signals (I don't understand why actually). When emitting dataChanged() once sorting is completed, QTableView is updated, but with the behaviour described above (selection not updated).
You can test this model using the following example :
class Ui_TableFilteringDialog(object):
def setupUi(self, TableFilteringDialog):
TableFilteringDialog.setObjectName("TableFilteringDialog")
TableFilteringDialog.resize(400, 300)
self.verticalLayout = QtWidgets.QVBoxLayout(TableFilteringDialog)
self.verticalLayout.setObjectName("verticalLayout")
self.tableView = QtWidgets.QTableView(TableFilteringDialog)
self.tableView.setObjectName("tableView")
self.tableView.setSortingEnabled(True)
self.tableView.setSelectionBehavior(QtWidgets.QAbstractItemView.SelectRows)
self.verticalLayout.addWidget(self.tableView)
self.groupBox = QtWidgets.QGroupBox(TableFilteringDialog)
self.groupBox.setObjectName("groupBox")
self.verticalLayout_2 = QtWidgets.QVBoxLayout(self.groupBox)
self.verticalLayout_2.setObjectName("verticalLayout_2")
self.formLayout = QtWidgets.QFormLayout()
self.formLayout.setObjectName("formLayout")
self.column1Label = QtWidgets.QLabel(self.groupBox)
self.column1Label.setObjectName("column1Label")
self.formLayout.setWidget(0, QtWidgets.QFormLayout.LabelRole, self.column1Label)
self.column1Field = QtWidgets.QLineEdit(self.groupBox)
self.column1Field.setObjectName("column1Field")
self.formLayout.setWidget(0, QtWidgets.QFormLayout.FieldRole, self.column1Field)
self.column2Label = QtWidgets.QLabel(self.groupBox)
self.column2Label.setObjectName("column2Label")
self.formLayout.setWidget(1, QtWidgets.QFormLayout.LabelRole, self.column2Label)
self.column2Field = QtWidgets.QLineEdit(self.groupBox)
self.column2Field.setObjectName("column2Field")
self.formLayout.setWidget(1, QtWidgets.QFormLayout.FieldRole, self.column2Field)
self.verticalLayout_2.addLayout(self.formLayout)
self.verticalLayout.addWidget(self.groupBox)
self.retranslateUi(TableFilteringDialog)
QtCore.QMetaObject.connectSlotsByName(TableFilteringDialog)
def retranslateUi(self, TableFilteringDialog):
_translate = QtCore.QCoreApplication.translate
TableFilteringDialog.setWindowTitle(_translate("TableFilteringDialog", "Dialog"))
self.groupBox.setTitle(_translate("TableFilteringDialog", "Filters"))
self.column1Label.setText(_translate("TableFilteringDialog", "Name"))
self.column2Label.setText(_translate("TableFilteringDialog", "Occupation"))
class TableFilteringDialog(QtWidgets.QDialog):
def __init__(self, parent=None):
super(TableFilteringDialog, self).__init__(parent)
self.ui = Ui_TableFilteringDialog()
self.ui.setupUi(self)
self.tableModel = PandasTableModel()
header = ['Name', 'Occupation']
data = [
['Abe', 'President'],
['Angela', 'Chancelor'],
['Donald', 'President'],
['François', 'President'],
['Jinping', 'President'],
['Justin', 'Prime minister'],
['Theresa', 'Prime minister'],
['Vladimir', 'President'],
['Donald', 'Duck']
]
self.tableModel.setupModel(header, data)
self.ui.tableView.setModel(self.tableModel)
self.ui.column1Field.textEdited.connect(self.filtersEdited)
self.ui.column2Field.textEdited.connect(self.filtersEdited)
def filtersEdited(self):
filters = {}
values = [
self.ui.column1Field.text().lower(),
self.ui.column2Field.text().lower()
]
for col, value in enumerate(values):
if value == '':
continue
column = self.tableModel.headerData(col, QtCore.Qt.Horizontal, QtCore.Qt.DisplayRole).value()
filters[column]=value
self.tableModel.setFilters(filters)
if __name__ == '__main__':
import sys
app = QtWidgets.QApplication(sys.argv)
dialog = TableFilteringDialog()
dialog.show()
sys.exit(app.exec_())
How can I make the selection follow the selected element when sorting ?
