Problem at hand
I a created a QAbstractTableModel to display a pandas dataframe which in turn is populated from a CSV file. In practice this application will be running in a CPython environment in a .NET application, but for now I am developing in PyCharm and running it there as well.
I need to process items row by row and update status while processing, problem is that the Status column is never updated.
The UI is not updating at all during the execution of the program, but it does when it is finished processing. When finished I see the expected results for the Status column update from None to their expected state Finished for all but row 1, 7 and 9, which report Error as expected in this case.
The issue reproduces both for the application in PyCharm during development as when ran in production environment (also for the minimal working example below).
I can see that the dataChanged signal is emitted properly from my logs.
Minimal example
Below I tried to create a minimal working example of my code (excluding the CSV parsing and excluding the real process, for which I tried to mimic so it should run on your end as well:
main.py
from enum import Enum
from pathlib import Path
from PySide6.QtWidgets import (
QApplication,
QMainWindow,
QPushButton,
QTableView,
)
from PySide6.QtCore import (
QAbstractTableModel,
QCoreApplication,
QFile,
QModelIndex,
QPersistentModelIndex,
QObject,
Qt,
)
from PySide6.QtUiTools import QUiLoader
from typing import Union, Any, Optional
# Make sure to import clr so we can import System.InvalidOperationException to catch (some) external application
# internal exceptions
import clr
from System import InvalidOperationException
import logging.handlers
import os
import pandas as pd
import socket
import sys
import time
import random
# Create a basic logger
logger = logging.getLogger(__name__)
logger.setLevel(logging.DEBUG)
# Console handler (RayStations Execution Details)
# Make sure to log to stdout instead of stderr to circumvent ERROR prefix in RayStation
c_handler = logging.StreamHandler(stream=sys.stdout)
c_handler.setLevel(logging.INFO)
c_format = logging.Formatter("%(asctime)s - %(levelname)s - %(message)s")
c_handler.setFormatter(c_format)
logger.addHandler(c_handler)
# Define some constants normally imported
COLUMNS = {
"CLINICAL_ID": "Clinical Id",
"RETAIN_SAFE_PRIVATE_ATTRIBUTES": "Retain Safe Private Attributes",
"RETAIN_UIDS": "Retain UIDs",
"RETAIN_INSITUTION_IDENTITY": "Retain Institution Identity",
"RETAIN_DEVICE_IDENTITY": "Retain Device Identity",
"RETAIN_DATES": "Retain Dates",
"ANONYMIZED_ID": "Anonymized Id",
"ANONYMIZED_NAME": "Anonymized Name",
}
COLUMN_TO_ANONYMIZATION_FIELD_MAPPING = {
COLUMNS["RETAIN_DATES"]: "RetainLongitudinalTemporalInformationFullDatesOption",
COLUMNS["RETAIN_DEVICE_IDENTITY"]: "RetainDeviceIdentityOption",
COLUMNS["RETAIN_INSITUTION_IDENTITY"]: "RetainInstitutionIdentityOption",
COLUMNS["RETAIN_UIDS"]: "RetainUIDs",
COLUMNS["RETAIN_SAFE_PRIVATE_ATTRIBUTES"]: "RetainSafePrivateOption",
}
class STATUS(Enum):
ERROR = "Error"
PROCESSING = "Processing"
SUCCESS = "Finished"
WARNING = "Warning"
class CopyToResearchModel(QAbstractTableModel):
def __init__(self, parent: Optional[QObject] = None) -> None:
super().__init__(parent)
self._data = None
# Wire up a slot to debug and show dataChanged is emitted properly
self.dataChanged.connect(self.debug_datachanged_slot)
def debug_datachanged_slot(self, first, last):
# Helper to show dataChanged signal is emitted
logger.info(f"dataChanged() emitted with ({first}, {last})")
def get_dataframe(self):
return self._data
def set_dataframe(self, dataframe: pd.DataFrame):
self._data = dataframe
# Mimic DefaultAnonymizationOptions return value
from dataclasses import dataclass
@dataclass
class DAO:
RetainDeviceIdentityOption: bool = True
RetainInstitutionIdentityOption: bool = False
RetainLongitudinalTemporalInformationFullDatesOption: bool = False
RetainLongitudinalTemporalInformationModifiedDatesOption: bool = False
RetainPatientCharacteristicsOption: bool = False
RetainSafePrivateOption: bool = False
RetainUIDs: bool = False
# Mimic behavior since connect.connect_cpython.PyScriptObject object does not have (default)
# __getattribute__ function, but uses __getattr__ instead. We rely on this function to retrieve
# attributes dynamically, since we can not reference them
# dynamically using dao.FieldName
def __getattr__(self, item):
return self.__getattribute__(item)
dao = DAO()
# Set defaults when not set:
for (column, field) in COLUMN_TO_ANONYMIZATION_FIELD_MAPPING.items():
self._data[column].fillna(
dao.__getattr__(field),
inplace=True,
)
# Add status column as first column
if not self._data.empty:
self._data.insert(loc=0, column="Status", value=None)
# Signal redraw is required
self.layoutChanged.emit()
def setData(
self,
index: Union[QModelIndex, QPersistentModelIndex],
value: Any,
role: int = Qt.EditRole | Qt.DisplayRole,
) -> bool:
row = index.row()
column = index.column()
if (
not index.isValid()
or role != Qt.EditRole
or not (0 <= row <= self.rowCount())
or not (0 <= column <= self.columnCount())
):
return False
self._data.iloc[row, column] = value
self.dataChanged.emit(index, index)
return True
def columnCount(
self,
parent: Union[QModelIndex, QPersistentModelIndex] = ...,
) -> int:
try:
return self._data.shape[1]
except:
pass
return 0
def data(
self,
index: Union[QModelIndex, QPersistentModelIndex],
role: Qt.ItemDataRole = ...,
) -> Any:
if index.isValid() and role == Qt.DisplayRole:
return str(self._data.iloc[index.row(), index.column()])
def headerData(
self, section: int, orientation: Qt.Orientation, role: int = ...
) -> Any:
if orientation == Qt.Horizontal and role == Qt.DisplayRole:
return self._data.columns[section]
def rowCount(
self,
parent: Union[QModelIndex, QPersistentModelIndex] = ...,
) -> int:
try:
return self._data.shape[0]
except:
pass
return 0
class CopyToResearch(QMainWindow):
def __init__(self) -> None:
super().__init__()
# Load form
self.load_ui()
# Find the QTbaleView and set it's model
self.view = self.findChild(QTableView, name="tableView")
self.model = CopyToResearchModel(self)
self.view.setModel(self.model)
# Make window appear on top of RayStation application window, otherwise it hides behind it
self.setWindowState(self.windowState() & ~Qt.WindowMinimized | Qt.WindowActive)
# Add actions to buttons
load_button = self.findChild(QPushButton, name="loadButton")
load_button.clicked.connect(self.load_file)
start_button = self.findChild(QPushButton, name="startButton")
start_button.clicked.connect(self.copy_patients)
def load_ui(self):
loader = QUiLoader()
path = os.path.join(os.fspath(Path(__file__).resolve().parent), "form.ui")
ui_file = QFile(path)
ui_file.open(QFile.ReadOnly)
loader.load(ui_file, self)
ui_file.close()
# Explicitly set window title, apparently the definition in ui file is not honoured
self.setWindowTitle(QCoreApplication.tr("Anonymize clinical patients"))
def load_file(self):
# Mimic loading from CSV file
count = 10
self.model.set_dataframe(
pd.DataFrame(
data=[
[
f"Clinical Id {i}",
f"Anonimyzed Id {i}",
f"Anonimyzed Name {i}",
random.choice([True, False]),
random.choice([True, False]),
random.choice([True, False]),
random.choice([True, False]),
random.choice([True, False]),
]
for i in range(1, count + 1)
],
columns=[
"Clinical Id",
"Anonymized Id",
"Anonymized Name",
"Retain Dates",
"Retain Device Identity",
"Retain Institution Identity",
"Retain UIDs",
"Retain Safe Private Attributes",
],
)
)
# Force resize to also fit column headers to fit
self.view.resizeColumnsToContents()
def copy_patients(self):
# Limit number of retries
# max_retries = 3
max_retries = 1
# Wait time (in seconds)
timeout = 1
df = self.model.get_dataframe()
for (row_index, row) in df.iterrows():
# Determine index into model to update status
index = self.model.index(
row_index,
df.columns.get_loc("Status"),
QModelIndex(),
)
# Extract patient_info object by querying the patient database
clinical_id = row[COLUMNS["CLINICAL_ID"]]
anonymized_id = row[COLUMNS["ANONYMIZED_ID"]]
logger.info(f"Copying patient {clinical_id} to {anonymized_id}")
status = STATUS.PROCESSING
self.model.setData(index, status.value)
# Seed counter for number of tries and maximum number of retries
tries_counter = 1
while tries_counter <= max_retries and not status == STATUS.SUCCESS:
# Progressive wait time
wait_time = timeout * tries_counter
try:
# Copy and anonymize patient
logger.info(f"Processing {clinical_id}")
# Insert random delay to mimic real process
time.sleep(random.randint(1, 3))
# Fire random exception(s) to mimic real process
if row_index % 6 == 0:
# Make patient 7 and every 6th thereafter fail
raise InvalidOperationException()
elif row_index % 8 == 0:
# Make patient 9 and every 8th thereafter fail
raise RuntimeError("Random runtime error")
else:
# Mimic success
pass
status = STATUS.SUCCESS
logger.info(f"Processed {clinical_id}")
except InvalidOperationException:
# Trap System.InvalidOperationException: RaySearch.CoreUtilities.SanityCheckException: Failed to resolve IndexService
if tries_counter < max_retries:
status = STATUS.WARNING
logger.warning(
f"Error transferring patient {clinical_id}, retry in {wait_time} seconds",
exc_info=True,
)
else:
status = STATUS.ERROR
logger.exception(
f"Error transferring patient {clinical_id}, maximum retries reached"
)
break
except (SystemExit, KeyboardInterrupt, SyntaxError):
raise
except Exception:
status = STATUS.ERROR
logger.exception(
f"Unexpected error transferring patient {clinical_id}"
)
finally:
self.model.setData(index, status.value)
self.view.resizeColumnsToContents()
if not status == STATUS.SUCCESS:
time.sleep(wait_time)
# Increase try counter
tries_counter += 1
logger.info(f"{status.value} processing patient {clinical_id}")
def main():
app = QApplication([])
widget = CopyToResearch()
widget.show()
sys.exit(app.exec())
if __name__ == "__main__":
main()
form.ui:
<?xml version="1.0" encoding="UTF-8"?>
<ui version="4.0">
<class>CopyToResearch</class>
<widget class="QWidget" name="CopyToResearch">
<property name="windowModality">
<enum>Qt::NonModal</enum>
</property>
<property name="geometry">
<rect>
<x>0</x>
<y>0</y>
<width>800</width>
<height>600</height>
</rect>
</property>
<property name="sizePolicy">
<sizepolicy hsizetype="MinimumExpanding" vsizetype="MinimumExpanding">
<horstretch>0</horstretch>
<verstretch>0</verstretch>
</sizepolicy>
</property>
<property name="windowTitle">
<string>Anonymize clinical patients</string>
</property>
<layout class="QVBoxLayout" name="verticalLayout">
<item alignment="Qt::AlignHCenter|Qt::AlignVCenter">
<widget class="QWidget" name="centralWidget" native="true">
<layout class="QVBoxLayout" name="verticalLayout_2">
<item>
<widget class="QPushButton" name="loadButton">
<property name="acceptDrops">
<bool>false</bool>
</property>
<property name="toolTip">
<string>Load data from file</string>
</property>
<property name="text">
<string>&Load from file...</string>
</property>
<property name="shortcut">
<string>Ctrl+L</string>
</property>
<property name="flat">
<bool>false</bool>
</property>
</widget>
</item>
<item>
<widget class="QTableView" name="tableView">
<property name="locale">
<locale language="English" country="Netherlands"/>
</property>
<property name="sizeAdjustPolicy">
<enum>QAbstractScrollArea::AdjustToContents</enum>
</property>
<property name="alternatingRowColors">
<bool>true</bool>
</property>
</widget>
</item>
<item>
<widget class="QPushButton" name="startButton">
<property name="toolTip">
<string>Start copying patients</string>
</property>
<property name="text">
<string>&Start</string>
</property>
<property name="shortcut">
<string>Ctrl+S</string>
</property>
</widget>
</item>
</layout>
</widget>
</item>
</layout>
</widget>
<resources/>
<connections/>
</ui>
Logs
2022-01-27 10:09:03,810 - INFO - Copying patient Clinical Id 1 to Anonimyzed Id 1
2022-01-27 10:09:03,811 - DEBUG - dataChanged() emitted with (<PySide6.QtCore.QModelIndex(0,0,0x0,CopyToResearchModel(0x2edec32a950)) at 0x000002EDED468F00>, <PySide6.QtCore.QModelIndex(0,0,0x0,CopyToResearchModel(0x2edec32a950)) at 0x000002EDED468F80>)
2022-01-27 10:09:03,811 - INFO - Processing Clinical Id 1
2022-01-27 10:09:04,837 - ERROR - Error transferring patient Clinical Id 1, maximum retries reached
Traceback (most recent call last):
File "C:\Users\jonathan.martens\Code\raystation-administration\copy_to_research\pyside6_qabstracttableview.py", line 298, in copy_patients
raise InvalidOperationException()
System.InvalidOperationException: Operation is not valid due to the current state of the object.
2022-01-27 10:09:04,839 - DEBUG - dataChanged() emitted with (<PySide6.QtCore.QModelIndex(0,0,0x0,CopyToResearchModel(0x2edec32a950)) at 0x000002EDF04F9040>, <PySide6.QtCore.QModelIndex(0,0,0x0,CopyToResearchModel(0x2edec32a950)) at 0x000002EDF04F90C0>)
2022-01-27 10:09:05,850 - INFO - Error processing patient Clinical Id 1
2022-01-27 10:09:05,851 - INFO - Copying patient Clinical Id 2 to Anonimyzed Id 2
2022-01-27 10:09:05,853 - DEBUG - dataChanged() emitted with (<PySide6.QtCore.QModelIndex(1,0,0x0,CopyToResearchModel(0x2edec32a950)) at 0x000002EDF04FBD80>, <PySide6.QtCore.QModelIndex(1,0,0x0,CopyToResearchModel(0x2edec32a950)) at 0x000002EDF04FBF80>)
2022-01-27 10:09:05,853 - INFO - Processing Clinical Id 2
2022-01-27 10:09:06,854 - INFO - Processed Clinical Id 2
2022-01-27 10:09:06,854 - DEBUG - dataChanged() emitted with (<PySide6.QtCore.QModelIndex(1,0,0x0,CopyToResearchModel(0x2edec32a950)) at 0x000002EDF04FBF80>, <PySide6.QtCore.QModelIndex(1,0,0x0,CopyToResearchModel(0x2edec32a950)) at 0x000002EDF04FC040>)
2022-01-27 10:09:06,863 - INFO - Finished processing patient Clinical Id 2
2022-01-27 10:09:06,863 - INFO - Copying patient Clinical Id 3 to Anonimyzed Id 3
2022-01-27 10:09:06,864 - DEBUG - dataChanged() emitted with (<PySide6.QtCore.QModelIndex(2,0,0x0,CopyToResearchModel(0x2edec32a950)) at 0x000002EDF04FED40>, <PySide6.QtCore.QModelIndex(2,0,0x0,CopyToResearchModel(0x2edec32a950)) at 0x000002EDF04FEEC0>)
2022-01-27 10:09:06,864 - INFO - Processing Clinical Id 3
2022-01-27 10:09:06,864 - INFO - Processed Clinical Id 3
2022-01-27 10:09:06,864 - DEBUG - dataChanged() emitted with (<PySide6.QtCore.QModelIndex(2,0,0x0,CopyToResearchModel(0x2edec32a950)) at 0x000002EDF04FEEC0>, <PySide6.QtCore.QModelIndex(2,0,0x0,CopyToResearchModel(0x2edec32a950)) at 0x000002EDF04FF040>)
2022-01-27 10:09:06,872 - INFO - Finished processing patient Clinical Id 3
2022-01-27 10:09:06,872 - INFO - Copying patient Clinical Id 4 to Anonimyzed Id 4
2022-01-27 10:09:06,872 - DEBUG - dataChanged() emitted with (<PySide6.QtCore.QModelIndex(3,0,0x0,CopyToResearchModel(0x2edec32a950)) at 0x000002EDF0501D40>, <PySide6.QtCore.QModelIndex(3,0,0x0,CopyToResearchModel(0x2edec32a950)) at 0x000002EDF0501EC0>)
2022-01-27 10:09:06,872 - INFO - Processing Clinical Id 4
2022-01-27 10:09:07,872 - INFO - Processed Clinical Id 4
2022-01-27 10:09:07,872 - DEBUG - dataChanged() emitted with (<PySide6.QtCore.QModelIndex(3,0,0x0,CopyToResearchModel(0x2edec32a950)) at 0x000002EDF0501EC0>, <PySide6.QtCore.QModelIndex(3,0,0x0,CopyToResearchModel(0x2edec32a950)) at 0x000002EDF0502040>)
2022-01-27 10:09:07,879 - INFO - Finished processing patient Clinical Id 4
2022-01-27 10:09:07,879 - INFO - Copying patient Clinical Id 5 to Anonimyzed Id 5
2022-01-27 10:09:07,880 - DEBUG - dataChanged() emitted with (<PySide6.QtCore.QModelIndex(4,0,0x0,CopyToResearchModel(0x2edec32a950)) at 0x000002EDF0504D00>, <PySide6.QtCore.QModelIndex(4,0,0x0,CopyToResearchModel(0x2edec32a950)) at 0x000002EDF0504E80>)
2022-01-27 10:09:07,880 - INFO - Processing Clinical Id 5
2022-01-27 10:09:07,880 - INFO - Processed Clinical Id 5
2022-01-27 10:09:07,880 - DEBUG - dataChanged() emitted with (<PySide6.QtCore.QModelIndex(4,0,0x0,CopyToResearchModel(0x2edec32a950)) at 0x000002EDF0504E80>, <PySide6.QtCore.QModelIndex(4,0,0x0,CopyToResearchModel(0x2edec32a950)) at 0x000002EDF0504F00>)
2022-01-27 10:09:07,887 - INFO - Finished processing patient Clinical Id 5
2022-01-27 10:09:07,887 - INFO - Copying patient Clinical Id 6 to Anonimyzed Id 6
2022-01-27 10:09:07,887 - DEBUG - dataChanged() emitted with (<PySide6.QtCore.QModelIndex(5,0,0x0,CopyToResearchModel(0x2edec32a950)) at 0x000002EDF0507CC0>, <PySide6.QtCore.QModelIndex(5,0,0x0,CopyToResearchModel(0x2edec32a950)) at 0x000002EDF0507E40>)
2022-01-27 10:09:07,887 - INFO - Processing Clinical Id 6
2022-01-27 10:09:08,888 - INFO - Processed Clinical Id 6
2022-01-27 10:09:08,888 - DEBUG - dataChanged() emitted with (<PySide6.QtCore.QModelIndex(5,0,0x0,CopyToResearchModel(0x2edec32a950)) at 0x000002EDF0507E40>, <PySide6.QtCore.QModelIndex(5,0,0x0,CopyToResearchModel(0x2edec32a950)) at 0x000002EDF0507FC0>)
2022-01-27 10:09:08,903 - INFO - Finished processing patient Clinical Id 6
2022-01-27 10:09:08,903 - INFO - Copying patient Clinical Id 7 to Anonimyzed Id 7
2022-01-27 10:09:08,903 - DEBUG - dataChanged() emitted with (<PySide6.QtCore.QModelIndex(6,0,0x0,CopyToResearchModel(0x2edec32a950)) at 0x000002EDF050AC80>, <PySide6.QtCore.QModelIndex(6,0,0x0,CopyToResearchModel(0x2edec32a950)) at 0x000002EDF050AE00>)
2022-01-27 10:09:08,903 - INFO - Processing Clinical Id 7
2022-01-27 10:09:09,904 - ERROR - Error transferring patient Clinical Id 7, maximum retries reached
Traceback (most recent call last):
File "C:\Users\jonathan.martens\Code\raystation-administration\copy_to_research\pyside6_qabstracttableview.py", line 298, in copy_patients
raise InvalidOperationException()
System.InvalidOperationException: Operation is not valid due to the current state of the object.
2022-01-27 10:09:09,904 - DEBUG - dataChanged() emitted with (<PySide6.QtCore.QModelIndex(6,0,0x0,CopyToResearchModel(0x2edec32a950)) at 0x000002EDF050AE00>, <PySide6.QtCore.QModelIndex(6,0,0x0,CopyToResearchModel(0x2edec32a950)) at 0x000002EDF050AE80>)
2022-01-27 10:09:10,912 - INFO - Error processing patient Clinical Id 7
2022-01-27 10:09:10,913 - INFO - Copying patient Clinical Id 8 to Anonimyzed Id 8
2022-01-27 10:09:10,914 - DEBUG - dataChanged() emitted with (<PySide6.QtCore.QModelIndex(7,0,0x0,CopyToResearchModel(0x2edec32a950)) at 0x000002EDF050DC80>, <PySide6.QtCore.QModelIndex(7,0,0x0,CopyToResearchModel(0x2edec32a950)) at 0x000002EDF050DE00>)
2022-01-27 10:09:10,914 - INFO - Processing Clinical Id 8
2022-01-27 10:09:11,915 - INFO - Processed Clinical Id 8
2022-01-27 10:09:11,916 - DEBUG - dataChanged() emitted with (<PySide6.QtCore.QModelIndex(7,0,0x0,CopyToResearchModel(0x2edec32a950)) at 0x000002EDF050DE00>, <PySide6.QtCore.QModelIndex(7,0,0x0,CopyToResearchModel(0x2edec32a950)) at 0x000002EDF050DFC0>)
2022-01-27 10:09:11,931 - INFO - Finished processing patient Clinical Id 8
2022-01-27 10:09:11,932 - INFO - Copying patient Clinical Id 9 to Anonimyzed Id 9
2022-01-27 10:09:11,932 - DEBUG - dataChanged() emitted with (<PySide6.QtCore.QModelIndex(8,0,0x0,CopyToResearchModel(0x2edec32a950)) at 0x000002EDF050AF80>, <PySide6.QtCore.QModelIndex(8,0,0x0,CopyToResearchModel(0x2edec32a950)) at 0x000002EDF0510100>)
2022-01-27 10:09:11,932 - INFO - Processing Clinical Id 9
2022-01-27 10:09:11,932 - ERROR - Unexpected error transferring patient Clinical Id 9
Traceback (most recent call last):
File "C:\Users\jonathan.martens\Code\raystation-administration\copy_to_research\pyside6_qabstracttableview.py", line 301, in copy_patients
raise RuntimeError("Random runtime error")
RuntimeError: Random runtime error
2022-01-27 10:09:11,933 - DEBUG - dataChanged() emitted with (<PySide6.QtCore.QModelIndex(8,0,0x0,CopyToResearchModel(0x2edec32a950)) at 0x000002EDF050AC80>, <PySide6.QtCore.QModelIndex(8,0,0x0,CopyToResearchModel(0x2edec32a950)) at 0x000002EDF050AF80>)
2022-01-27 10:09:12,944 - INFO - Error processing patient Clinical Id 9
2022-01-27 10:09:12,944 - INFO - Copying patient Clinical Id 10 to Anonimyzed Id 10
2022-01-27 10:09:12,946 - DEBUG - dataChanged() emitted with (<PySide6.QtCore.QModelIndex(9,0,0x0,CopyToResearchModel(0x2edec32a950)) at 0x000002EDF050AF80>, <PySide6.QtCore.QModelIndex(9,0,0x0,CopyToResearchModel(0x2edec32a950)) at 0x000002EDF0510200>)
2022-01-27 10:09:12,946 - INFO - Processing Clinical Id 10
2022-01-27 10:09:13,947 - INFO - Processed Clinical Id 10
2022-01-27 10:09:13,948 - DEBUG - dataChanged() emitted with (<PySide6.QtCore.QModelIndex(9,0,0x0,CopyToResearchModel(0x2edec32a950)) at 0x000002EDF050AF80>, <PySide6.QtCore.QModelIndex(9,0,0x0,CopyToResearchModel(0x2edec32a950)) at 0x000002EDF0507F40>)
2022-01-27 10:09:13,967 - INFO - Finished processing patient Clinical Id 10