In my project I have three threads (QThreads, pyqt-based with PySide 6), one for generating data, one for processing, and one for visualization/forwarding. As data processing typically is much slower than generating, the buffer for the incoming data fills up quite rapidly, which leads to a visible delay in output.
One approach for solving this issue I had in mind was to use a ring buffer (or similar). If the data is not processed on time, it will simply be erased and replaced with the most current data, until it is retrieved for processing.
Unfortunately, I was not able to find any sample implementation of such an approach in PyQt. For C++, there are mentions of a QRingBuffer and a QCircularBuffer, but apparently they weren't destined for public use either.
For initial testing I created the following demo-code, containing three classes responsible for generating, processing and "visualizing" data, and a main class to control all of them.
The data-generator is implemented as
#
# Created on Thu Aug 03 2023
#
# Copyright (c) 2023 Your Company
#
import time
from PySide6 import QtCore
class DataServer(QtCore.QObject):
"""_summary_"""
finished = QtCore.Signal()
started = QtCore.Signal()
data_signal = QtCore.Signal(object)
def __init__(self) -> None:
"""_summary_"""
super().__init__()
self.run_server: bool = True
self.server_counter: int = 0
@QtCore.Slot()
def stop_server(self) -> None:
"""_summary_"""
self.run_server = False
def send_data(self, data_to_send: list[int]) -> None:
"""_summary_"""
self.data_signal.emit(data_to_send)
@QtCore.Slot()
def run_server_thread(self) -> None:
"""_summary_"""
self.server_counter = 0
self.started.emit()
while self.run_server:
time.sleep(0.01)
data_to_send: list[int] = [self.server_counter, time.time_ns(), 0, 0, 0]
self.send_data(data_to_send=data_to_send)
self.server_counter += 1
self.run_server = True
self.finished.emit()
The processing-part is implemented as
#
# Created on Thu Aug 03 2023
#
# Copyright (c) 2023 Your Company
#
import time
from PySide6 import QtCore
class DataProcessor(QtCore.QObject):
"""_summary_"""
finished = QtCore.Signal()
started = QtCore.Signal()
data_signal = QtCore.Signal(object)
def __init__(self) -> None:
"""_summary_"""
super().__init__()
self.run_processor: bool = True
self.received_data: bool = False
self.data: list[int] = []
@QtCore.Slot(object)
def receive_data(self, data: list[int]) -> None:
"""_summary_"""
self.data = data
self.process_data()
self.send_data()
def process_data(self) -> None:
"""_summary_"""
self.data[2] = time.time_ns()
# time.sleep(1)
time.sleep(0.1)
self.data[3] = time.time_ns()
def send_data(self) -> None:
"""_summary_"""
self.data_signal.emit(self.data)
@QtCore.Slot()
def stop_processor(self) -> None:
"""_summary_"""
self.run_processor = False
@QtCore.Slot()
def run_data_processing(self) -> None:
"""_summary_"""
self.started.emit()
while self.run_processor:
time.sleep(0.001)
self.run_processor = True
self.finished.emit()
The client itself is implemented as
#
# Created on Thu Aug 03 2023
#
# Copyright (c) 2023 Your Company
#
import time
from PySide6 import QtCore
class DataClient(QtCore.QObject):
"""_summary_"""
finished = QtCore.Signal()
started = QtCore.Signal()
data_signal = QtCore.Signal(object)
def __init__(self) -> None:
"""_summary_"""
super().__init__()
self.run_client: bool = True
self.received_data: bool = False
self.data: list[int] = []
@QtCore.Slot()
def stop_client(self) -> None:
"""_summary_"""
self.run_client = False
@QtCore.Slot(object)
def receive_data(self, data: list[int]) -> None:
"""_summary_
Args:
data (list[int]): _description_
"""
self.data = data
self.data[-1] = time.time_ns()
self.print_delta_t()
def print_delta_t(self) -> None:
"""_summary_"""
for i in range(len(self.data) - 2):
print(
f"{self.data[0]}/{i + 2}: {((self.data[i + 2] - self.data[i + 1]) / 1e6):.2f} ms"
)
@QtCore.Slot()
def run_client_thread(self) -> None:
"""_summary_"""
self.started.emit()
while self.run_client:
time.sleep(0.001)
self.finished.emit()
and the main thread as
#
# Created on Thu Aug 03 2023
#
# Copyright (c) 2023 Your Company
#
import time
from PySide6 import QtCore
from qt_threaded_comm_test.communication_handler.server import DataServer
from qt_threaded_comm_test.communication_handler.client import DataClient
from qt_threaded_comm_test.communication_handler.processor import DataProcessor
class RingBuffer:
"""_summary_"""
buffer_used_bytes: int = 0
buffer_size: int = 8
buffer: list[int] = [0] * buffer_size
buffer_not_empty: QtCore.QWaitCondition = QtCore.QWaitCondition()
buffer_not_full: QtCore.QWaitCondition = QtCore.QWaitCondition()
buffer_mutex: QtCore.QMutex = QtCore.QMutex()
class CommunicationHandler(QtCore.QObject):
"""_summary_"""
stop_transmission: QtCore.Signal = QtCore.Signal()
def __init__(self) -> None:
"""_summary_"""
super().__init__()
self.local_server: DataServer = DataServer()
self.local_client: DataClient = DataClient()
self.local_processor: DataProcessor = DataProcessor()
self.server_thread = QtCore.QThread()
self.client_thread = QtCore.QThread()
self.processor_thread = QtCore.QThread()
def start_communication(self) -> None:
"""_summary_"""
self.local_server.moveToThread(self.server_thread)
self.local_client.moveToThread(self.client_thread)
self.local_processor.moveToThread(self.processor_thread)
self.local_server.finished.connect(
self.server_thread.quit, QtCore.Qt.DirectConnection
)
self.local_client.finished.connect(
self.client_thread.quit, QtCore.Qt.DirectConnection
)
self.local_processor.finished.connect(
self.processor_thread.quit, QtCore.Qt.DirectConnection
)
self.local_server.finished.connect(
self.server_thread.deleteLater, QtCore.Qt.DirectConnection
)
self.local_client.finished.connect(
self.client_thread.deleteLater, QtCore.Qt.DirectConnection
)
self.local_processor.finished.connect(
self.processor_thread.deleteLater, QtCore.Qt.DirectConnection
)
self.local_server.finished.connect(
lambda: print("Stopped server"), QtCore.Qt.DirectConnection
)
self.local_client.finished.connect(
lambda: print("Stopped client"), QtCore.Qt.DirectConnection
)
self.local_processor.finished.connect(
lambda: print("Stopped processor"), QtCore.Qt.DirectConnection
)
self.stop_transmission.connect(
self.local_server.stop_server, QtCore.Qt.DirectConnection
)
self.stop_transmission.connect(
self.local_client.stop_client, QtCore.Qt.DirectConnection
)
self.stop_transmission.connect(
self.local_processor.stop_processor, QtCore.Qt.DirectConnection
)
self.local_server.data_signal.connect(
self.local_processor.receive_data, QtCore.Qt.DirectConnection
)
self.local_processor.data_signal.connect(
self.local_client.receive_data, QtCore.Qt.DirectConnection
)
self.local_server.started.connect(lambda: print("Started server"))
self.local_client.started.connect(lambda: print("Started client"))
self.local_processor.started.connect(lambda: print("Started processor"))
self.server_thread.started.connect(self.local_server.run_server_thread)
self.client_thread.started.connect(self.local_client.run_client_thread)
self.processor_thread.started.connect(self.local_processor.run_data_processing)
self.server_thread.start()
self.client_thread.start()
self.processor_thread.start()
time.sleep(10)
self.stop_transmission.emit()
self.server_thread.wait()
self.client_thread.wait()
self.processor_thread.wait()
The code is executed via
#
# Created on Thu Aug 03 2023
#
# Copyright (c) 2023 Your Company
#
import sys
import os
from PySide6 import QtWidgets
from qt_threaded_comm_test.communication_handler.communication_handler import (
CommunicationHandler,
)
os.environ["PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION"] = "python"
basedir: str = os.path.dirname(__file__)
try:
from ctypes import windll
MYAPPID: str = "EMPA.Classification_Controller.1.0"
windll.shell32.SetCurrentProcessExplicitAppUserModelID(MYAPPID)
except ImportError:
pass
def except_hook(cls, exception, traceback) -> None:
"""_summary_
Args:
exception (_type_): _description_
traceback (_type_): _description_
"""
sys.__excepthook__(cls, exception, traceback)
if __name__ == "__main__":
sys.excepthook = except_hook
app: QtWidgets.QApplication = QtWidgets.QApplication([])
local_comm_handler = CommunicationHandler()
local_comm_handler.start_communication()
However, I currently do not know how and where to include a buffer between those threads, to avoid additional delays due to longer processing/visualization. How could I solve that issue? Or should I look at a completely different approach?
