I am trying to test a code that uses multiprocessing.Queue. I tried using the additional_skip_names, calling multiprocessing.freeze_support() and multiprocessing.set_start_method('forkserver'), but nothing seems to work.
Here is a minimal example:
import pytest
import multiprocessing
from pyfakefs.fake_filesystem_unittest import Patcher
def f():
q = multiprocessing.Manager().Queue()
q.put(1)
return q.get()
@pytest.fixture
def fs():
with Patcher(additional_skip_names=['multiprocessing.connection']) as patcher:
yield patcher.fs
def test_f1(fs):
assert f() == 1
def test_f2():
assert f() == 1
resulting in
$ pytest -q test.py
FF [100%]
=================================================== FAILURES ====================================================
____________________________________________________ test_f1 ____________________________________________________
fs = <pyfakefs.fake_filesystem.FakeFilesystem object at 0x7f12acbeadd0>
def test_f1(fs):
> assert f() == 1
/tmp/test.py:18:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
/tmp/test.py:6: in f
q = multiprocessing.Manager().Queue()
/usr/lib/python3.10/multiprocessing/context.py:57: in Manager
m.start()
/usr/lib/python3.10/multiprocessing/managers.py:566: in start
self._address = reader.recv()
/usr/lib/python3.10/multiprocessing/connection.py:250: in recv
buf = self._recv_bytes()
/usr/lib/python3.10/multiprocessing/connection.py:414: in _recv_bytes
buf = self._recv(4)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <multiprocessing.connection.Connection object at 0x7f12acc40670>, size = 4
read = <bound method FakeOsModule.read of <pyfakefs.fake_filesystem.FakeOsModule object at 0x7f12acbeb190>>
def _recv(self, size, read=_read):
buf = io.BytesIO()
handle = self._handle
remaining = size
while remaining > 0:
chunk = read(handle, remaining)
n = len(chunk)
if n == 0:
if remaining == size:
> raise EOFError
E EOFError
/usr/lib/python3.10/multiprocessing/connection.py:383: EOFError
--------------------------------------------- Captured stderr call ----------------------------------------------
Process SyncManager-1:
Traceback (most recent call last):
File "/usr/lib/python3.10/multiprocessing/process.py", line 314, in _bootstrap
self.run()
File "/usr/lib/python3.10/multiprocessing/process.py", line 108, in run
self._target(*self._args, **self._kwargs)
File "/usr/lib/python3.10/multiprocessing/managers.py", line 591, in _run_server
server = cls._Server(registry, address, authkey, serializer)
File "/usr/lib/python3.10/multiprocessing/managers.py", line 156, in __init__
self.listener = Listener(address=address, backlog=16)
File "/usr/lib/python3.10/multiprocessing/connection.py", line 448, in __init__
self._listener = SocketListener(address, family, backlog)
File "/usr/lib/python3.10/multiprocessing/connection.py", line 591, in __init__
self._socket.bind(address)
FileNotFoundError: [Errno 2] No such file or directory
____________________________________________________ test_f2 ____________________________________________________
def test_f2():
> assert f() == 1
test.py:21:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
test.py:6: in f
q = multiprocessing.Manager().Queue()
/usr/lib/python3.10/multiprocessing/managers.py:723: in temp
token, exp = self._create(typeid, *args, **kwds)
/usr/lib/python3.10/multiprocessing/managers.py:606: in _create
conn = self._Client(self._address, authkey=self._authkey)
/usr/lib/python3.10/multiprocessing/connection.py:508: in Client
answer_challenge(c, authkey)
/usr/lib/python3.10/multiprocessing/connection.py:752: in answer_challenge
message = connection.recv_bytes(256) # reject large message
/usr/lib/python3.10/multiprocessing/connection.py:216: in recv_bytes
buf = self._recv_bytes(maxlength)
/usr/lib/python3.10/multiprocessing/connection.py:414: in _recv_bytes
buf = self._recv(4)
/usr/lib/python3.10/multiprocessing/connection.py:379: in _recv
chunk = read(handle, remaining)
/home/xxx/.venv/lib/python3.10/site-packages/pyfakefs/fake_filesystem.py:5218: in wrapped
return f(*args, **kwargs)
/home/xxx/.venv/lib/python3.10/site-packages/pyfakefs/fake_filesystem.py:4241: in read
file_handle = self.filesystem.get_open_file(fd)
/home/xxx/.venv/lib/python3.10/site-packages/pyfakefs/fake_filesystem.py:1578: in get_open_file
self.raise_os_error(errno.EBADF, str(file_des))
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <pyfakefs.fake_filesystem.FakeFilesystem object at 0x7f12acbeadd0>, err_no = 9, filename = '12'
winerror = None
def raise_os_error(
self,
err_no: int,
filename: Optional[AnyString] = None,
winerror: Optional[int] = None,
) -> NoReturn:
"""Raises OSError.
The error message is constructed from the given error code and shall
start with the error string issued in the real system.
Note: this is not true under Windows if winerror is given - in this
case a localized message specific to winerror will be shown in the
real file system.
Args:
err_no: A numeric error code from the C variable errno.
filename: The name of the affected file, if any.
winerror: Windows only - the specific Windows error code.
"""
message = os.strerror(err_no) + " in the fake filesystem"
if winerror is not None and sys.platform == "win32" and self.is_windows_fs:
raise OSError(err_no, message, filename, winerror)
> raise OSError(err_no, message, filename)
E OSError: [Errno 9] Bad file descriptor in the fake filesystem: '12'
/home/xxx/.venv/lib/python3.10/site-packages/pyfakefs/fake_filesystem.py:1149: OSError
============================================ short test summary info ============================================
FAILED test.py::test_f1 - EOFError
FAILED test.py::test_f2 - OSError: [Errno 9] Bad file descriptor in the fake filesystem: '12'
2 failed in 0.34s
Exception ignored in: <function _ConnectionBase.__del__ at 0x7f12acc80ca0>
Traceback (most recent call last):
File "/usr/lib/python3.10/multiprocessing/connection.py", line 132, in __del__
self._close()
File "/usr/lib/python3.10/multiprocessing/connection.py", line 361, in _close
_close(self._handle)
File "/home/xxx/.venv/lib/python3.10/site-packages/pyfakefs/fake_filesystem.py", line 5218, in wrapped
return f(*args, **kwargs)
File "/home/xxx/.venv/lib/python3.10/site-packages/pyfakefs/fake_filesystem.py", line 4224, in close
file_handle = self.filesystem.get_open_file(fd)
File "/home/xxx/.venv/lib/python3.10/site-packages/pyfakefs/fake_filesystem.py", line 1577, in get_open_file
return file_list[0]
IndexError: list index out of range
Exception ignored in: <function _ConnectionBase.__del__ at 0x7f12acc80ca0>
Traceback (most recent call last):
File "/usr/lib/python3.10/multiprocessing/connection.py", line 132, in __del__
File "/usr/lib/python3.10/multiprocessing/connection.py", line 361, in _close
File "/home/xxx/.venv/lib/python3.10/site-packages/pyfakefs/fake_filesystem.py", line 5218, in wrapped
File "/home/xxx/.venv/lib/python3.10/site-packages/pyfakefs/fake_filesystem.py", line 4224, in close
File "/home/xxx/.venv/lib/python3.10/site-packages/pyfakefs/fake_filesystem.py", line 1578, in get_open_file
File "/home/xxx/.venv/lib/python3.10/site-packages/pyfakefs/fake_filesystem.py", line 1149, in raise_os_error
OSError: [Errno 9] Bad file descriptor in the fake filesystem: '12'
