Running an asyncio server from a pytest fixture

Question

I'm trying to run a server using asyncio in a pytest fixture

@pytest.fixture(autouse=True)
@pytest.mark.asyncio
async def start_endpoints(
    endpoint1: ServerEndpoint,
    endpoint2: ServerEndpoint
):
    pool = ThreadPoolExecutor(max_workers=2)
    loop = asyncio.get_running_loop()

    await loop.run_in_executor(pool, endpoint1.start)
    await loop.run_in_executor(pool, endpoint2.start)

The start method is like the following

async def start(self):
        try:

            server = await asyncio.start_server(self.handle_req, self.addr, self.port)
            addr = server.sockets[0].getsockname()
            print(f'{self.name}: serving on {addr}')

            async with server:
                await server.serve_forever()

Whereas the test prints this error once it tries to open a connection with the server

self = <_WindowsSelectorEventLoop running=False closed=False debug=False>
fut = <Future finished exception=ConnectionRefusedError(10061, "Connect call failed ('127.0.0.1', 9000)")>
sock = <socket.socket [closed] fd=-1, family=AddressFamily.AF_INET, type=SocketKind.SOCK_STREAM, proto=6>
address = ('127.0.0.1', 9000)

    def _sock_connect_cb(self, fut, sock, address):
        if fut.cancelled():
            return
    
        try:
            err = sock.getsockopt(socket.SOL_SOCKET, socket.SO_ERROR)
            if err != 0:
                # Jump to any except clause below.
>               raise OSError(err, f'Connect call failed {address}')
E               ConnectionRefusedError: [Errno 10061] Connect call failed ('127.0.0.1', 9000)

EDIT: The problem is that the event loop is closed right after so I tried to mark all my fixture with (scope="module") but now I get

ScopeMismatch: You tried to access the 'function' scoped fixture 'event_loop' with a 'module' scoped request object, involved factories
test\e2e\test_peer.py:380:  def start_endpoints

EDIT2:

So I added the event_loop fixture

@pytest.fixture(scope="module")
def event_loop():
    loop = asyncio.get_event_loop()
    yield loop
    loop.close()

that should override the default loop for each fixture using @pytest.mark.asyncio.

@pytest.fixture(autouse=True, scope="module")
@pytest.mark.asyncio
async def start_endpoints(
    event_loop,
    endpoint1: ServerEndpoint,
    endpoint2: ServerEndpoint
):
    pool = ThreadPoolExecutor(max_workers=2)

    await event_loop.run_in_executor(pool, endpoint1.start)
    await event_loop.run_in_executor(pool, endpoint2.start)

By debugging inside my test the event_loop is equal to the loop that I'm storing inside the ServerEndpoint (that is asyncio.get_running_loop()) but I'm still getting the ConnectionRefusedError

not quite sure why you'd want to do that. you can use an async http client like `httpx` to load a wsgi/asgi app and test calls on that. — LotB, Jun 19 '21 at 12:45
more info here https://www.python-httpx.org/advanced/#calling-into-python-web-apps . Just make that a fixture and you should be good to go for your tests. — LotB, Jun 19 '21 at 12:47

Ayaz Yakupov · Answer 1 · 2021-06-20T17:00:29.113

Probably it should works like this:

async def handle(reader, writer):
    data = await reader.read(100)

    message = data.decode()
    print(f"SERVER: Received {message!r}")
    writer.write(data)
    await writer.drain()
    print(f"SERVER: Sent: {message!r}")

    writer.close()
    print("SERVER: Closed the connection")

async def start():
    server = await asyncio.start_server(handle, host, port)

    addr = server.sockets[0].getsockname()
    print(f'Server is running on {addr[0:2]}')

    async with server:
        await server.serve_forever()


async def _async_wait_for_server(event_loop, host, port):
    while True:
        a_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        try:
            await event_loop.sock_connect(a_socket, (host, port))
            return
        except ConnectionRefusedError:
            await asyncio.sleep(0.01)

        finally:
            a_socket.close()

@pytest.fixture()
def server(event_loop, host, port):
    cancel_handle = asyncio.ensure_future(start(host, port), loop=event_loop)
    event_loop.run_until_complete(
        asyncio.wait_for(_async_wait_for_server(event_loop, host, port), 5.0)
    )

    try:
        yield
    finally:
        cancel_handle.cancel()

But I recommend you make functional tests in another way:

create docker image for server and image for tests
create docker-compose.yml file (use depends_on and healthcheck for tests container)
run tests after starting server

Running an asyncio server from a pytest fixture

1 Answers1