Deadlock when running celery tests under pytest with xdist

Question

If I run without xdist involved, like this:

pytest --disable-warnings --verbose -s test_celery_chords.py

Works just fine. I see the DB created, the tasks run and it exits as expected.

If I run with xdist involved (-n 2), like this:

pytest --disable-warnings --verbose -n 2 -s test_celery_chords.py

I end up with a hung process (and sometimes these messages):

Destroying old test database for alias 'default'...
Chord callback '4c7664ce-89e0-475e-81a7-4973929d2256' raised: ValueError('4c7664ce-89e0-475e-81a7-4973929d2256')
Traceback (most recent call last):
  File "/Users/bob/.virtualenv/testme/lib/python3.10/site-packages/celery/backends/base.py", line 1019, in on_chord_part_return
    raise ValueError(gid)
ValueError: 4c7664ce-89e0-475e-81a7-4973929d2256
Chord callback '4c7664ce-89e0-475e-81a7-4973929d2256' raised: ValueError('4c7664ce-89e0-475e-81a7-4973929d2256')
Traceback (most recent call last):
  File "/Users/bob/.virtualenv/testme/lib/python3.10/site-packages/celery/backends/base.py", line 1019, in on_chord_part_return
    raise ValueError(gid)
ValueError: 4c7664ce-89e0-475e-81a7-4973929d2256
Chord callback '4c7664ce-89e0-475e-81a7-4973929d2256' raised: ValueError('4c7664ce-89e0-475e-81a7-4973929d2256')
Traceback (most recent call last):
  File "/Users/bob/.virtualenv/testme/lib/python3.10/site-packages/celery/backends/base.py", line 1019, in on_chord_part_return
    raise ValueError(gid)
ValueError: 4c7664ce-89e0-475e-81a7-4973929d2256
Chord callback '4c7664ce-89e0-475e-81a7-4973929d2256' raised: ValueError('4c7664ce-89e0-475e-81a7-4973929d2256')
Traceback (most recent call last):
  File "/Users/bob/.virtualenv/testme/lib/python3.10/site-packages/celery/backends/base.py", line 1019, in on_chord_part_return
    raise ValueError(gid)
ValueError: 4c7664ce-89e0-475e-81a7-4973929d2256
Chord callback '4c7664ce-89e0-475e-81a7-4973929d2256' raised: ValueError('4c7664ce-89e0-475e-81a7-4973929d2256')
Traceback (most recent call last):
  File "/Users/bob/.virtualenv/testme/lib/python3.10/site-packages/celery/backends/base.py", line 1019, in on_chord_part_return
    raise ValueError(gid)
ValueError: 4c7664ce-89e0-475e-81a7-4973929d2256

[gw0] ERROR test_celery_chords.py::test_chords Destroying test database for alias 'default'...

Only way to end it is with ^C

These are my two tests (essentially the same test). The DB isn't needed for these tasks (simple add and average example tests) but will be needed for the other Django tests that do use the DB.

def test_chords(transactional_db, celery_app, celery_worker, celery_not_eager):

    celery_app.config_from_object("django.conf:settings", namespace="CELERY")
    task = do_average.delay()
    results = task.get()
    assert task.state == "SUCCESS"
    assert len(results[0][1][1]) == 10


def test_chord_differently(transactional_db, celery_app, celery_worker, celery_not_eager):

    celery_app.config_from_object("django.conf:settings", namespace="CELERY")
    task = do_average.delay()
    results = task.get()
    assert task.state == "SUCCESS"
    assert len(results[0][1][1]) == 10

and the tasks (shouldn't matter)

@shared_task
def _add(x: int, y: int) -> int:
    print(f"{x} + {y} {time.time()}")
    return x + y


@shared_task
def _average(numbers: List[int]) -> float:
    print(f"AVERAGING {sum(numbers)} / {len(numbers)}")
    return sum(numbers) / len(numbers)


@shared_task
def do_average():
    tasks = [_add.s(i, i) for i in range(10)]
    print(f"Creating chord of {len(tasks)} tasks at {time.time()}")
    return chord(tasks)(_average.s())

using a conftest.py of this:

@pytest.fixture
def celery_not_eager(settings):
    settings.CELERY_TASK_ALWAYS_EAGER = False
    settings.CELERY_TASK_EAGER_PROPAGATES = False

pytest --fixtures

celery_app -- .../python3.10/site packages/celery/contrib/pytest.py:173
    Fixture creating a Celery application instance.

celery_worker -- .../python3.10/site-packages/celery/contrib/pytest.py:195
    Fixture: Start worker in a thread, stop it when the test returns.

Using

django=4.1.2
pytest-celery==0.0.0
pytest-cov==3.0.0
pytest-django==4.5.2
pytest-xdist==2.5.0

boatcoder · Answer 1 · 2022-11-07T14:22:29.357

While I have not solved this, I have found a workaround of sorts using @pytest.mark.xdist_group(name="celery") to decorate the test class and I can do the following:

@pytest.mark.xdist_group(name="celery")
@override_settings(CELERY_TASK_ALWAYS_EAGER=False)
@override_settings(CELERY_TASK_EAGER_PROPAGATES=False)
class SyncTaskTestCase2(TransactionTestCase):
    @classmethod
    def setUpClass(cls):
        super().setUpClass()
        cls.celery_worker = start_worker(app, perform_ping_check=False)
        cls.celery_worker.__enter__()
        print(f"Celery Worker started {time.time()}")

    @classmethod
    def tearDownClass(cls):
        print(f"Tearing down Superclass {time.time()}")
        super().tearDownClass()
        print(f"Tore down Superclass {time.time()}")
        cls.celery_worker.__exit__(None, None, None)
        print(f"Celery Worker torn down {time.time()}")

    def test_success(self):
        print(f"Starting test at {time.time()}")
        self.task = do_average_in_chord.delay()
        self.task.get()
        print(f"Finished Averaging at {time.time()}")
        assert self.task.successful()

This, combined with the command line option --dist loadgroup forces all of the "celery" group to be run on the same runner process which prevents the deadlock and allows --numprocesses 10 to run to completion.

The biggest drawback here is the 9 second penalty to teardown the celery worker which makes you prone to push all of your celery testing into one class.

# This accomplishes the same things as the unitest above WITHOUT having a Class wrapped around the tests it also eliminates the 9 second teardown wait.
@pytest.mark.xdist_group(name="celery")
@pytest.mark.django_db # Why do I need this and transactional_db???
def test_averaging_in_a_chord(
    transactional_db,
    celery_session_app,
    celery_session_worker,
    use_actual_celery_worker,
):
    task = do_average_in_chord.delay()
    task.get()
    assert task.successful()

You do need this in your conftest.py

from typing import Type
import time

import pytest
from pytest_django.fixtures import SettingsWrapper

from celery import Celery
from celery.contrib.testing.worker import start_worker


@pytest.fixture(scope="function")
def use_actual_celery_worker(settings: SettingsWrapper) -> SettingsWrapper:
    """Turns of CELERY_TASK_ALWAYS_EAGER and CELERY_TASK_EAGER_PROPAGATES for a single test. """
    settings.CELERY_TASK_ALWAYS_EAGER = False
    settings.CELERY_TASK_EAGER_PROPAGATES = False
    return settings


@pytest.fixture(scope="session")
def celery_session_worker(celery_session_app: Celery):
    """Re-implemented this so that my celery app gets used.  This keeps the priority queue stuff the same
    as it is in production.  If BROKER_BACKEND is set to "memory" then rabbit shouldn't be involved anyway."""
    celery_worker = start_worker(
        celery_session_app, perform_ping_check=False, shutdown_timeout=0.5
    )
    celery_worker.__enter__()
    yield celery_worker

    # This causes the worker to exit immediately so that we don't have a 9 second wait for the timeout.
    celery_session_app.control.shutdown()

    print(f"Tearing down Celery Worker {time.time()}")
    celery_worker.__exit__(None, None, None)
    print(f"Celery Worker torn down {time.time()}")


@pytest.fixture(scope="session")
def celery_session_app() -> Celery:

    from workshop.celery import app

    """ Get the app you would regularly use for celery tasks and return it.  This insures all of your default
    app options mirror what you use at runtime."""
    yield app

Deadlock when running celery tests under pytest with xdist

1 Answers1