I am using Flask with Celery to add tasks to my event queue powered by redis. Currently, I have a flask route that creates a celery chord, and runs it on the worker. The header of this chord is a chain, and the body is a group of chain (a bunch of chains that run concurrently). The issue that I am facing is that when the header chain runs, the result is not passed down to the next job in the chain, causing a TypeError for a missing argument. I have ignore_result=False
set for all of the jobs, and that doesn't make a difference. This chain was working before I wrapped it in a chord.
Here are the jobs & a function that builds the chord:
def enqueue_tasks_for_policy():
chains = []
for question in questions:
chains.append(
chain(
job1.s(question),
job2.s(question),
job3.s(question)
)
)
full_chord = chord(
chain(
job1.s(),
job2.s(),
job3.s(),
job4.s()
),
group(*chains)
)
return full_chord
@shared_task(ignore_result=False)
def job1(source: dict[str, str]):
# doing stuff
return json.dumps(jsonable_encoder(data))
@shared_task(ignore_result=False)
def job2(result: str):
#doing other stuff but should have result
return json.dumps(jsonable_encoder(data))
... job 3/4
This is how I am setting up celery in my flask app
def celery_init_app(app: Flask) -> Celery:
class FlaskTask(Task):
def __call__(self, *args: object, **kwargs: object) -> object:
with app.app_context():
return self.run(*args, **kwargs)
celery_app = Celery(app.name, task_cls=FlaskTask)
celery_app.config_from_object({
"broker_url": app.config["CELERY_BROKER_URL"],
"result_backend": app.config["CELERY_RESULT_BACKEND"]
})
celery_app.set_default()
app.extensions["celery"] = celery_app
return celery_app
def create_app(config_class=Config):
app = Flask(__name__)
cors = CORS(app)
app.config.from_object(config_class)
celery_init_app(app)
app.register_blueprint(main)
app.register_blueprint(auth)
app.register_blueprint(errors)
app.register_blueprint(api, url_prefix='/api')
return app
I saw these important notes regarding chains: https://docs.celeryq.dev/en/stable/userguide/canvas.html#chord-important-notes But I'm not really sure what to make of it and would be very appreciative of some guidance. Thanks for your time