0

I'm trying to migrate from LocalExecutor to CeleryExecutor in Airflow 2.1.3 using Docker with Redis. I made separate containers for webserver, scheduler, worker, redis and database. Problem: tasks are queued but not executed.

docker-compose.yml:

version: "3.3"

services:
  redis:
    image: redis:6.0.9-alpine
    healthcheck:
    test: ["CMD", "redis-cli", "ping"]
    interval: 5s
    timeout: 30s
    retries: 50
    restart: always
    ports:
      - "6793:6793"

  database:
    image: postgres:12-alpine
    restart: always
    environment:
      POSTGRES_DB: airflow
      POSTGRES_USER: airflow
      POSTGRES_PASSWORD: airflow
    volumes:
      - airflow_database:/var/lib/postgresql/data/

  webserver:
    image: airflow:latest
    restart: always
    depends_on:
      - database
      - redis
    environment:
      AIRFLOW__CORE__SQL_ALCHEMY_CONN: postgresql+psycopg2://airflow:airflow@database/airflow
      GUNICORN_CMD_ARGS: --log-level WARNING
      EXECUTOR: Celery
    volumes:
      - airflow_logs:/var/log/airflow
      - airflow_data:/var/spool/airflow
      - ./airflow/dags:/usr/local/airflow/dags
      - ./airflow/plugins:/usr/local/airflow/plugins
    ports:
      - 8080:8080
      - 8888:8888
    command: webserver
    healthcheck:
      test: ["CMD-SHELL", "[ -f /usr/local/airflow/airflow-webserver.pid ]"]
      interval: 30s
      timeout: 30s
      retries: 3

  flower:
    image: airflow:latest
    restart: always
    depends_on:
      - redis
    environment:
      AIRFLOW__CORE__SQL_ALCHEMY_CONN: postgresql+psycopg2://airflow:airflow@database/airflow
      EXECUTOR: Celery
    ports:
      - "5555:5555"
    command: celery flower -b "redis://redis:6379/1"

  scheduler:
    image: airflow:latest
    restart: always
    depends_on:
      - webserver
    volumes:
      - airflow_logs:/var/log/airflow
      - airflow_data:/var/spool/airflow
      - ./airflow/dags:/usr/local/airflow/dags
      - ./airflow/plugins:/usr/local/airflow/plugins
    environment:
      AIRFLOW__CORE__SQL_ALCHEMY_CONN: postgresql+psycopg2://airflow:airflow@database/airflow
      EXECUTOR: Celery
    command: scheduler

  worker:
    image: airflow:latest
    restart: always
    depends_on:
      - scheduler
    volumes:
      - airflow_logs:/var/log/airflow
      - airflow_data:/var/spool/airflow
      - ./airflow/dags:/usr/local/airflow/dags
      - ./airflow/plugins:/usr/local/airflow/plugins
    environment:
      AIRFLOW__CORE__SQL_ALCHEMY_CONN: postgresql+psycopg2://airflow:airflow@database/airflow
      EXECUTOR: Celery
    healthcheck:
    test:
      - "CMD-SHELL"
      - 'celery --app airflow.executors.celery_executor.app inspect ping -d "celery@$${HOSTNAME}"'
    interval: 10s
    timeout: 10s
    retries: 5
    command: celery worker -b "redis://redis:6379/1" --result-backend "db+postgresql://airflow:airflow@database/airflow"

volumes:
  airflow_database:
  airflow_data:
  airflow_logs:
  staging_database:

Dockerfile, airflow.cfg, entrypoint.sh

All containers loaded normally. I tried to do celery_result_backend == broker_url == 'redis://redis:6379/1' but to no avail. The flower shows what the worker itself created, but the worker container doesn't show a single line of logs. I also tried to use the worker container separately - it did not help.

Flower

Stepan Pryn
  • 1
  • 1
  • Please trim your code to make it easier to find your problem. Follow these guidelines to create a [minimal reproducible example](https://stackoverflow.com/help/minimal-reproducible-example). – Community Mar 14 '22 at 18:14

1 Answers1

0

As I can see, there's an obvious port number inconsistency for Redis. '6793' above and '6379' down below

Freddie
  • 1
  • 1