Task level failures are not triggered, in fact the dag level is repeated with alert TWICE

Question

from datetime import timedelta
from datetime import datetime
from airflow import DAG
from airflow.decorators import task, dag
from airflow.providers.ssh.operators.ssh import SSHOperator
from airflow.operators.bash_operator import BashOperator
from airflow.operators.python_operator import PythonOperator
import airflow
import sending_mail

# Task-level callback for sending mail on task success and failure
def on_trigger(context):
    task_instance = context.get("task_instance")
    state = task_instance.state
    task_id = task_instance.task_id
    dag_id = task_instance.dag_id
    run_id = task_instance.run_id
    start_date = task_instance.start_date
    end_date = task_instance.end_date
    log_url = task_instance.log_url

    SUBJECT = f"Airflow {state.upper()} alert for Task {task_id} in DAG {dag_id}"
    MESSAGE = f"""
        <b>STATE:</b> {state} <BR>
        <b>DAG:</b> {dag_id} <BR>
        <b>RUN ID:</b> {run_id} <BR>
        <b>TASK:</b> {task_id} <BR>
        <b>START TIME:</b> {start_date} <BR>
        <b>END TIME:</b> {end_date} <BR>
        <b>LOG URL:</b> {log_url} <BR>
    """
    sending_mail.sending(MESSAGE, SUBJECT)

# on_success_dag definition for overall DAG, which would be called post all tasks are completed only when all are successful
def on_success_dag(context):
    dag = context.get("task_instance").dag_id
    SUBJECT = "Airflow success alert for DAG - " + dag
    MESSAGE = """
        <b>SUCCESSFULLY COMPLETED THE DAG:</b> {dag} <BR>
        <b>DURATION:</b> {time} <BR>
        <b>RUN ID:</b> {run_id} <BR>
        """.format(
            dag=context.get("task_instance").dag_id,
            time=context.get("task_instance").duration,
            run_id=context["dag_run"].run_id,
        )
    sending_mail.sending(MESSAGE, SUBJECT)

# on_failure_dag definition for overall DAG, which would be called post all tasks are completed only EVEN any 1 of tasks failure
def on_failure_dag(context):
    dag = context.get("task_instance").dag_id
    SUBJECT = "Airflow failure alert for DAG - " + dag

    failed_task_ids = []
    dag_run = context.get("dag_run")
    for task in dag_run.get_task_instances(state="failed"):
        failed_task_ids.append(task.task_id)

    MESSAGE = """
        <b>FAILURE OF THE DAG:</b> {dag} <BR>
        <b>DURATION:</b> {time} <BR>
        <b>FOLLOWING TASK/s FAILED IN THE DAG:</b> {failed_task_ids} <BR>
        <b>EXECUTION DATE:</b> {execution_date} <BR>
        <b>LOG URL:</b> {log_url} <BR>
    """.format(
        dag=dag,
        time=context.get("task_instance").duration,
        failed_task_ids=failed_task_ids,
        execution_date=context["execution_date"],
        log_url=context.get("task_instance"),
    )
    sending_mail.sending(MESSAGE, SUBJECT)

# Default DAG syntax and its arguments
with DAG(
    dag_id="dag_id_name",
    schedule_interval="0 * * * *",
    max_active_runs=1,
    default_args={
        "start_date": datetime(2023, 6, 24),
        "retries": 1,
        "retry_delay": timedelta(minutes=5),
        "catchup": False,
    },
    on_success_callback=on_success_dag,
    on_failure_callback=on_failure_dag,
    render_template_as_native_obj=True,
    tags=["production"],
) as dag:

# Executes the container
task1 = SSHOperator(
task_id="oprt1", 
ssh_conn_id='connection1', 
command='sh -x script.sh ', 
dag=dag,
on_success_callback=on_trigger,
on_failure_callback=on_trigger,
)

task1

As per the above code there should be FAILURE or SUCCESS at TASK_LEVEL and even DAG_LEVEL. However the success level works very well, but when it comes to the failed level the dag_level is triggered TWICE, but the task_levels is not triggered at all for the failed level

As even tried setting this within dag definition like this, but same result
 # Default DAG syntax and its arguments
    with DAG(
        dag_id="dag_id_name",
        schedule_interval="0 * * * *",
        max_active_runs=1,
        default_args={
            "start_date": datetime(2023, 6, 24),
            "retries": 1,
            "retry_delay": timedelta(minutes=5),
            "on_failure_callback": on_trigger,
            "on_success_callback": on_trigger,
            "catchup": False,
        },
        on_success_callback=on_success_dag,
        on_failure_callback=on_failure_dag,
        render_template_as_native_obj=True,
        tags=["production"],
    ) as dag:

How can I get the alerts for even task level and dag level even for the failure

Task level failures are not triggered, in fact the dag level is repeated with alert TWICE

0 Answers0