0

I have following DAG below. My concern is about HttpSensor. My target is if the endpoint is not accessible for whatever reason i do not want want the next task to be reached get_data. Moreover i would like sensor to check enpoint continously and only if endpoint is reachable then go to get_data task. What i observed at the moment is get_data is reached even if sensor do not reach endpoint or conn is no defined, then get_data is anyway reached. How to solve that?

import os
import json
import datetime
import requests
from airflow.models import DAG
from airflow.operators.python import PythonOperator
from airflow.providers.sftp.operators.sftp import SFTPOperator
from airflow.providers.sftp.sensors.sftp import SFTPSensor
from airflow.utils.dates import days_ago
from airflow.models import Variable
from airflow.sensors.http_sensor import HttpSensor
from airflow.hooks.base_hook import BaseHook

def get_data():
    response = requests.get("https://exampleweb/dis")

    if (response.status_code == 200):
        print("The request was a success!")
        print(response.json()) # Return a string representation of the data payload
    elif (response.status_code == 404):
        print("Result not found!")

with DAG(
         dag_id='request_test',
         schedule_interval=None,
         start_date=days_ago(2)) as dag:

    task_is_api_active = HttpSensor(
        task_id='is_api_active',
        http_conn_id='someconn',
        endpoint='exact'
    )

    get_data = PythonOperator(task_id="get_data",
                                  python_callable=get_data)

    task_is_api_active >> get_data
Henry
  • 537
  • 1
  • 9
  • 22

0 Answers0