I'm learning Apache Airflow from course of Marc Lamberti, I stumbled on a problem, in my task "process_user"
def _process_user(ti):
user = ti.xcom_pull(task_id = "extract_user")
user = user['results'][0]
processed_user = json_normalize({
'first_name': user['name']['first'],
'last_name': user['name']['last'],
'country': user['location']['country'],
'username': user['login']['username'],
'password': user['login']['password'],
'email': user['email']})
processed_user.to_csv('/tmp/processed_user.csv', index=None, header=False)
...
with DAG(dag_id='user_processing',
start_date=datetime(2022, 1, 1),
schedule_interval='@daily',
catchup=False) as dag:
...
extract_user = SimpleHttpOperator(
task_id='extract_user',
http_conn_id='user_api',
endpoint='api/',
method='GET',
response_filter = lambda response:
json.loads(response.text),
log_response = True
)
process_user = PythonOperator(
task_id='process_user',
python_callable = _process_user
)
...
I keep getting this error:
TypeError: xcom_pull() got an unexpected keyword argument 'task_id'
where did I do wrong ?
I found solution:
I used this code:
def _process_user(**kwargs):
user = kwargs['ti'].xcom_pull(task_ids="extract_user")