I'm trying use airflow on Docker. my_python.py file in dags directory like :
from airflow import DAG
from airflow.operators.python import PythonOperator
from datetime import datetime, timedelta
import argparse
import psycopg2
import csv
import os
import sys
from datetime import datetime
from google.cloud import bigquery
from google.oauth2 import service_account
def postgresql_database_connection(table_name, data_file):
...
def write_to_bigquery(dataset_name, table_name, data_file):
...
dag = DAG('my_python',
default_args=default_args,
schedule_interval='00-59/30 * * * *',
catchup=False,
max_active_runs=1)
task1 = PythonOperator(
task_id='table_database_connection',
python_callable=postgresql_database_connection,
op_args=[TABLE_NAME,DATA_FILE],
dag=dag)
task2 = PythonOperator(
task_id='table_write_to_bigquery',
python_callable=write_to_bigquery,
op_args=[DATASET_NAME,TABLE_NAME,DATA_FILE,args.env],
dag=dag)
task1 >> task2
My .dockerfile like :
FROM python:3.7
ARG AIRFLOW_USER_HOME=/usr/local/airflow
ENV AIRFLOW_HOME=${AIRFLOW_USER_HOME}
RUN ...
&& pip install apache-airflow==2.0.0 \
&& pip install psycopg2-binary \
&& pip install google-cloud \
&& pip install google-oauth
EXPOSE 8080 8793
USER airflow
WORKDIR ${AIRFLOW_USER_HOME}
I got an error on airflow webserver :
Folders path:
-dags: --my_python.py
-airflow-test.Dockerfile
-docker-compose.yaml
Where is my fault? Is it python or airflow versions problem, or my dockerfile problem?
