SHARE
TWEET

Untitled

a guest Jun 25th, 2019 42 Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
  1. from datetime import datetime
  2. from airflow import DAG
  3. from airflow.operators.python_operator import PythonOperator
  4. from airflow.operators.dummy_operator import DummyOperator
  5.  
  6.  
  7. def create_dag(dag_id,
  8.                schedule,
  9.                dag_number,
  10.                default_args):
  11.  
  12.     def hello_world_py(*args):
  13.         print('Hello World')
  14.         print('This is DAG: {}'.format(str(dag_number)))
  15.  
  16.     dag = DAG(dag_id,
  17.               schedule_interval=schedule,
  18.               default_args=default_args)
  19.  
  20.     with dag:
  21.  
  22.         d1 = DummyOperator(task_id="start")
  23.  
  24.         for i in range(0, 10):
  25.  
  26.             t1 = PythonOperator(
  27.                 task_id='hello_world_{0}'.format(i),
  28.                 python_callable=hello_world_py,
  29.                 dag_number=dag_number)
  30.  
  31.             d1 >> t1
  32.  
  33.     return dag
  34.  
  35.  
  36. # build a dag for each number in range(10)
  37. for n in range(1, 5):
  38.     dag_id = 'hello_world_{}'.format(str(n))
  39.  
  40.     default_args = {'owner': 'airflow',
  41.                     'start_date': datetime(2019, 1, 1)
  42.                     }
  43.  
  44.     schedule = '@daily'
  45.  
  46.     dag_number = n
  47.  
  48.     globals()[dag_id] = create_dag(dag_id,
  49.                                    schedule,
  50.                                    dag_number,
  51.                                    default_args)
RAW Paste Data
We use cookies for various purposes including analytics. By continuing to use Pastebin, you agree to our use of cookies as described in the Cookies Policy. OK, I Understand
 
Top