Advertisement
Guest User

Untitled

a guest
Jun 25th, 2019
77
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 1.24 KB | None | 0 0
  1. from datetime import datetime
  2. from airflow import DAG
  3. from airflow.operators.python_operator import PythonOperator
  4. from airflow.operators.dummy_operator import DummyOperator
  5.  
  6.  
  7. def create_dag(dag_id,
  8. schedule,
  9. dag_number,
  10. default_args):
  11.  
  12. def hello_world_py(*args):
  13. print('Hello World')
  14. print('This is DAG: {}'.format(str(dag_number)))
  15.  
  16. dag = DAG(dag_id,
  17. schedule_interval=schedule,
  18. default_args=default_args)
  19.  
  20. with dag:
  21.  
  22. d1 = DummyOperator(task_id="start")
  23.  
  24. for i in range(0, 10):
  25.  
  26. t1 = PythonOperator(
  27. task_id='hello_world_{0}'.format(i),
  28. python_callable=hello_world_py,
  29. dag_number=dag_number)
  30.  
  31. d1 >> t1
  32.  
  33. return dag
  34.  
  35.  
  36. # build a dag for each number in range(10)
  37. for n in range(1, 5):
  38. dag_id = 'hello_world_{}'.format(str(n))
  39.  
  40. default_args = {'owner': 'airflow',
  41. 'start_date': datetime(2019, 1, 1)
  42. }
  43.  
  44. schedule = '@daily'
  45.  
  46. dag_number = n
  47.  
  48. globals()[dag_id] = create_dag(dag_id,
  49. schedule,
  50. dag_number,
  51. default_args)
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement