Advertisement
Guest User

Untitled

a guest
Mar 21st, 2019
70
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 1.22 KB | None | 0 0
  1. from datetime import datetime, timedelta
  2. import time
  3. import random
  4. from airflow import DAG
  5. from airflow.operators.python_operator import PythonOperator
  6. from pprint import pprint
  7.  
  8.  
  9. default_args = {
  10. "owner": "airflow",
  11. "depends_on_past": False,
  12. "start_date": datetime(2019, 4, 1),
  13. "email": ["airflow@example.com"],
  14. "email_on_failure": False,
  15. "email_on_retry": False,
  16. "retries": 1,
  17. "retry_delay": timedelta(minutes=5)
  18. }
  19.  
  20. dag = DAG("01_basic_usage", default_args=default_args)
  21.  
  22.  
  23. def print_context(ds, **kwargs):
  24. pprint(kwargs)
  25. print(ds)
  26. return "Print to Log"
  27.  
  28.  
  29. print_task = PythonOperator(
  30. task_id="print_task",
  31. provide_context=True,
  32. python_callable=print_context,
  33. dag=dag,
  34. )
  35.  
  36.  
  37. def sleep(seconds):
  38. time.sleep(seconds)
  39.  
  40.  
  41. def make_sleep_task(task_name, dag):
  42. seconds = random.randint(1, 3)
  43. task = PythonOperator(
  44. task_id=task_name,
  45. python_callable=sleep,
  46. op_kwargs={"seconds": float(seconds) / 10},
  47. dag=dag,
  48. )
  49. return task
  50.  
  51.  
  52. # print_task > sleep_task
  53. first_sleep = make_sleep_task("first_sleep", dag)
  54. last_sleep = make_sleep_task("last_sleep", dag)
  55.  
  56. print_task.set_downstream(last_sleep)
  57.  
  58. # sleep_task > print_task
  59. first_sleep >> print_task
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement