Guest User

Airflow-Scheduler-Error-Logs

a guest
Sep 17th, 2018
106
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
  1. IntegrityError: (psycopg2.IntegrityError) duplicate key value violates unique constraint "task_instance_pkey"
  2. DETAIL: Key (task_id, dag_id, execution_date)=(PB_BPNZ, master_v2, 2018-09-12 15:18:00) already exists.
  3. [SQL: 'INSERT INTO task_instance (task_id, dag_id, execution_date, start_date, end_date, duration, state, try_number, hostname, unixname, job_id, pool, queue, priority_weight, operator, queued_dttm, pid) VALUES (%(task_id)s, %(dag_id)s, %(execution_date)s, %(start_date)s, %(end_date)s, %(duration)s, %(state)s, %(try_number)s, %(hostname)s, %(unixname)s, %(job_id)s, %(pool)s, %(queue)s, %(priority_weight)s, %(operator)s, %(queued_dttm)s, %(pid)s)'] [parameters: ({'unixname': 'infoworksuser', 'task_id': 'PB_BPNZ', 'end_date': None, 'pid': None, 'execution_date': datetime.datetime(2018, 9, 12, 15, 18), 'queued_dttm': None, 'job_id': None, 'try_number': 0, 'queue': 'default', 'operator': None, 'state': None, 'start_date': None, 'duration': None, 'priority_weight': 65, 'hostname': u'', 'pool': None, 'dag_id': 'master_v2'}, {'unixname': 'infoworksuser', 'task_id': 'PB_316G', 'end_date': None, 'pid': None, 'execution_date': datetime.datetime(2018, 9, 12, 15, 18), 'queued_dttm': None, 'job_id': None, 'try_number': 0, 'queue': 'default', 'operator': None, 'state': None, 'start_date': None, 'duration': None, 'priority_weight': 65, 'hostname': u'', 'pool': None, 'dag_id': 'master_v2'}, {'unixname': 'infoworksuser', 'task_id': 'BS_F9T4', 'end_date': None, 'pid': None, 'execution_date': datetime.datetime(2018, 9, 12, 15, 18), 'queued_dttm': None, 'job_id': None, 'try_number': 0, 'queue': 'default', 'operator': None, 'state': None, 'start_date': None, 'duration': None, 'priority_weight': 2, 'hostname': u'', 'pool': None, 'dag_id': 'master_v2'}, {'unixname': 'infoworksuser', 'task_id': 'BS_FCZS', 'end_date': None, 'pid': None, 'execution_date': datetime.datetime(2018, 9, 12, 15, 18), 'queued_dttm': None, 'job_id': None, 'try_number': 0, 'queue': 'default', 'operator': None, 'state': None, 'start_date': None, 'duration': None, 'priority_weight': 62, 'hostname': u'', 'pool': None, 'dag_id': 'master_v2'}, {'unixname': 'infoworksuser', 'task_id': 'BS_XGF7', 'end_date': None, 'pid': None, 'execution_date': datetime.datetime(2018, 9, 12, 15, 18), 'queued_dttm': None, 'job_id': None, 'try_number': 0, 'queue': 'default', 'operator': None, 'state': None, 'start_date': None, 'duration': None, 'priority_weight': 64, 'hostname': u'', 'pool': None, 'dag_id': 'master_v2'}, {'unixname': 'infoworksuser', 'task_id': 'BS_QJIU', 'end_date': None, 'pid': None, 'execution_date': datetime.datetime(2018, 9, 12, 15, 18), 'queued_dttm': None, 'job_id': None, 'try_number': 0, 'queue': 'default', 'operator': None, 'state': None, 'start_date': None, 'duration': None, 'priority_weight': 2, 'hostname': u'', 'pool': None, 'dag_id': 'master_v2'}, {'unixname': 'infoworksuser', 'task_id': 'BS_CLY3', 'end_date': None, 'pid': None, 'execution_date': datetime.datetime(2018, 9, 12, 15, 18), 'queued_dttm': None, 'job_id': None, 'try_number': 0, 'queue': 'default', 'operator': None, 'state': None, 'start_date': None, 'duration': None, 'priority_weight': 2, 'hostname': u'', 'pool': None, 'dag_id': 'master_v2'}, {'unixname': 'infoworksuser', 'task_id': 'BS_NZEA', 'end_date': None, 'pid': None, 'execution_date': datetime.datetime(2018, 9, 12, 15, 18), 'queued_dttm': None, 'job_id': None, 'try_number': 0, 'queue': 'default', 'operator': None, 'state': None, 'start_date': None, 'duration': None, 'priority_weight': 61, 'hostname': u'', 'pool': None, 'dag_id': 'master_v2'} ... displaying 10 of 242 total bound parameter sets ... {'unixname': 'infoworksuser', 'task_id': 'BS_BRX2', 'end_date': None, 'pid': None, 'execution_date': datetime.datetime(2018, 9, 12, 15, 18), 'queued_dttm': None, 'job_id': None, 'try_number': 0, 'queue': 'default', 'operator': None, 'state': None, 'start_date': None, 'duration': None, 'priority_weight': 62, 'hostname': u'', 'pool': None, 'dag_id': 'master_v2'}, {'unixname': 'infoworksuser', 'task_id': 'DM_DAW4', 'end_date': None, 'pid': None, 'execution_date': datetime.datetime(2018, 9, 12, 15, 18), 'queued_dttm': None, 'job_id': None, 'try_number': 0, 'queue': 'default', 'operator': None, 'state': None, 'start_date': None, 'duration': None, 'priority_weight': 241, 'hostname': u'', 'pool': None, 'dag_id': 'master_v2'})]
  4. Process DagFileProcessor4584467-Process:
  5. Traceback (most recent call last):
  6. File "/home/ec2-user/resources/python27/lib/python2.7/multiprocessing/process.py", line 258, in _bootstrap
  7. self.run()
  8. File "/home/ec2-user/resources/python27/lib/python2.7/multiprocessing/process.py", line 114, in run
  9. self._target(*self._args, **self._kwargs)
  10. File "/home/ec2-user/resources/python27/lib/python2.7/site-packages/airflow/jobs.py", line 347, in helper
  11. pickle_dags)
  12. File "/home/ec2-user/resources/python27/lib/python2.7/site-packages/airflow/utils/db.py", line 53, in wrapper
  13. result = func(*args, **kwargs)
  14. File "/home/ec2-user/resources/python27/lib/python2.7/site-packages/airflow/jobs.py", line 1584, in process_file
  15. self._process_dags(dagbag, dags, ti_keys_to_schedule)
  16. File "/home/ec2-user/resources/python27/lib/python2.7/site-packages/airflow/jobs.py", line 1176, in _process_dags
  17. self._process_task_instances(dag, tis_out)
  18. File "/home/ec2-user/resources/python27/lib/python2.7/site-packages/airflow/jobs.py", line 879, in _process_task_instances
  19. run.verify_integrity(session=session)
  20. File "/home/ec2-user/resources/python27/lib/python2.7/site-packages/airflow/utils/db.py", line 53, in wrapper
  21. result = func(*args, **kwargs)
  22. File "/home/ec2-user/resources/python27/lib/python2.7/site-packages/airflow/models.py", line 4298, in verify_integrity
  23. session.commit()
  24. File "/home/ec2-user/resources/python27/lib/python2.7/site-packages/sqlalchemy/orm/session.py", line 874, in commit
  25. self.transaction.commit()
  26. File "/home/ec2-user/resources/python27/lib/python2.7/site-packages/sqlalchemy/orm/session.py", line 461, in commit
  27. self._prepare_impl()
  28. File "/home/ec2-user/resources/python27/lib/python2.7/site-packages/sqlalchemy/orm/session.py", line 441, in _prepare_impl
  29. self.session.flush()
  30. File "/home/ec2-user/resources/python27/lib/python2.7/site-packages/sqlalchemy/orm/session.py", line 2137, in flush
  31. self._flush(objects)
  32. File "/home/ec2-user/resources/python27/lib/python2.7/site-packages/sqlalchemy/orm/session.py", line 2257, in _flush
  33. transaction.rollback(_capture_exception=True)
  34. File "/home/ec2-user/resources/python27/lib/python2.7/site-packages/sqlalchemy/util/langhelpers.py", line 60, in __exit__
  35. compat.reraise(exc_type, exc_value, exc_tb)
  36. File "/home/ec2-user/resources/python27/lib/python2.7/site-packages/sqlalchemy/orm/session.py", line 2221, in _flush
  37. flush_context.execute()
  38. File "/home/ec2-user/resources/python27/lib/python2.7/site-packages/sqlalchemy/orm/unitofwork.py", line 389, in execute
  39. rec.execute(self)
  40. File "/home/ec2-user/resources/python27/lib/python2.7/site-packages/sqlalchemy/orm/unitofwork.py", line 548, in execute
  41. uow
  42. File "/home/ec2-user/resources/python27/lib/python2.7/site-packages/sqlalchemy/orm/persistence.py", line 181, in save_obj
  43. mapper, table, insert)
  44. File "/home/ec2-user/resources/python27/lib/python2.7/site-packages/sqlalchemy/orm/persistence.py", line 799, in _emit_insert_statements
  45. execute(statement, multiparams)
  46. File "/home/ec2-user/resources/python27/lib/python2.7/site-packages/sqlalchemy/engine/base.py", line 945, in execute
  47. return meth(self, multiparams, params)
  48. File "/home/ec2-user/resources/python27/lib/python2.7/site-packages/sqlalchemy/sql/elements.py", line 263, in _execute_on_connection
  49. return connection._execute_clauseelement(self, multiparams, params)
  50. File "/home/ec2-user/resources/python27/lib/python2.7/site-packages/sqlalchemy/engine/base.py", line 1053, in _execute_clauseelement
  51. compiled_sql, distilled_params
  52. File "/home/ec2-user/resources/python27/lib/python2.7/site-packages/sqlalchemy/engine/base.py", line 1189, in _execute_context
  53. context)
  54. File "/home/ec2-user/resources/python27/lib/python2.7/site-packages/sqlalchemy/engine/base.py", line 1393, in _handle_dbapi_exception
  55. exc_info
  56. File "/home/ec2-user/resources/python27/lib/python2.7/site-packages/sqlalchemy/util/compat.py", line 202, in raise_from_cause
  57. reraise(type(exception), exception, tb=exc_tb, cause=cause)
  58. File "/home/ec2-user/resources/python27/lib/python2.7/site-packages/sqlalchemy/engine/base.py", line 1182, in _execute_context
  59. context)
  60. File "/home/ec2-user/resources/python27/lib/python2.7/site-packages/sqlalchemy/engine/default.py", line 469, in do_execute
  61. cursor.execute(statement, parameters)
  62. IntegrityError: (psycopg2.IntegrityError) duplicate key value violates unique constraint "task_instance_pkey"
  63. DETAIL: Key (task_id, dag_id, execution_date)=(SI_PS3H, cdc, 2018-09-12 15:25:21) already exists.
  64. [SQL: 'INSERT INTO task_instance (task_id, dag_id, execution_date, start_date, end_date, duration, state, try_number, hostname, unixname, job_id, pool, queue, priority_weight, operator, queued_dttm, pid) VALUES (%(task_id)s, %(dag_id)s, %(execution_date)s, %(start_date)s, %(end_date)s, %(duration)s, %(state)s, %(try_number)s, %(hostname)s, %(unixname)s, %(job_id)s, %(pool)s, %(queue)s, %(priority_weight)s, %(operator)s, %(queued_dttm)s, %(pid)s)'] [parameters: {'unixname': 'infoworksuser', 'end_date': None, 'job_id': None, 'task_id': 'SI_PS3H', 'pid': None, 'execution_date': datetime.datetime(2018, 9, 12, 15, 25, 21), 'queued_dttm': None, 'try_number': 0, 'queue': 'default', 'duration': None, 'state': None, 'start_date': None, 'operator': None, 'priority_weight': 1, 'hostname': u'', 'pool': None, 'dag_id': 'cdc'}]
  65. Process DagFileProcessor4741693-Process:
  66. Traceback (most recent call last):
  67. File "/home/ec2-user/resources/python27/lib/python2.7/multiprocessing/process.py", line 258, in _bootstrap
  68. self.run()
  69. File "/home/ec2-user/resources/python27/lib/python2.7/multiprocessing/process.py", line 114, in run
  70. self._target(*self._args, **self._kwargs)
  71. File "/home/ec2-user/resources/python27/lib/python2.7/site-packages/airflow/jobs.py", line 347, in helper
  72. pickle_dags)
  73. File "/home/ec2-user/resources/python27/lib/python2.7/site-packages/airflow/utils/db.py", line 53, in wrapper
  74. result = func(*args, **kwargs)
  75. File "/home/ec2-user/resources/python27/lib/python2.7/site-packages/airflow/jobs.py", line 1584, in process_file
  76. self._process_dags(dagbag, dags, ti_keys_to_schedule)
  77. File "/home/ec2-user/resources/python27/lib/python2.7/site-packages/airflow/jobs.py", line 1176, in _process_dags
  78. self._process_task_instances(dag, tis_out)
  79. File "/home/ec2-user/resources/python27/lib/python2.7/site-packages/airflow/jobs.py", line 879, in _process_task_instances
  80. run.verify_integrity(session=session)
  81. File "/home/ec2-user/resources/python27/lib/python2.7/site-packages/airflow/utils/db.py", line 53, in wrapper
  82. result = func(*args, **kwargs)
  83. File "/home/ec2-user/resources/python27/lib/python2.7/site-packages/airflow/models.py", line 4298, in verify_integrity
  84. session.commit()
  85. File "/home/ec2-user/resources/python27/lib/python2.7/site-packages/sqlalchemy/orm/session.py", line 874, in commit
  86. self.transaction.commit()
  87. File "/home/ec2-user/resources/python27/lib/python2.7/site-packages/sqlalchemy/orm/session.py", line 461, in commit
  88. self._prepare_impl()
  89. File "/home/ec2-user/resources/python27/lib/python2.7/site-packages/sqlalchemy/orm/session.py", line 441, in _prepare_impl
  90. self.session.flush()
  91. File "/home/ec2-user/resources/python27/lib/python2.7/site-packages/sqlalchemy/orm/session.py", line 2137, in flush
  92. self._flush(objects)
  93. File "/home/ec2-user/resources/python27/lib/python2.7/site-packages/sqlalchemy/orm/session.py", line 2257, in _flush
  94. transaction.rollback(_capture_exception=True)
  95. File "/home/ec2-user/resources/python27/lib/python2.7/site-packages/sqlalchemy/util/langhelpers.py", line 60, in __exit__
  96. compat.reraise(exc_type, exc_value, exc_tb)
  97. File "/home/ec2-user/resources/python27/lib/python2.7/site-packages/sqlalchemy/orm/session.py", line 2221, in _flush
  98. flush_context.execute()
  99. File "/home/ec2-user/resources/python27/lib/python2.7/site-packages/sqlalchemy/orm/unitofwork.py", line 389, in execute
  100. rec.execute(self)
  101. File "/home/ec2-user/resources/python27/lib/python2.7/site-packages/sqlalchemy/orm/unitofwork.py", line 548, in execute
  102. uow
  103. File "/home/ec2-user/resources/python27/lib/python2.7/site-packages/sqlalchemy/orm/persistence.py", line 181, in save_obj
  104. mapper, table, insert)
  105. File "/home/ec2-user/resources/python27/lib/python2.7/site-packages/sqlalchemy/orm/persistence.py", line 799, in _emit_insert_statements
  106. execute(statement, multiparams)
  107. File "/home/ec2-user/resources/python27/lib/python2.7/site-packages/sqlalchemy/engine/base.py", line 945, in execute
  108. return meth(self, multiparams, params)
  109. File "/home/ec2-user/resources/python27/lib/python2.7/site-packages/sqlalchemy/sql/elements.py", line 263, in _execute_on_connection
  110. return connection._execute_clauseelement(self, multiparams, params)
  111. File "/home/ec2-user/resources/python27/lib/python2.7/site-packages/sqlalchemy/engine/base.py", line 1053, in _execute_clauseelement
  112. compiled_sql, distilled_params
  113. File "/home/ec2-user/resources/python27/lib/python2.7/site-packages/sqlalchemy/engine/base.py", line 1189, in _execute_context
  114. context)
  115. File "/home/ec2-user/resources/python27/lib/python2.7/site-packages/sqlalchemy/engine/base.py", line 1393, in _handle_dbapi_exception
  116. exc_info
  117. File "/home/ec2-user/resources/python27/lib/python2.7/site-packages/sqlalchemy/util/compat.py", line 202, in raise_from_cause
  118. reraise(type(exception), exception, tb=exc_tb, cause=cause)
  119. File "/home/ec2-user/resources/python27/lib/python2.7/site-packages/sqlalchemy/engine/base.py", line 1182, in _execute_context
  120. context)
  121. File "/home/ec2-user/resources/python27/lib/python2.7/site-packages/sqlalchemy/engine/default.py", line 469, in do_execute
  122. cursor.execute(statement, parameters)
  123. IntegrityError: (psycopg2.IntegrityError) duplicate key value violates unique constraint "task_instance_pkey"
  124. DETAIL: Key (task_id, dag_id, execution_date)=(SI_PS3H, cdc, 2018-09-12 20:45:25) already exists.
  125. [SQL: 'INSERT INTO task_instance (task_id, dag_id, execution_date, start_date, end_date, duration, state, try_number, hostname, unixname, job_id, pool, queue, priority_weight, operator, queued_dttm, pid) VALUES (%(task_id)s, %(dag_id)s, %(execution_date)s, %(start_date)s, %(end_date)s, %(duration)s, %(state)s, %(try_number)s, %(hostname)s, %(unixname)s, %(job_id)s, %(pool)s, %(queue)s, %(priority_weight)s, %(operator)s, %(queued_dttm)s, %(pid)s)'] [parameters: {'task_id': 'SI_PS3H', 'execution_date': datetime.datetime(2018, 9, 12, 20, 45, 25), 'job_id': None, 'end_date': None, 'pid': None, 'hostname': u'', 'queued_dttm': None, 'try_number': 0, 'queue': 'default', 'duration': None, 'state': None, 'pool': None, 'operator': None, 'priority_weight': 1, 'start_date': None, 'dag_id': 'cdc', 'unixname': 'infoworksuser'}]
  126. Process DagFileProcessor5152353-Process:
  127. Traceback (most recent call last):
  128. File "/home/ec2-user/resources/python27/lib/python2.7/multiprocessing/process.py", line 258, in _bootstrap
  129. self.run()
  130. File "/home/ec2-user/resources/python27/lib/python2.7/multiprocessing/process.py", line 114, in run
  131. self._target(*self._args, **self._kwargs)
  132. File "/home/ec2-user/resources/python27/lib/python2.7/site-packages/airflow/jobs.py", line 347, in helper
  133. pickle_dags)
  134. File "/home/ec2-user/resources/python27/lib/python2.7/site-packages/airflow/utils/db.py", line 53, in wrapper
  135. result = func(*args, **kwargs)
  136. File "/home/ec2-user/resources/python27/lib/python2.7/site-packages/airflow/jobs.py", line 1584, in process_file
  137. self._process_dags(dagbag, dags, ti_keys_to_schedule)
  138. File "/home/ec2-user/resources/python27/lib/python2.7/site-packages/airflow/jobs.py", line 1176, in _process_dags
  139. self._process_task_instances(dag, tis_out)
  140. File "/home/ec2-user/resources/python27/lib/python2.7/site-packages/airflow/jobs.py", line 879, in _process_task_instances
  141. run.verify_integrity(session=session)
  142. File "/home/ec2-user/resources/python27/lib/python2.7/site-packages/airflow/utils/db.py", line 53, in wrapper
  143. result = func(*args, **kwargs)
  144. File "/home/ec2-user/resources/python27/lib/python2.7/site-packages/airflow/models.py", line 4298, in verify_integrity
  145. session.commit()
  146. File "/home/ec2-user/resources/python27/lib/python2.7/site-packages/sqlalchemy/orm/session.py", line 874, in commit
  147. self.transaction.commit()
  148. File "/home/ec2-user/resources/python27/lib/python2.7/site-packages/sqlalchemy/orm/session.py", line 461, in commit
  149. self._prepare_impl()
  150. File "/home/ec2-user/resources/python27/lib/python2.7/site-packages/sqlalchemy/orm/session.py", line 441, in _prepare_impl
  151. self.session.flush()
  152. File "/home/ec2-user/resources/python27/lib/python2.7/site-packages/sqlalchemy/orm/session.py", line 2137, in flush
  153. self._flush(objects)
  154. File "/home/ec2-user/resources/python27/lib/python2.7/site-packages/sqlalchemy/orm/session.py", line 2257, in _flush
  155. transaction.rollback(_capture_exception=True)
  156. File "/home/ec2-user/resources/python27/lib/python2.7/site-packages/sqlalchemy/util/langhelpers.py", line 60, in __exit__
  157. compat.reraise(exc_type, exc_value, exc_tb)
  158. File "/home/ec2-user/resources/python27/lib/python2.7/site-packages/sqlalchemy/orm/session.py", line 2221, in _flush
  159. flush_context.execute()
  160. File "/home/ec2-user/resources/python27/lib/python2.7/site-packages/sqlalchemy/orm/unitofwork.py", line 389, in execute
  161. rec.execute(self)
  162. File "/home/ec2-user/resources/python27/lib/python2.7/site-packages/sqlalchemy/orm/unitofwork.py", line 548, in execute
  163. uow
  164. File "/home/ec2-user/resources/python27/lib/python2.7/site-packages/sqlalchemy/orm/persistence.py", line 181, in save_obj
  165. mapper, table, insert)
  166. File "/home/ec2-user/resources/python27/lib/python2.7/site-packages/sqlalchemy/orm/persistence.py", line 799, in _emit_insert_statements
  167. execute(statement, multiparams)
  168. File "/home/ec2-user/resources/python27/lib/python2.7/site-packages/sqlalchemy/engine/base.py", line 945, in execute
  169. return meth(self, multiparams, params)
  170. File "/home/ec2-user/resources/python27/lib/python2.7/site-packages/sqlalchemy/sql/elements.py", line 263, in _execute_on_connection
  171. return connection._execute_clauseelement(self, multiparams, params)
  172. File "/home/ec2-user/resources/python27/lib/python2.7/site-packages/sqlalchemy/engine/base.py", line 1053, in _execute_clauseelement
  173. compiled_sql, distilled_params
  174. File "/home/ec2-user/resources/python27/lib/python2.7/site-packages/sqlalchemy/engine/base.py", line 1189, in _execute_context
  175. context)
  176. File "/home/ec2-user/resources/python27/lib/python2.7/site-packages/sqlalchemy/engine/base.py", line 1393, in _handle_dbapi_exception
  177. exc_info
  178. File "/home/ec2-user/resources/python27/lib/python2.7/site-packages/sqlalchemy/util/compat.py", line 202, in raise_from_cause
  179. reraise(type(exception), exception, tb=exc_tb, cause=cause)
  180. File "/home/ec2-user/resources/python27/lib/python2.7/site-packages/sqlalchemy/engine/base.py", line 1182, in _execute_context
  181. context)
  182. File "/home/ec2-user/resources/python27/lib/python2.7/site-packages/sqlalchemy/engine/default.py", line 469, in do_execute
  183. cursor.execute(statement, parameters)
  184. IntegrityError: (psycopg2.IntegrityError) duplicate key value violates unique constraint "task_instance_pkey"
  185. DETAIL: Key (task_id, dag_id, execution_date)=(SI_PS3H, cdc, 2018-09-13 10:30:43) already exists.
  186. [SQL: 'INSERT INTO task_instance (task_id, dag_id, execution_date, start_date, end_date, duration, state, try_number, hostname, unixname, job_id, pool, queue, priority_weight, operator, queued_dttm, pid) VALUES (%(task_id)s, %(dag_id)s, %(execution_date)s, %(start_date)s, %(end_date)s, %(duration)s, %(state)s, %(try_number)s, %(hostname)s, %(unixname)s, %(job_id)s, %(pool)s, %(queue)s, %(priority_weight)s, %(operator)s, %(queued_dttm)s, %(pid)s)'] [parameters: {'execution_date': datetime.datetime(2018, 9, 13, 10, 30, 43), 'end_date': None, 'job_id': None, 'task_id': 'SI_PS3H', 'queued_dttm': None, 'unixname': 'infoworksuser', 'pid': None, 'try_number': 0, 'queue': 'default', 'operator': None, 'state': None, 'start_date': None, 'duration': None, 'priority_weight': 1, 'hostname': u'', 'pool': None, 'dag_id': 'cdc'}]
RAW Paste Data