API tools faq
paste
Advertisement
Guest User

azure batch error

a guest
Dec 6th, 2017
525
0
Never
Add comment
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 25.99 KB | None | 0 0
raw download clone embed print report
  1. ■■20171205T231242.613Z■e67f6090-0024-419e-b680-acc34dbc92f3■DEBUG■agent.stats.task■task.py■_sample_stats■164■1398■MainThread■139690855581440■task stats sample: fqtn=pd1batch 22F55DC6E98C8653$1a-python 22F4F1C234F19066$job-1$t20171129-0010-06$0 store=True current ts=2017-12-05 23:12:42.613643+00:00 prev_delta=5.022486 walltime=1730588636430
  2. ■■20171205T231242.623Z■e67f6090-0024-419e-b680-acc34dbc92f3■DEBUG■agent.stats.task■task.py■_sample_stats■175■1398■MainThread■139690855581440■getting stats for process: 118158
  3. ■■20171205T231242.623Z■e67f6090-0024-419e-b680-acc34dbc92f3■DEBUG■agent.stats.task■task.py■_sample_stats■175■1398■MainThread■139690855581440■getting stats for process: 118171
  4. ■■20171205T231242.624Z■e67f6090-0024-419e-b680-acc34dbc92f3■DEBUG■agent.stats.task■task.py■_sample_stats■175■1398■MainThread■139690855581440■getting stats for process: 118172
  5. ■■20171205T231242.624Z■e67f6090-0024-419e-b680-acc34dbc92f3■DEBUG■agent.stats.task■task.py■_sample_async■575■1398■MainThread■139690855581440■scheduling task stats callback in 5 sec
  6. ■■20171205T231243.964Z■00000000-0000-0000-0000-000000000000■DEBUG■agent.server■server.py■_log_request_details■190■1398■MainThread■139690855581440■received message TVMHeartBeatRequest via PUT request from 104.41.129.99 to path / with content-length 111
  7. ■■20171205T231243.965Z■00000000-0000-0000-0000-000000000000■DEBUG■agent.server■server.py■_log_request_details■194■1398■MainThread■139690855581440■b'{"TypeName":"TVMHeartBeatRequest","ProtocolVersion":1,"ProtocolCapability":0,"activityId":[{"Data1":0,"Data2":0,"Data3":0,"Data4":0}],"absGuid":[{"Data1":4007836145,"Data2":4998,"Data3":17596,"Data4":14494908688715898254}],"absSequenceNumber":[591037234],"sasInfo":[],"heartBeatKindFlags":1,"lastTvmGuid":{"Data1":2315755910,"Data2":64981,"Data3":19433,"Data4":2774790199005499265},"lastTvmSequenceNumber":641526,"lastTvmAutoscaleStatsSequenceNumber":0,"enumerateAllTasks":false,"needVmCommunication":false,"memberIdType":null,"poolMembershipEventList":null,"isDeltaPoolMembershipEventList":[true]}'
  8. ■■20171205T231243.965Z■00000000-0000-0000-0000-000000000000■DEBUG■agent.node■node.py■validate_abs_sequence_number■579■1398■MainThread■139690855581440■processing TVMHeartBeatRequest request from abs guid: eee2b9f1-1386-44bc-8e5d-a2b2a74228c9 seqno: 591037234
  9. ■■20171205T231243.965Z■00000000-0000-0000-0000-000000000000■DEBUG■agent.node■node.py■validate_abs_sequence_number■623■1398■MainThread■139690855581440■valid abs sequence number set to: 591037234
  10. ■■20171205T231243.973Z■00000000-0000-0000-0000-000000000000■INFO■agent.handlers.heartbeat■heartbeat.py■handle_heartbeat_request_async■117■1398■MainThread■139690855581440■received heartbeat abs seqno: 591037234 last tvm seqno: 641526
  11. ■■20171205T231243.986Z■00000000-0000-0000-0000-000000000000■DEBUG■agent.handlers.heartbeat■heartbeat.py■handle_heartbeat_request_async■180■1398■MainThread■139690855581440■status from state->health: Status.Ok
  12. ■■20171205T231243.995Z■00000000-0000-0000-0000-000000000000■DEBUG■agent.handlers.heartbeat■heartbeat.py■_populate_taskinfo■267■1398■MainThread■139690855581440■retrieving historical task data, lastseqno: 641526 all: False
  13. ■■20171205T231243.995Z■00000000-0000-0000-0000-000000000000■DEBUG■agent.handlers.heartbeat■heartbeat.py■_populate_taskinfo■272■1398■MainThread■139690855581440■task history list returned 0 entries
  14. ■■20171205T231243.996Z■00000000-0000-0000-0000-000000000000■INFO■agent.handlers.heartbeat■heartbeat.py■handle_heartbeat_request_async■224■1398■MainThread■139690855581440■status for heartbeat: Status.Ok
  15. ■■20171205T231243.997Z■00000000-0000-0000-0000-000000000000■DEBUG■agent.server■server.py■bond_request_handler_async■268■1398■MainThread■139690855581440■sending response message: <class 'bond.node.TVMHeartBeatResponse'>
  16. ■■20171205T231243.997Z■00000000-0000-0000-0000-000000000000■DEBUG■agent.server■server.py■bond_request_handler_async■277■1398■MainThread■139690855581440■b'{"TypeName":"TVMHeartBeatResponse","ProtocolVersion":1,"ProtocolCapability":0,"activityId":[{"Data1":0,"Data2":0,"Data3":0,"Data4":0}],"status":[0],"tvmGuid":[{"Data1":2315755910,"Data2":64981,"Data3":19433,"Data4":2774790199005499265}],"tvmSequenceNumber":[641527],"sasInfo":[],"healthStatus":0,"alertType":0,"poolName":["pd1batch 22F55DC6E98C8653$f4_pool 22F554D2201172F5"],"isDeltaTaskList":[true],"taskList":null,"tvmAutoScaleStatisticsDataSet":null,"maxPoolMembershipEventNumber":null,"startTaskCompletionState":[4]}'
  17. ■■20171205T231244.483Z■e67f6090-0024-419e-b680-acc34dbc92f3■DEBUG■agent.task.lease■lease.py■_renew_lease_unsafe_async■106■1398■MainThread■139690855581440■extending lease for pd1batch 22F55DC6E98C8653$1a-python 22F4F1C234F19066$job-1$t20171129-0010-06
  18. ■■20171205T231244.486Z■00000000-0000-0000-0000-000000000000■INFO■requests.packages.urllib3.connectionpool■connectionpool.py■_new_conn■758■1398■Thread-1■139690661328640■Starting new HTTPS connection (1): watbl2prod1.queue.core.windows.net
  19. ■■20171205T231244.568Z■00000000-0000-0000-0000-000000000000■DEBUG■requests.packages.urllib3.connectionpool■connectionpool.py■_make_request■387■1398■Thread-1■139690661328640■"PUT /pd1batch-a-fa357c64-5c3d-4db8-9366-680943d2c20d/messages/821bf60d-3ba5-43a1-9c3d-c7500758bfea?sv=2015-07-08&se=2017-12-06T00%3A42%3A17Z&sp=up&sig=XXX&visibilitytimeout=360&popreceipt=AwAAAAMAAAAAAAAAFePc%2BR5u0wEBAAAA HTTP/1.1" 404 221
  20. ■■20171205T231244.571Z■e67f6090-0024-419e-b680-acc34dbc92f3■ERROR■azurestorage.helper■helper.py■handle_exception■520■1398■MainThread■139690855581440■404 Client Error: The specified message does not exist. for url: https://watbl2prod1.queue.core.windows.net/pd1batch-a-fa357c64-5c3d-4db8-9366-680943d2c20d/messages/821bf60d-3ba5-43a1-9c3d-c7500758bfea?sv=2015-07-08&se=2017-12-06T00%3A42%3A17Z&sp=up&sig=mU9501N4HHuDeRWuA7qMNni9M%2Fbb83OWLF8AW0%2B4nQE%3D&visibilitytimeout=360&popreceipt=AwAAAAMAAAAAAAAAFePc%2BR5u0wEBAAAA
  21. Traceback (most recent call last):
  22. File "/var/build/workspace/AzBatch/NodeAgentUbuntu1604/src/azurestorage/helper.py", line 413, in http_request_wrapper
  23. File "/var/build/workspace/AzBatch/NodeAgentUbuntu1604/venv/lib/python3.5/site-packages/requests/models.py", line 840, in raise_for_status
  24. requests.exceptions.HTTPError: 404 Client Error: The specified message does not exist. for url: https://watbl2prod1.queue.core.windows.net/pd1batch-a-fa357c64-5c3d-4db8-9366-680943d2c20d/messages/821bf60d-3ba5-43a1-9c3d-c7500758bfea?sv=2015-07-08&se=2017-12-06T00%3A42%3A17Z&sp=up&sig=mU9501N4HHuDeRWuA7qMNni9M%2Fbb83OWLF8AW0%2B4nQE%3D&visibilitytimeout=360&popreceipt=AwAAAAMAAAAAAAAAFePc%2BR5u0wEBAAAA
  25.  
  26. The above exception was the direct cause of the following exception:
  27.  
  28. Traceback (most recent call last):
  29. File "/var/build/workspace/AzBatch/NodeAgentUbuntu1604/src/agent/azure.py", line 771, in task_queue_interaction_async
  30. File "/var/build/workspace/AzBatch/NodeAgentUbuntu1604/src/azurestorage/helper.py", line 1416, in update_message_with_sas_async
  31. File "/usr/lib/python3.5/asyncio/futures.py", line 361, in __iter__
  32. yield self # This tells Task to wait for completion.
  33. File "/usr/lib/python3.5/asyncio/tasks.py", line 296, in _wakeup
  34. future.result()
  35. File "/usr/lib/python3.5/asyncio/futures.py", line 274, in result
  36. raise self._exception
  37. File "/usr/lib/python3.5/concurrent/futures/thread.py", line 55, in run
  38. result = self.fn(*self.args, **self.kwargs)
  39. File "/var/build/workspace/AzBatch/NodeAgentUbuntu1604/src/azurestorage/helper.py", line 1384, in update_message_with_sas
  40. File "/var/build/workspace/AzBatch/NodeAgentUbuntu1604/src/azurestorage/helper.py", line 447, in http_request_wrapper
  41. azurestorage.helper.HTTPNotFoundError: 404 Client Error: The specified message does not exist. for url: https://watbl2prod1.queue.core.windows.net/pd1batch-a-fa357c64-5c3d-4db8-9366-680943d2c20d/messages/821bf60d-3ba5-43a1-9c3d-c7500758bfea?sv=2015-07-08&se=2017-12-06T00%3A42%3A17Z&sp=up&sig=mU9501N4HHuDeRWuA7qMNni9M%2Fbb83OWLF8AW0%2B4nQE%3D&visibilitytimeout=360&popreceipt=AwAAAAMAAAAAAAAAFePc%2BR5u0wEBAAAA
  42. ■■20171205T231244.610Z■e67f6090-0024-419e-b680-acc34dbc92f3■ERROR■azurestorage.helper■helper.py■handle_exception■523■1398■MainThread■139690855581440■<?xml version="1.0" encoding="utf-8"?><Error><Code>MessageNotFound</Code><Message>The specified message does not exist.
  43. RequestId:4e339b01-0003-0092-291e-6e4cc3000000
  44. Time:2017-12-05T23:12:44.3380615Z</Message></Error>
  45. ■■20171205T231244.610Z■e67f6090-0024-419e-b680-acc34dbc92f3■DEBUG■azurestorage.helper■helper.py■handle_exception■525■1398■MainThread■139690855581440■request headers: {'User-Agent': 'python-requests/2.9.1', 'Accept-Encoding': 'gzip, deflate', 'Content-Length': '0', 'Connection': 'keep-alive', 'Accept': '*/*'}
  46. ■■20171205T231244.610Z■e67f6090-0024-419e-b680-acc34dbc92f3■ERROR■agent.task.lease■lease.py■_renew_lease_unsafe_async■119■1398■MainThread■139690855581440■failed to extend lease for pd1batch 22F55DC6E98C8653$1a-python 22F4F1C234F19066$job-1$t20171129-0010-06
  47. ■■20171205T231244.610Z■e67f6090-0024-419e-b680-acc34dbc92f3■DEBUG■common.asyncsharedlocks■asyncsharedlocks.py■shared_lock_async■61■1398■MainThread■139690855581440■Shared lock acquired for 1a-python 22F4F1C234F19066
  48. ■■20171205T231244.610Z■e67f6090-0024-419e-b680-acc34dbc92f3■DEBUG■common.asyncsharedlocks■asyncsharedlocks.py■shared_lock_async■61■1398■MainThread■139690855581440■Shared lock acquired for job-1
  49. ■■20171205T231244.611Z■e67f6090-0024-419e-b680-acc34dbc92f3■DEBUG■common.asyncsharedlocks■asyncsharedlocks.py■exclusive_lock_async■77■1398■MainThread■139690855581440■Exclusive lock acquired for t20171129-0010-06
  50. ■■20171205T231244.611Z■e67f6090-0024-419e-b680-acc34dbc92f3■WARNING■agent.task.manager■manager.py■handle_task_lease_extension_error_async■4713■1398■MainThread■139690855581440■deleting task pd1batch 22F55DC6E98C8653$1a-python 22F4F1C234F19066$job-1$t20171129-0010-06$0 because lease was lost
  51. ■■20171205T231244.611Z■e67f6090-0024-419e-b680-acc34dbc92f3■DEBUG■agent.task.manager■manager.py■end_task_async■3900■1398■MainThread■139690855581440■ending task pd1batch 22F55DC6E98C8653$1a-python 22F4F1C234F19066$job-1$t20171129-0010-06$0 action=EndTaskAction_Delete term_state=None se_dict={<Status.TvmTaskEnded: 2215248644>: (None, None)} force_delete_data=True
  52. ■■20171205T231244.620Z■e67f6090-0024-419e-b680-acc34dbc92f3■DEBUG■common.asyncsharedlocks■asyncsharedlocks.py■exclusive_lock_async■77■1398■MainThread■139690855581440■Exclusive lock acquired for pd1batch 22F55DC6E98C8653$1a-python 22F4F1C234F19066$job-1$t20171129-0010-06$0
  53. ■■20171205T231244.627Z■e67f6090-0024-419e-b680-acc34dbc92f3■DEBUG■agent.task.process■process.py■cancel_process_cbhandle■1414■1398■MainThread■139690855581440■attempting to cancel proc callback for pd1batch 22F55DC6E98C8653$1a-python 22F4F1C234F19066$job-1$t20171129-0010-06$0
  54. ■■20171205T231244.627Z■e67f6090-0024-419e-b680-acc34dbc92f3■INFO■agent.task.process■process.py■cancel_process_cbhandle■1418■1398■MainThread■139690855581440■proc callback cancelled for pd1batch 22F55DC6E98C8653$1a-python 22F4F1C234F19066$job-1$t20171129-0010-06$0
  55. ■■20171205T231244.628Z■00000000-0000-0000-0000-000000000000■DEBUG■agent.task.data■data.py■set_sched_error■746■1398■MainThread■139690855581440■setting task pd1batch 22F55DC6E98C8653$1a-python 22F4F1C234F19066$job-1$t20171129-0010-06$0 sched_error to Status.TvmTaskEnded
  56. ■■20171205T231244.633Z■e67f6090-0024-419e-b680-acc34dbc92f3■INFO■agent.task.process■process.py■_terminate_process_progression■1685■1398■MainThread■139690855581440■SIGINT sent to pgid=118158 pid=118158 task=pd1batch 22F55DC6E98C8653$1a-python 22F4F1C234F19066$job-1$t20171129-0010-06$0
  57.  
  58. ■■20171205T231245.135Z■e67f6090-0024-419e-b680-acc34dbc92f3■DEBUG■agent.task.manager■manager.py■end_task_async■3934■1398■MainThread■139690855581440■task pd1batch 22F55DC6E98C8653$1a-python 22F4F1C234F19066$job-1$t20171129-0010-06$0 pid=118158 actively terminated
  59. ■■20171205T231245.136Z■e67f6090-0024-419e-b680-acc34dbc92f3■INFO■agent.task.manager■manager.py■handle_task_lease_extension_error_async■4718■1398■MainThread■139690855581440■task pd1batch 22F55DC6E98C8653$1a-python 22F4F1C234F19066$job-1$t20171129-0010-06$0 deletion request completed with status Status.Ok
  60. ■■20171205T231245.137Z■e67f6090-0024-419e-b680-acc34dbc92f3■DEBUG■common.asyncsharedlocks■asyncsharedlocks.py■unlock_async■92■1398■MainThread■139690855581440■Exclusive lock released for pd1batch 22F55DC6E98C8653$1a-python 22F4F1C234F19066$job-1$t20171129-0010-06$0
  61. ■■20171205T231245.137Z■e67f6090-0024-419e-b680-acc34dbc92f3■DEBUG■common.asyncsharedlocks■asyncsharedlocks.py■unlock_async■92■1398■MainThread■139690855581440■Exclusive lock released for t20171129-0010-06
  62. ■■20171205T231245.137Z■e67f6090-0024-419e-b680-acc34dbc92f3■DEBUG■common.asyncsharedlocks■asyncsharedlocks.py■unlock_async■105■1398■MainThread■139690855581440■Shared lock released for job-1
  63. ■■20171205T231245.137Z■e67f6090-0024-419e-b680-acc34dbc92f3■DEBUG■common.asyncsharedlocks■asyncsharedlocks.py■unlock_async■105■1398■MainThread■139690855581440■Shared lock released for 1a-python 22F4F1C234F19066
  64. ■■20171205T231245.138Z■e67f6090-0024-419e-b680-acc34dbc92f3■DEBUG■agent.task.process■process.py■spawn_task_process_async■1152■1398■MainThread■139690855581440■task pd1batch 22F55DC6E98C8653$1a-python 22F4F1C234F19066$job-1$t20171129-0010-06$0 process exited: pid=118158 exitcode=2
  65. ■■20171205T231245.138Z■e67f6090-0024-419e-b680-acc34dbc92f3■DEBUG■agent.stats.task■task.py■unregister_process■630■1398■MainThread■139690855581440■unregistering stats for task=pd1batch 22F55DC6E98C8653$1a-python 22F4F1C234F19066$job-1$t20171129-0010-06$0 completion_state=TaskCompletion_Retry
  66. ■■20171205T231245.138Z■e67f6090-0024-419e-b680-acc34dbc92f3■DEBUG■agent.stats.task■task.py■unregister_process■642■1398■MainThread■139690855581440■attempting to cancel task stats sampler pd1batch 22F55DC6E98C8653$1a-python 22F4F1C234F19066$job-1$t20171129-0010-06$0
  67. ■■20171205T231245.138Z■e67f6090-0024-419e-b680-acc34dbc92f3■INFO■agent.stats.task■task.py■unregister_process■645■1398■MainThread■139690855581440■task stats sampler cancelled for pd1batch 22F55DC6E98C8653$1a-python 22F4F1C234F19066$job-1$t20171129-0010-06$0
  68. ■■20171205T231245.139Z■e67f6090-0024-419e-b680-acc34dbc92f3■DEBUG■common.fsm.base■base.py■transition■154■1398■MainThread■139690855581440■TaskFsm state transition id=pd1batch 22F55DC6E98C8653$1a-python 22F4F1C234F19066$job-1$t20171129-0010-06$0: Process_Wait --UploadFiles--> FileUpload_Wait
  69. ■■20171205T231245.141Z■e67f6090-0024-419e-b680-acc34dbc92f3■DEBUG■agent.task.manager■manager.py■_upload_files_async■3140■1398■MainThread■139690855581440■output_files is None for task pd1batch 22F55DC6E98C8653$1a-python 22F4F1C234F19066$job-1$t20171129-0010-06$0, skipping output file upload
  70. ■■20171205T231245.142Z■e67f6090-0024-419e-b680-acc34dbc92f3■WARNING■agent.task.manager■manager.py■handle_task_error_async■1927■1398■MainThread■139690855581440■handling task error fqtn=pd1batch 22F55DC6E98C8653$1a-python 22F4F1C234F19066$job-1$t20171129-0010-06$0 retry=True internal_error=False complete_task=True proc={'stderr': None, 'pid': 118158, 'exitcode': 2, 'stdout': None}
  71. ■■20171205T231245.142Z■e67f6090-0024-419e-b680-acc34dbc92f3■DEBUG■agent.task.manager■manager.py■handle_task_error_async■1946■1398■MainThread■139690855581440■force no retries set for pd1batch 22F55DC6E98C8653$1a-python 22F4F1C234F19066$job-1$t20171129-0010-06$0,True,False
  72. ■■20171205T231245.142Z■e67f6090-0024-419e-b680-acc34dbc92f3■INFO■agent.task.manager■manager.py■terminate_task_async■3775■1398■MainThread■139690855581440■terminating task pd1batch 22F55DC6E98C8653$1a-python 22F4F1C234F19066$job-1$t20171129-0010-06$0 term_state=None release_msg=False delete_msg=True
  73. ■■20171205T231245.142Z■00000000-0000-0000-0000-000000000000■DEBUG■agent.task.data■data.py■finish_time■308■1398■MainThread■139690855581440■setting task pd1batch 22F55DC6E98C8653$1a-python 22F4F1C234F19066$job-1$t20171129-0010-06$0 finish time to 131569891651426040
  74. ■■20171205T231245.142Z■e67f6090-0024-419e-b680-acc34dbc92f3■DEBUG■agent.task.data■data.py■transition_task_state■795■1398■MainThread■139690855581440■task pd1batch 22F55DC6E98C8653$1a-python 22F4F1C234F19066$job-1$t20171129-0010-06$0 new previous state: 512
  75. ■■20171205T231245.142Z■e67f6090-0024-419e-b680-acc34dbc92f3■DEBUG■agent.task.data■data.py■transition_task_state■811■1398■MainThread■139690855581440■task pd1batch 22F55DC6E98C8653$1a-python 22F4F1C234F19066$job-1$t20171129-0010-06$0 new state: TaskStateCompleted
  76. ■■20171205T231245.142Z■00000000-0000-0000-0000-000000000000■DEBUG■agent.task.data■data.py■exit_code■186■1398■MainThread■139690855581440■setting task pd1batch 22F55DC6E98C8653$1a-python 22F4F1C234F19066$job-1$t20171129-0010-06$0 exit code to 2
  77. ■■20171205T231245.143Z■e67f6090-0024-419e-b680-acc34dbc92f3■DEBUG■agent.task.data■data.py■update_task_completion■860■1398■MainThread■139690855581440■task pd1batch 22F55DC6E98C8653$1a-python 22F4F1C234F19066$job-1$t20171129-0010-06$0 completion updated: {'StartTime': 131568161040304650, 'TVMSD': 'sd:[pd1batch 22f55dc6e98c8653$f4_pool 22f554d2201172f5:ce8d131f-e8ba-4bc6-9f0b-3522870e50fd-azurebatch-cloudservice:ce8d131f-e8ba-4bc6-9f0b-3522870e50fd-azurebatch-deployment_7:TVM]', 'PreviousState': 512, 'PreviousStateTransitionTime': 131568161040303660, 'MaxRetainAfterTerminateTime': 9223372036854775807, 'LastModificationTime': 131563842541492582, 'TaskRootPath': 'workitems/1A-python/job-1/t20171129-0010-06', 'StateTransitionTime': 131569891651426040, 'SchedulingError': -2079718652, 'PartitionKey': 'pd1batch 22F55DC6E98C8653$1a-python 22F4F1C234F19066$job-1', 'CasePreservedPoolName': 'F4_pool', 'AffinityId': 'TVM:tvm-4155946844_36-20171118t093410z-p', 'MaxTaskRetryCount': 0, 'State': 768, 'ExitCode': 2, 'FinishTime': 131569891651426040, 'TVM': 'pd1batch 22F55DC6E98C8653$f4_pool 22F554D2201172F5$tvm-4155946844_36-20171118t093410z-p', 'MaxExecutionTime': 4320000000000, 'RowKey': 't20171129-0010-06', 'RetryCount': 0, 'TenantName': 'edf67375b2c846bb96581188cdfc3f3a'}
  78. ■■20171205T231245.147Z■e67f6090-0024-419e-b680-acc34dbc92f3■DEBUG■common.fsm.base■base.py■transition■154■1398■MainThread■139690855581440■TaskFsm state transition id=pd1batch 22F55DC6E98C8653$1a-python 22F4F1C234F19066$job-1$t20171129-0010-06$0: FileUpload_Wait --TerminateTask--> UpdateTask_Wait
  79. ■■20171205T231245.149Z■e67f6090-0024-419e-b680-acc34dbc92f3■DEBUG■agent.task.manager■manager.py■terminate_task_async■3805■1398■MainThread■139690855581440■will execute postprocess on task pd1batch 22F55DC6E98C8653$1a-python 22F4F1C234F19066$job-1$t20171129-0010-06$0
  80. ■■20171205T231245.150Z■00000000-0000-0000-0000-000000000000■DEBUG■agent.task.userconfiguration■userconfiguration.py■deserialize_user_identity■36■1398■MainThread■139690855581440■setting task pd1batch 22F55DC6E98C8653$1a-python 22F4F1C234F19066$job-1$t20171129-0010-06$0 user_identity to PoolAdmin
  81. ■■20171205T231245.152Z■e67f6090-0024-419e-b680-acc34dbc92f3■INFO■agent.task.manager■manager.py■_postprocess_execute_task_async■2226■1398■MainThread■139690855581440■beginning postprocess for pd1batch 22F55DC6E98C8653$1a-python 22F4F1C234F19066$job-1$t20171129-0010-06$0 registered=True
  82. ■■20171205T231245.152Z■e67f6090-0024-419e-b680-acc34dbc92f3■DEBUG■common.fsm.base■base.py■transition■154■1398■MainThread■139690855581440■TaskFsm state transition id=pd1batch 22F55DC6E98C8653$1a-python 22F4F1C234F19066$job-1$t20171129-0010-06$0: UpdateTask_Wait --TerminateLease--> LeaseStop_Wait
  83. ■■20171205T231245.152Z■e67f6090-0024-419e-b680-acc34dbc92f3■DEBUG■agent.task.exitconditions■exitconditions.py■should_send_completion_msg■82■1398■MainThread■139690855581440■Got job action JobAction_None and dependency action DependencyAction_Undefined for exit code 2 upload failure False
  84. ■■20171205T231245.152Z■e67f6090-0024-419e-b680-acc34dbc92f3■DEBUG■agent.task.exitconditions■exitconditions.py■should_send_completion_msg■104■1398■MainThread■139690855581440■Should send completion message = False, job action = None, dependency action = None
  85. ■■20171205T231245.153Z■00000000-0000-0000-0000-000000000000■DEBUG■agent.task.data■data.py■job_action■211■1398■MainThread■139690855581440■setting task pd1batch 22F55DC6E98C8653$1a-python 22F4F1C234F19066$job-1$t20171129-0010-06$0 job action to None
  86. ■■20171205T231245.153Z■00000000-0000-0000-0000-000000000000■DEBUG■agent.task.data■data.py■dependency_action■236■1398■MainThread■139690855581440■setting task pd1batch 22F55DC6E98C8653$1a-python 22F4F1C234F19066$job-1$t20171129-0010-06$0 dependency action to None
  87. ■■20171205T231245.155Z■e67f6090-0024-419e-b680-acc34dbc92f3■DEBUG■agent.task.manager■manager.py■_postprocess_execute_task_async■2255■1398■MainThread■139690855581440■updating row in task table for: pd1batch 22F55DC6E98C8653$1a-python 22F4F1C234F19066$job-1$t20171129-0010-06$0
  88. ■■20171205T231245.155Z■e67f6090-0024-419e-b680-acc34dbc92f3■WARNING■agent.task.manager■manager.py■_update_tasktable_entity_async■1624■1398■MainThread■139690855581440■aborting attempt to update task table without an active queue lease for pd1batch 22F55DC6E98C8653$1a-python 22F4F1C234F19066$job-1$t20171129-0010-06$0
  89. ■■20171205T231245.155Z■e67f6090-0024-419e-b680-acc34dbc92f3■ERROR■agent.task.manager■manager.py■_postprocess_execute_task_async■2261■1398■MainThread■139690855581440■could not update task table for: pd1batch 22F55DC6E98C8653$1a-python 22F4F1C234F19066$job-1$t20171129-0010-06$0
  90. ■■20171205T231245.155Z■e67f6090-0024-419e-b680-acc34dbc92f3■WARNING■agent.task.manager■manager.py■_postprocess_execute_task_async■2324■1398■MainThread■139690855581440■could not add task completion message for pd1batch 22F55DC6E98C8653$1a-python 22F4F1C234F19066$job-1$t20171129-0010-06$0 state=768 lease=False delete_msg=True release_msg=False
  91. ■■20171205T231245.155Z■e67f6090-0024-419e-b680-acc34dbc92f3■DEBUG■agent.task.manager■manager.py■remove_task_sasinfo■3470■1398■MainThread■139690855581440■Removing sas token for existing task pd1batch 22F55DC6E98C8653$1a-python 22F4F1C234F19066$job-1$t20171129-0010-06$0
  92. ■■20171205T231245.157Z■00000000-0000-0000-0000-000000000000■DEBUG■agent.task.manager■manager.py■fsm_transition_callback■4343■1398■MainThread■139690855581440■in task fsm transition callback task=pd1batch 22F55DC6E98C8653$1a-python 22F4F1C234F19066$job-1$t20171129-0010-06$0 jp=False jr=False
  93. ■■20171205T231245.160Z■00000000-0000-0000-0000-000000000000■INFO■agent.task.history■history.py■add_task_to_history■275■1398■MainThread■139690855581440■new state TaskStateCompleted added for task pd1batch 22F55DC6E98C8653$1a-python 22F4F1C234F19066$job-1$t20171129-0010-06$0:v0 ec:2 se:-2079718652
  94. ■■20171205T231245.160Z■e67f6090-0024-419e-b680-acc34dbc92f3■DEBUG■common.fsm.base■base.py■transition■154■1398■MainThread■139690855581440■TaskFsm state transition id=pd1batch 22F55DC6E98C8653$1a-python 22F4F1C234F19066$job-1$t20171129-0010-06$0: LeaseStop_Wait --Finalize--> Completed
  95. ■■20171205T231245.165Z■e67f6090-0024-419e-b680-acc34dbc92f3■DEBUG■agent.fsm.node■node.py■unregister_running_task■210■1398■MainThread■139690855581440■number of running tasks -> node: 0 user: 0
  96. ■■20171205T231245.165Z■e67f6090-0024-419e-b680-acc34dbc92f3■DEBUG■common.fsm.base■base.py■transition■154■1398■MainThread■139690855581440■NodeFsm state transition id=None: RunningTask --CompleteTask--> Ready
  97. ■■20171205T231245.165Z■e67f6090-0024-419e-b680-acc34dbc92f3■DEBUG■agent.task.manager■manager.py■get_recent_task_history■4368■1398■MainThread■139690855581440■retrieving recent task history
  98. ■■20171205T231245.165Z■e67f6090-0024-419e-b680-acc34dbc92f3■DEBUG■agent.task.history■history.py■get_recent_task_history■414■1398■MainThread■139690855581440■taskinfolist entry: task pd1batch 22F55DC6E98C8653$1gpu-20171114 22F4FCF7E06C6EE4$job-1$j7$0 state=TaskStateCompleted starttime=131554712644783050 endtime=131557663952578220 ec=2 se=-2079718652
  99. ■■20171205T231245.166Z■e67f6090-0024-419e-b680-acc34dbc92f3■DEBUG■agent.task.history■history.py■get_recent_task_history■414■1398■MainThread■139690855581440■taskinfolist entry: task pd1batch 22F55DC6E98C8653$1a-python 22F4F1C234F19066$job-1$t20171129-0008-01$0 state=TaskStateCompleted starttime=131563841030415770 endtime=131568161033397640 ec=2 se=-2079718652
  100. ■■20171205T231245.166Z■e67f6090-0024-419e-b680-acc34dbc92f3■DEBUG■agent.task.history■history.py■get_recent_task_history■414■1398■MainThread■139690855581440■taskinfolist entry: task pd1batch 22F55DC6E98C8653$1a-python 22F4F1C234F19066$job-1$t20171129-0010-06$0 state=TaskStateCompleted starttime=131568161040304650 endtime=131569891651426040 ec=2 se=-2079718652
  101. ■■20171205T231245.166Z■e67f6090-0024-419e-b680-acc34dbc92f3■DEBUG■agent.node■node.py■_update_vmtable_state_async■864■1398■MainThread■139690855581440■recent tasks list returned 3 entries
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement
Public Pastes
Advertisement
We use cookies for various purposes including analytics. By continuing to use Pastebin, you agree to our use of cookies as described in the Cookies Policy.  OK, I Understand
Not a member of Pastebin yet?
Sign Up, it unlocks many cool features!