Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- /usr/local/cuda/samples/1_Utilities/deviceQuery# ./deviceQuery
- ./deviceQuery Starting...
- CUDA Device Query (Runtime API) version (CUDART static linking)
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.288088] INFO: task kworker/0:0:4 blocked for more than 120 seconds.
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.291751] Tainted: P OX 3.13.0-91-generic #138-Ubuntu
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.295340] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.299760] kworker/0:0 D ffff8803dfc13180 0 4 2 0x00000000
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.299852] Workqueue: events os_execute_work_item [nvidia]
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.299860] ffff8803caa11b70 0000000000000046 ffff8803ca97c800 0000000000013180
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.299863] ffff8803caa11fd8 0000000000013180 ffff8803ca97c800 ffff8803c94f1848
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.299865] 7fffffffffffffff ffff8803ca97c800 0000000000000002 0000000000000000
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.299868] Call Trace:
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.299874] [<ffffffff8172dd89>] schedule+0x29/0x70
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.299877] [<ffffffff8172cfd9>] schedule_timeout+0x279/0x310
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.299886] [<ffffffff8172d8f4>] ? __schedule+0x384/0x7f0
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.299889] [<ffffffff81730276>] __down_common+0xf7/0x144
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.299891] [<ffffffff817302e0>] __down+0x1d/0x1f
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.299894] [<ffffffff810b36a1>] down+0x41/0x50
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.299963] [<ffffffffa063bc4f>] os_acquire_mutex+0x3f/0x50 [nvidia]
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.300037] [<ffffffffa061dfd8>] _nv012536rm+0x18/0x30 [nvidia]
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.300109] [<ffffffffa05cfbbd>] ? _nv014113rm+0x3d/0x120 [nvidia]
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.300212] [<ffffffffa0486a2d>] ? _nv013080rm+0x14d/0x260 [nvidia]
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.300283] [<ffffffffa0621d39>] ? _nv000671rm+0x229/0xb10 [nvidia]
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.300350] [<ffffffffa0624d79>] ? rm_execute_work_item+0x49/0xc0 [nvidia]
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.300419] [<ffffffffa063c196>] ? os_execute_work_item+0x46/0x70 [nvidia]
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.300423] [<ffffffff81086078>] ? process_one_work+0x178/0x470
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.300426] [<ffffffff81086857>] ? manage_workers.isra.25+0x1f7/0x2e0
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.300428] [<ffffffff81086e91>] ? worker_thread+0x121/0x410
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.300431] [<ffffffff81086d70>] ? rescuer_thread+0x430/0x430
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.300434] [<ffffffff8108dc79>] ? kthread+0xc9/0xe0
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.300436] [<ffffffff8108dbb0>] ? kthread_create_on_node+0x1c0/0x1c0
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.300439] [<ffffffff8173a3e8>] ? ret_from_fork+0x58/0x90
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.300441] [<ffffffff8108dbb0>] ? kthread_create_on_node+0x1c0/0x1c0
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.300452] INFO: task kworker/7:0:74 blocked for more than 120 seconds.
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.304186] Tainted: P OX 3.13.0-91-generic #138-Ubuntu
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.307853] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.312045] kworker/7:0 D ffff8803dfcf3180 0 74 2 0x00000000
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.312126] Workqueue: events os_execute_work_item [nvidia]
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.312133] ffff8803ca401b80 0000000000000046 ffff8803cabfb000 0000000000013180
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.312136] ffff8803ca401fd8 0000000000013180 ffff8803cabfb000 ffff8803c94f1848
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.312139] 7fffffffffffffff ffff8803cabfb000 0000000000000002 0000000000000000
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.312141] Call Trace:
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.312146] [<ffffffff8172dd89>] schedule+0x29/0x70
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.312148] [<ffffffff8172cfd9>] schedule_timeout+0x279/0x310
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.312150] [<ffffffff8172d8f4>] ? __schedule+0x384/0x7f0
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.312153] [<ffffffff81730276>] __down_common+0xf7/0x144
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.312163] [<ffffffff817302e0>] __down+0x1d/0x1f
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.312167] [<ffffffff810b36a1>] down+0x41/0x50
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.312235] [<ffffffffa063bc4f>] os_acquire_mutex+0x3f/0x50 [nvidia]
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.312305] [<ffffffffa061dfd8>] _nv012536rm+0x18/0x30 [nvidia]
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.312375] [<ffffffffa05cfbbd>] ? _nv014113rm+0x3d/0x120 [nvidia]
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.312468] [<ffffffffa0451327>] ? _nv010588rm+0x57/0xe0 [nvidia]
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.312535] [<ffffffffa0621d39>] ? _nv000671rm+0x229/0xb10 [nvidia]
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.312602] [<ffffffffa0624d79>] ? rm_execute_work_item+0x49/0xc0 [nvidia]
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.312674] [<ffffffffa063c196>] ? os_execute_work_item+0x46/0x70 [nvidia]
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.312677] [<ffffffff81086078>] ? process_one_work+0x178/0x470
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.312680] [<ffffffff81086857>] ? manage_workers.isra.25+0x1f7/0x2e0
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.312682] [<ffffffff81086e91>] ? worker_thread+0x121/0x410
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.312684] [<ffffffff81086d70>] ? rescuer_thread+0x430/0x430
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.312687] [<ffffffff8108dc79>] ? kthread+0xc9/0xe0
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.312689] [<ffffffff8108dbb0>] ? kthread_create_on_node+0x1c0/0x1c0
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.312692] [<ffffffff8173a3e8>] ? ret_from_fork+0x58/0x90
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.312694] [<ffffffff8108dbb0>] ? kthread_create_on_node+0x1c0/0x1c0
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.312698] INFO: task kworker/7:1:87 blocked for more than 120 seconds.
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.316392] Tainted: P OX 3.13.0-91-generic #138-Ubuntu
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.320002] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.324179] kworker/7:1 D ffff8803dfcf3180 0 87 2 0x00000000
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.324261] Workqueue: events os_execute_work_item [nvidia]
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.324268] ffff8803ca663b80 0000000000000046 ffff8803ca5b9800 0000000000013180
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.324271] ffff8803ca663fd8 0000000000013180 ffff8803ca5b9800 ffff8803c94f1848
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.324273] 7fffffffffffffff ffff8803ca5b9800 0000000000000002 0000000000000000
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.324276] Call Trace:
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.324280] [<ffffffff8172dd89>] schedule+0x29/0x70
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.324283] [<ffffffff8172cfd9>] schedule_timeout+0x279/0x310
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.324285] [<ffffffff81730276>] __down_common+0xf7/0x144
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.324288] [<ffffffff817302e0>] __down+0x1d/0x1f
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.324299] [<ffffffff810b36a1>] down+0x41/0x50
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.324369] [<ffffffffa063bc4f>] os_acquire_mutex+0x3f/0x50 [nvidia]
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.324436] [<ffffffffa061dfd8>] _nv012536rm+0x18/0x30 [nvidia]
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.324506] [<ffffffffa05cfbbd>] ? _nv014113rm+0x3d/0x120 [nvidia]
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.324598] [<ffffffffa0451327>] ? _nv010588rm+0x57/0xe0 [nvidia]
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.324668] [<ffffffffa0621d39>] ? _nv000671rm+0x229/0xb10 [nvidia]
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.324735] [<ffffffffa0624d79>] ? rm_execute_work_item+0x49/0xc0 [nvidia]
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.324804] [<ffffffffa063c196>] ? os_execute_work_item+0x46/0x70 [nvidia]
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.324807] [<ffffffff81086078>] ? process_one_work+0x178/0x470
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.324810] [<ffffffff81086e91>] ? worker_thread+0x121/0x410
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.324813] [<ffffffff81086d70>] ? rescuer_thread+0x430/0x430
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.324815] [<ffffffff8108dc79>] ? kthread+0xc9/0xe0
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.324817] [<ffffffff8108dbb0>] ? kthread_create_on_node+0x1c0/0x1c0
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.324820] [<ffffffff8173a3e8>] ? ret_from_fork+0x58/0x90
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.324822] [<ffffffff8108dbb0>] ? kthread_create_on_node+0x1c0/0x1c0
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.324839] INFO: task nvidia-smi:4514 blocked for more than 120 seconds.
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.328549] Tainted: P OX 3.13.0-91-generic #138-Ubuntu
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.332079] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.336261] nvidia-smi D ffff8803dfcf3180 0 4514 1 0x00000004
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.336264] ffff8803c5393990 0000000000000082 ffff8803c77e1800 0000000000013180
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.336267] ffff8803c5393fd8 0000000000013180 ffff8803c77e1800 ffff8803c94f1848
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.336270] 7fffffffffffffff ffff8803c77e1800 0000000000000002 0000000000000000
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.336276] Call Trace:
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.336281] [<ffffffff8172dd89>] schedule+0x29/0x70
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.336283] [<ffffffff8172cfd9>] schedule_timeout+0x279/0x310
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.336287] [<ffffffff8109d302>] ? default_wake_function+0x12/0x20
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.336290] [<ffffffff810ad3e8>] ? __wake_up_common+0x58/0x90
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.336297] [<ffffffff81730276>] __down_common+0xf7/0x144
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.336302] [<ffffffff817302e0>] __down+0x1d/0x1f
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.336304] [<ffffffff810b36a1>] down+0x41/0x50
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.336381] [<ffffffffa063bc4f>] os_acquire_mutex+0x3f/0x50 [nvidia]
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.336456] [<ffffffffa061dfd8>] _nv012536rm+0x18/0x30 [nvidia]
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.336526] [<ffffffffa05cfbbd>] ? _nv014113rm+0x3d/0x120 [nvidia]
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.336594] [<ffffffffa0626ff0>] ? rm_get_gpu_uuid_raw+0x70/0x120 [nvidia]
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.336665] [<ffffffffa06321ae>] ? nvidia_open+0x74e/0x960 [nvidia]
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.336733] [<ffffffffa0630bc9>] ? nvidia_frontend_open+0x49/0xa0 [nvidia]
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.336737] [<ffffffff811c592f>] ? chrdev_open+0x9f/0x1d0
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.336740] [<ffffffff811be423>] ? do_dentry_open+0x233/0x2e0
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.336742] [<ffffffff811c5890>] ? cdev_put+0x30/0x30
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.336744] [<ffffffff811be759>] ? vfs_open+0x49/0x50
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.336747] [<ffffffff811cfd42>] ? do_last+0x562/0x1370
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.336750] [<ffffffff811d0c0b>] ? path_openat+0xbb/0x670
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.336752] [<ffffffff811d202a>] ? do_filp_open+0x3a/0x90
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.336755] [<ffffffff811dee67>] ? __alloc_fd+0xa7/0x130
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.336757] [<ffffffff811c0299>] ? do_sys_open+0x129/0x2a0
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.336760] [<ffffffff811c042e>] ? SyS_open+0x1e/0x20
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.336763] [<ffffffff8173a49d>] ? system_call_fastpath+0x1a/0x1f
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.336765] INFO: task kworker/0:2:4523 blocked for more than 120 seconds.
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.340617] Tainted: P OX 3.13.0-91-generic #138-Ubuntu
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.344226] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.348416] kworker/0:2 D ffff8803dfc13180 0 4523 2 0x00000000
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.348497] Workqueue: events os_execute_work_item [nvidia]
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.348501] ffff8803c98edb70 0000000000000046 ffff8803c5a7c800 0000000000013180
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.348503] ffff8803c98edfd8 0000000000013180 ffff8803c5a7c800 ffff8803c94f1848
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.348506] 7fffffffffffffff ffff8803c5a7c800 0000000000000002 0000000000000000
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.348508] Call Trace:
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.348513] [<ffffffff8172dd89>] schedule+0x29/0x70
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.348521] [<ffffffff8172cfd9>] schedule_timeout+0x279/0x310
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.348525] [<ffffffff810a9443>] ? find_busiest_group+0x133/0x890
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.348528] [<ffffffff81730276>] __down_common+0xf7/0x144
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.348530] [<ffffffff817302e0>] __down+0x1d/0x1f
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.348533] [<ffffffff810b36a1>] down+0x41/0x50
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.348605] [<ffffffffa063bc4f>] os_acquire_mutex+0x3f/0x50 [nvidia]
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.348677] [<ffffffffa061dfd8>] _nv012536rm+0x18/0x30 [nvidia]
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.348743] [<ffffffffa05cfbbd>] ? _nv014113rm+0x3d/0x120 [nvidia]
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.348833] [<ffffffffa0486a2d>] ? _nv013080rm+0x14d/0x260 [nvidia]
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.348899] [<ffffffffa0621d39>] ? _nv000671rm+0x229/0xb10 [nvidia]
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.348967] [<ffffffffa0624d79>] ? rm_execute_work_item+0x49/0xc0 [nvidia]
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.349036] [<ffffffffa063c196>] ? os_execute_work_item+0x46/0x70 [nvidia]
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.349039] [<ffffffff81086078>] ? process_one_work+0x178/0x470
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.349042] [<ffffffff81086e91>] ? worker_thread+0x121/0x410
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.349044] [<ffffffff81086d70>] ? rescuer_thread+0x430/0x430
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.349047] [<ffffffff8108dc79>] ? kthread+0xc9/0xe0
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.349049] [<ffffffff8108dbb0>] ? kthread_create_on_node+0x1c0/0x1c0
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.349052] [<ffffffff8173a3e8>] ? ret_from_fork+0x58/0x90
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.349054] [<ffffffff8108dbb0>] ? kthread_create_on_node+0x1c0/0x1c0
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.349056] INFO: task kworker/0:3:4524 blocked for more than 120 seconds.
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.352831] Tainted: P OX 3.13.0-91-generic #138-Ubuntu
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.356395] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.360683] kworker/0:3 D ffff8803dfc13180 0 4524 2 0x00000000
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.360770] Workqueue: events os_execute_work_item [nvidia]
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.360773] ffff8803c48ddb70 0000000000000046 ffff8803c5f6b000 0000000000013180
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.360776] ffff8803c48ddfd8 0000000000013180 ffff8803c5f6b000 ffff8803c94f1848
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.360778] 7fffffffffffffff ffff8803c5f6b000 0000000000000002 0000000000000000
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.360781] Call Trace:
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.360786] [<ffffffff8172dd89>] schedule+0x29/0x70
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.360788] [<ffffffff8172cfd9>] schedule_timeout+0x279/0x310
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.360790] [<ffffffff8172d8f4>] ? __schedule+0x384/0x7f0
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.360801] [<ffffffff81730276>] __down_common+0xf7/0x144
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.360805] [<ffffffff817302e0>] __down+0x1d/0x1f
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.360808] [<ffffffff810b36a1>] down+0x41/0x50
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.360876] [<ffffffffa063bc4f>] os_acquire_mutex+0x3f/0x50 [nvidia]
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.360946] [<ffffffffa061dfd8>] _nv012536rm+0x18/0x30 [nvidia]
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.361020] [<ffffffffa05cfbbd>] ? _nv014113rm+0x3d/0x120 [nvidia]
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.361110] [<ffffffffa0486a2d>] ? _nv013080rm+0x14d/0x260 [nvidia]
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.361177] [<ffffffffa0621d39>] ? _nv000671rm+0x229/0xb10 [nvidia]
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.361244] [<ffffffffa0624d79>] ? rm_execute_work_item+0x49/0xc0 [nvidia]
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.361313] [<ffffffffa063c196>] ? os_execute_work_item+0x46/0x70 [nvidia]
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.361317] [<ffffffff81086078>] ? process_one_work+0x178/0x470
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.361319] [<ffffffff81086857>] ? manage_workers.isra.25+0x1f7/0x2e0
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.361321] [<ffffffff81086e91>] ? worker_thread+0x121/0x410
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.361324] [<ffffffff81086d70>] ? rescuer_thread+0x430/0x430
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.361327] [<ffffffff8108dc79>] ? kthread+0xc9/0xe0
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.361329] [<ffffffff8108dbb0>] ? kthread_create_on_node+0x1c0/0x1c0
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.361332] [<ffffffff8173a3e8>] ? ret_from_fork+0x58/0x90
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.361334] [<ffffffff8108dbb0>] ? kthread_create_on_node+0x1c0/0x1c0
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.361336] INFO: task nvidia-smi:4526 blocked for more than 120 seconds.
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.365081] Tainted: P OX 3.13.0-91-generic #138-Ubuntu
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.368757] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.373067] nvidia-smi D ffff8803dfcf3180 0 4526 1 0x00000004
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.373072] ffff8803c5f2daf0 0000000000000082 ffff8803c6010000 0000000000013180
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.373075] ffff8803c5f2dfd8 0000000000013180 ffff8803c6010000 ffffffffa09188c0
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.373078] 7fffffffffffffff ffff8803c6010000 0000000000000002 0000000000000000
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.373088] Call Trace:
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.373095] [<ffffffff8172dd89>] schedule+0x29/0x70
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.373097] [<ffffffff8172cfd9>] schedule_timeout+0x279/0x310
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.373102] [<ffffffff811e4e13>] ? single_open+0x33/0xb0
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.373105] [<ffffffff81730276>] __down_common+0xf7/0x144
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.373108] [<ffffffff81094c00>] ? set_current_groups+0x60/0x70
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.373110] [<ffffffff817302e0>] __down+0x1d/0x1f
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.373114] [<ffffffff810b36a1>] down+0x41/0x50
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.373208] [<ffffffffa0630ba3>] nvidia_frontend_open+0x23/0xa0 [nvidia]
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.373213] [<ffffffff811c592f>] chrdev_open+0x9f/0x1d0
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.373216] [<ffffffff811be423>] do_dentry_open+0x233/0x2e0
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.373218] [<ffffffff811c5890>] ? cdev_put+0x30/0x30
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.373220] [<ffffffff811be759>] vfs_open+0x49/0x50
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.373223] [<ffffffff811cfd42>] do_last+0x562/0x1370
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.373226] [<ffffffff811d0c0b>] path_openat+0xbb/0x670
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.373229] [<ffffffff811d11e2>] ? final_putname+0x22/0x50
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.373231] [<ffffffff811d13e9>] ? putname+0x29/0x40
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.373234] [<ffffffff811d1f3f>] ? user_path_at_empty+0x5f/0x90
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.373244] [<ffffffff811d202a>] do_filp_open+0x3a/0x90
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.373248] [<ffffffff811dee67>] ? __alloc_fd+0xa7/0x130
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.373251] [<ffffffff811c0299>] do_sys_open+0x129/0x2a0
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.373253] [<ffffffff811c042e>] SyS_open+0x1e/0x20
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.373256] [<ffffffff8173a49d>] system_call_fastpath+0x1a/0x1f
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.373259] INFO: task kworker/0:4:4532 blocked for more than 120 seconds.
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.377095] Tainted: P OX 3.13.0-91-generic #138-Ubuntu
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.380772] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.385103] kworker/0:4 D ffff8803dfc13180 0 4532 2 0x00000000
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.385193] Workqueue: events os_execute_work_item [nvidia]
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.385197] ffff8800367e1b70 0000000000000046 ffff8803c5f6e000 0000000000013180
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.385199] ffff8800367e1fd8 0000000000013180 ffff8803c5f6e000 ffff8803c94f1848
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.385202] 7fffffffffffffff ffff8803c5f6e000 0000000000000002 0000000000000000
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.385204] Call Trace:
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.385209] [<ffffffff8172dd89>] schedule+0x29/0x70
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.385220] [<ffffffff8172cfd9>] schedule_timeout+0x279/0x310
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.385224] [<ffffffff810a9443>] ? find_busiest_group+0x133/0x890
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.385228] [<ffffffff81730276>] __down_common+0xf7/0x144
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.385233] [<ffffffff817302e0>] __down+0x1d/0x1f
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.385235] [<ffffffff810b36a1>] down+0x41/0x50
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.385304] [<ffffffffa063bc4f>] os_acquire_mutex+0x3f/0x50 [nvidia]
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.385380] [<ffffffffa061dfd8>] _nv012536rm+0x18/0x30 [nvidia]
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.385450] [<ffffffffa05cfbbd>] ? _nv014113rm+0x3d/0x120 [nvidia]
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.385541] [<ffffffffa0486a2d>] ? _nv013080rm+0x14d/0x260 [nvidia]
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.385608] [<ffffffffa0621d39>] ? _nv000671rm+0x229/0xb10 [nvidia]
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.385678] [<ffffffffa0624d79>] ? rm_execute_work_item+0x49/0xc0 [nvidia]
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.385747] [<ffffffffa063c196>] ? os_execute_work_item+0x46/0x70 [nvidia]
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.385751] [<ffffffff81086078>] ? process_one_work+0x178/0x470
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.385753] [<ffffffff81086e91>] ? worker_thread+0x121/0x410
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.385756] [<ffffffff81086d70>] ? rescuer_thread+0x430/0x430
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.385759] [<ffffffff8108dc79>] ? kthread+0xc9/0xe0
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.385761] [<ffffffff8108dbb0>] ? kthread_create_on_node+0x1c0/0x1c0
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.385764] [<ffffffff8173a3e8>] ? ret_from_fork+0x58/0x90
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.385766] [<ffffffff8108dbb0>] ? kthread_create_on_node+0x1c0/0x1c0
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.385769] INFO: task kworker/0:5:4533 blocked for more than 120 seconds.
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.389548] Tainted: P OX 3.13.0-91-generic #138-Ubuntu
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.393189] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.397526] kworker/0:5 D ffff8803dfc13180 0 4533 2 0x00000000
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.397607] Workqueue: events os_execute_work_item [nvidia]
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.397614] ffff880036575b70 0000000000000046 ffff8803c6f03000 0000000000013180
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.397616] ffff880036575fd8 0000000000013180 ffff8803c6f03000 ffff8803c94f1848
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.397619] 7fffffffffffffff ffff8803c6f03000 0000000000000002 0000000000000000
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.397621] Call Trace:
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.397625] [<ffffffff8172dd89>] schedule+0x29/0x70
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.397627] [<ffffffff8172cfd9>] schedule_timeout+0x279/0x310
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.397629] [<ffffffff8172d8f4>] ? __schedule+0x384/0x7f0
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.397639] [<ffffffff81730276>] __down_common+0xf7/0x144
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.397643] [<ffffffff817302e0>] __down+0x1d/0x1f
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.397645] [<ffffffff810b36a1>] down+0x41/0x50
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.397716] [<ffffffffa063bc4f>] os_acquire_mutex+0x3f/0x50 [nvidia]
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.397788] [<ffffffffa061dfd8>] _nv012536rm+0x18/0x30 [nvidia]
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.397860] [<ffffffffa05cfbbd>] ? _nv014113rm+0x3d/0x120 [nvidia]
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.397950] [<ffffffffa0486a2d>] ? _nv013080rm+0x14d/0x260 [nvidia]
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.398018] [<ffffffffa0621d39>] ? _nv000671rm+0x229/0xb10 [nvidia]
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.398085] [<ffffffffa0624d79>] ? rm_execute_work_item+0x49/0xc0 [nvidia]
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.398154] [<ffffffffa063c196>] ? os_execute_work_item+0x46/0x70 [nvidia]
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.398157] [<ffffffff81086078>] ? process_one_work+0x178/0x470
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.398159] [<ffffffff81086857>] ? manage_workers.isra.25+0x1f7/0x2e0
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.398162] [<ffffffff81086e91>] ? worker_thread+0x121/0x410
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.398164] [<ffffffff81086d70>] ? rescuer_thread+0x430/0x430
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.398166] [<ffffffff8108dc79>] ? kthread+0xc9/0xe0
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.398169] [<ffffffff8108dbb0>] ? kthread_create_on_node+0x1c0/0x1c0
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.398171] [<ffffffff8173a3e8>] ? ret_from_fork+0x58/0x90
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.398174] [<ffffffff8108dbb0>] ? kthread_create_on_node+0x1c0/0x1c0
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.398176] INFO: task kworker/0:6:4534 blocked for more than 120 seconds.
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.401921] Tainted: P OX 3.13.0-91-generic #138-Ubuntu
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.405540] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.409768] kworker/0:6 D ffff8803dfc13180 0 4534 2 0x00000000
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.409848] Workqueue: events os_execute_work_item [nvidia]
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.409856] ffff8800364e7b70 0000000000000046 ffff8803c6f06000 0000000000013180
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.409859] ffff8800364e7fd8 0000000000013180 ffff8803c6f06000 ffff8803c94f1848
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.409861] 7fffffffffffffff ffff8803c6f06000 0000000000000002 0000000000000000
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.409864] Call Trace:
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.409868] [<ffffffff8172dd89>] schedule+0x29/0x70
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.409870] [<ffffffff8172cfd9>] schedule_timeout+0x279/0x310
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.409881] [<ffffffff810a9443>] ? find_busiest_group+0x133/0x890
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.409884] [<ffffffff81730276>] __down_common+0xf7/0x144
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.409886] [<ffffffff817302e0>] __down+0x1d/0x1f
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.409888] [<ffffffff810b36a1>] down+0x41/0x50
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.409957] [<ffffffffa063bc4f>] os_acquire_mutex+0x3f/0x50 [nvidia]
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.410034] [<ffffffffa061dfd8>] _nv012536rm+0x18/0x30 [nvidia]
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.410104] [<ffffffffa05cfbbd>] ? _nv014113rm+0x3d/0x120 [nvidia]
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.410195] [<ffffffffa0486a2d>] ? _nv013080rm+0x14d/0x260 [nvidia]
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.410263] [<ffffffffa0621d39>] ? _nv000671rm+0x229/0xb10 [nvidia]
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.410330] [<ffffffffa0624d79>] ? rm_execute_work_item+0x49/0xc0 [nvidia]
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.410399] [<ffffffffa063c196>] ? os_execute_work_item+0x46/0x70 [nvidia]
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.410403] [<ffffffff81086078>] ? process_one_work+0x178/0x470
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.410405] [<ffffffff81086e91>] ? worker_thread+0x121/0x410
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.410408] [<ffffffff81086d70>] ? rescuer_thread+0x430/0x430
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.410410] [<ffffffff8108dc79>] ? kthread+0xc9/0xe0
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.410412] [<ffffffff8108dbb0>] ? kthread_create_on_node+0x1c0/0x1c0
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.410415] [<ffffffff8173a3e8>] ? ret_from_fork+0x58/0x90
- Jul 5 17:58:15 ip-172-31-14-73 kernel: [ 481.410417] [<ffffffff8108dbb0>] ? kthread_create_on_node+0x1c0/0x1c0
Add Comment
Please, Sign In to add comment