API tools faq
paste
Guest User

Untitled

a guest
Sep 2nd, 2014
370
0
Never
Add comment
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 64.54 KB | None | 0 0
raw download clone embed print report
  1. INFO [o.a.c.f.j.i.AsyncJobMonitor] (API-Job-Executor-34:ctx-7da15416 job-11381) Add job-11381 into job monitoring
  2. WARN [c.c.u.d.Merovingian2] (API-Job-Executor-34:ctx-7da15416 job-11381 ctx-59d48f9d) Was unable to find lock for the key vm_instance1202 and thread id 1332628904
  3. INFO [o.a.c.f.j.i.AsyncJobMonitor] (Work-Job-Executor-61:ctx-3f930f76 job-11381/job-11382) Add job-11382 into job monitoring
  4. INFO [o.a.c.s.v.VolumeServiceImpl] (Work-Job-Executor-61:ctx-3f930f76 job-11381/job-11382 ctx-26e958a3) lock is acquired for VMTemplateStoragePool 113
  5. WARN [o.a.c.h.x.XenServerResourceNewBase] (DirectAgent-326:ctx-91e9f2dd) No event for task OpaqueRef:fdceb200-e125-991f-0628-29e14f1565ba
  6. WARN [c.c.h.x.r.XenServerStorageProcessor] (DirectAgent-326:ctx-91e9f2dd) Catch Exception java.util.concurrent.TimeoutException for template due to java.util.concurrent.TimeoutException: No event for task OpaqueRef:fdceb200-e125-991f-0628-29e14f1565ba
  7. java.util.concurrent.TimeoutException: No event for task OpaqueRef:fdceb200-e125-991f-0628-29e14f1565ba
  8. at org.apache.cloudstack.hypervisor.xenserver.XenServerResourceNewBase.waitForTask(XenServerResourceNewBase.java:124)
  9. at com.cloud.hypervisor.xen.resource.Xenserver625StorageProcessor.copyTemplateToPrimaryStorage(Xenserver625StorageProcessor.java:233)
  10. at com.cloud.storage.resource.StorageSubsystemCommandHandlerBase.execute(StorageSubsystemCommandHandlerBase.java:77)
  11. at com.cloud.storage.resource.StorageSubsystemCommandHandlerBase.handleStorageCommands(StorageSubsystemCommandHandlerBase.java:52)
  12. at com.cloud.hypervisor.xen.resource.CitrixResourceBase.executeRequest(CitrixResourceBase.java:546)
  13. at com.cloud.hypervisor.xen.resource.XenServer56Resource.executeRequest(XenServer56Resource.java:61)
  14. at com.cloud.hypervisor.xen.resource.XenServer610Resource.executeRequest(XenServer610Resource.java:102)
  15. at com.cloud.hypervisor.xen.resource.XenServer620SP1Resource.executeRequest(XenServer620SP1Resource.java:65)
  16. at com.cloud.agent.manager.DirectAgentAttache$Task.runInContext(DirectAgentAttache.java:216)
  17. at org.apache.cloudstack.managed.context.ManagedContextRunnable$1.run(ManagedContextRunnable.java:49)
  18. at org.apache.cloudstack.managed.context.impl.DefaultManagedContext$1.call(DefaultManagedContext.java:56)
  19. at org.apache.cloudstack.managed.context.impl.DefaultManagedContext.callWithContext(DefaultManagedContext.java:103)
  20. at org.apache.cloudstack.managed.context.impl.DefaultManagedContext.runWithContext(DefaultManagedContext.java:53)
  21. at org.apache.cloudstack.managed.context.ManagedContextRunnable.run(ManagedContextRunnable.java:46)
  22. at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
  23. at java.util.concurrent.FutureTask.run(FutureTask.java:262)
  24. at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:178)
  25. at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:292)
  26. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
  27. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
  28. at java.lang.Thread.run(Thread.java:745)
  29. WARN [c.c.h.x.r.CitrixResourceBase] (DirectAgent-326:ctx-91e9f2dd) Host 10.10.10.132 OpaqueRef:4b3249dd-7510-ca27-5636-aa09007bf1ff: Unable to remove SR
  30. INFO [o.a.c.s.v.VolumeServiceImpl] (Work-Job-Executor-61:ctx-3f930f76 job-11381/job-11382 ctx-26e958a3) releasing lock for VMTemplateStoragePool 113
  31. WARN [c.c.u.d.Merovingian2] (Work-Job-Executor-61:ctx-3f930f76 job-11381/job-11382 ctx-26e958a3) Was unable to find lock for the key template_spool_ref113 and thread id 153921868
  32. INFO [c.c.v.VirtualMachineManagerImpl] (Work-Job-Executor-61:ctx-3f930f76 job-11381/job-11382 ctx-26e958a3) Unable to contact resource.
  33. com.cloud.exception.StorageUnavailableException: Resource [StoragePool:9] is unreachable: Unable to create Vol[1070|vm=1202|ROOT]:Catch Exception java.util.concurrent.TimeoutException for template due to java.util.concurrent.TimeoutException: No event for task OpaqueRef:fdceb200-e125-991f-0628-29e14f1565ba
  34. at org.apache.cloudstack.engine.orchestration.VolumeOrchestrator.recreateVolume(VolumeOrchestrator.java:1203)
  35. at org.apache.cloudstack.engine.orchestration.VolumeOrchestrator.prepare(VolumeOrchestrator.java:1255)
  36. at com.cloud.vm.VirtualMachineManagerImpl.orchestrateStart(VirtualMachineManagerImpl.java:987)
  37. at com.cloud.vm.VirtualMachineManagerImpl.orchestrateStart(VirtualMachineManagerImpl.java:5190)
  38. at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
  39. at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
  40. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
  41. at java.lang.reflect.Method.invoke(Method.java:606)
  42. at com.cloud.vm.VmWorkJobHandlerProxy.handleVmWorkJob(VmWorkJobHandlerProxy.java:107)
  43. at com.cloud.vm.VirtualMachineManagerImpl.handleVmWorkJob(VirtualMachineManagerImpl.java:5335)
  44. at com.cloud.vm.VmWorkJobDispatcher.runJob(VmWorkJobDispatcher.java:102)
  45. at org.apache.cloudstack.framework.jobs.impl.AsyncJobManagerImpl$5.runInContext(AsyncJobManagerImpl.java:503)
  46. at org.apache.cloudstack.managed.context.ManagedContextRunnable$1.run(ManagedContextRunnable.java:49)
  47. at org.apache.cloudstack.managed.context.impl.DefaultManagedContext$1.call(DefaultManagedContext.java:56)
  48. at org.apache.cloudstack.managed.context.impl.DefaultManagedContext.callWithContext(DefaultManagedContext.java:103)
  49. at org.apache.cloudstack.managed.context.impl.DefaultManagedContext.runWithContext(DefaultManagedContext.java:53)
  50. at org.apache.cloudstack.managed.context.ManagedContextRunnable.run(ManagedContextRunnable.java:46)
  51. at org.apache.cloudstack.framework.jobs.impl.AsyncJobManagerImpl$5.run(AsyncJobManagerImpl.java:460)
  52. at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
  53. at java.util.concurrent.FutureTask.run(FutureTask.java:262)
  54. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
  55. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
  56. at java.lang.Thread.run(Thread.java:745)
  57. INFO [c.c.h.x.r.CitrixResourceBase] (DirectAgent-75:ctx-62587bd4) VM does not exist on XenServer2859a585-caeb-43de-904e-f11078f74469
  58. ERROR [c.c.v.VmWorkJobHandlerProxy] (Work-Job-Executor-61:ctx-3f930f76 job-11381/job-11382 ctx-26e958a3) Invocation exception, caused by: com.cloud.exception.InsufficientServerCapacityException: Unable to create a deployment for VM[User|i-6-1202-VM]Scope=interface com.cloud.dc.DataCenter; id=7
  59. INFO [c.c.v.VmWorkJobHandlerProxy] (Work-Job-Executor-61:ctx-3f930f76 job-11381/job-11382 ctx-26e958a3) Rethrow exception com.cloud.exception.InsufficientServerCapacityException: Unable to create a deployment for VM[User|i-6-1202-VM]Scope=interface com.cloud.dc.DataCenter; id=7
  60. ERROR [c.c.v.VmWorkJobDispatcher] (Work-Job-Executor-61:ctx-3f930f76 job-11381/job-11382) Unable to complete AsyncJobVO {id:11382, userId: 5, accountId: 6, instanceType: null, instanceId: null, cmd: com.cloud.vm.VmWorkStart, cmdInfo: rO0ABXNyABhjb20uY2xvdWQudm0uVm1Xb3JrU3RhcnR9cMGsvxz73gIAC0oABGRjSWRMAAZhdm9pZHN0ADBMY29tL2Nsb3VkL2RlcGxveS9EZXBsb3ltZW50UGxhbm5lciRFeGNsdWRlTGlzdDtMAAljbHVzdGVySWR0ABBMamF2YS9sYW5nL0xvbmc7TAAGaG9zdElkcQB-AAJMAAtqb3VybmFsTmFtZXQAEkxqYXZhL2xhbmcvU3RyaW5nO0wAEXBoeXNpY2FsTmV0d29ya0lkcQB-AAJMAAdwbGFubmVycQB-AANMAAVwb2RJZHEAfgACTAAGcG9vbElkcQB-AAJMAAlyYXdQYXJhbXN0AA9MamF2YS91dGlsL01hcDtMAA1yZXNlcnZhdGlvbklkcQB-AAN4cgATY29tLmNsb3VkLnZtLlZtV29ya5-ZtlbwJWdrAgAESgAJYWNjb3VudElkSgAGdXNlcklkSgAEdm1JZEwAC2hhbmRsZXJOYW1lcQB-AAN4cAAAAAAAAAAGAAAAAAAAAAUAAAAAAAAEsnQAGVZpcnR1YWxNYWNoaW5lTWFuYWdlckltcGwAAAAAAAAAB3BzcgAOamF2YS5sYW5nLkxvbmc7i-SQzI8j3wIAAUoABXZhbHVleHIAEGphdmEubGFuZy5OdW1iZXKGrJUdC5TgiwIAAHhwAAAAAAAAAAhzcQB-AAgAAAAAAAAAIXBwcHNxAH4ACAAAAAAAAAAHcHNyABFqYXZhLnV0aWwuSGFzaE1hcAUH2sHDFmDRAwACRgAKbG9hZEZhY3RvckkACXRocmVzaG9sZHhwP0AAAAAAAAx3CAAAABAAAAABdAAKVm1QYXNzd29yZHQAHHJPMEFCWFFBRG5OaGRtVmtYM0JoYzNOM2IzSmt4cA, cmdVersion: 0, status: IN_PROGRESS, processStatus: 0, resultCode: 0, result: null, initMsid: 253503110386408, completeMsid: null, lastUpdated: null, lastPolled: null, created: Tue Sep 02 11:43:33 EEST 2014}, job origin:11381
  61. com.cloud.exception.InsufficientServerCapacityException: Unable to create a deployment for VM[User|i-6-1202-VM]Scope=interface com.cloud.dc.DataCenter; id=7
  62. at com.cloud.vm.VirtualMachineManagerImpl.orchestrateStart(VirtualMachineManagerImpl.java:946)
  63. at com.cloud.vm.VirtualMachineManagerImpl.orchestrateStart(VirtualMachineManagerImpl.java:5190)
  64. at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
  65. at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
  66. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
  67. at java.lang.reflect.Method.invoke(Method.java:606)
  68. at com.cloud.vm.VmWorkJobHandlerProxy.handleVmWorkJob(VmWorkJobHandlerProxy.java:107)
  69. at com.cloud.vm.VirtualMachineManagerImpl.handleVmWorkJob(VirtualMachineManagerImpl.java:5335)
  70. at com.cloud.vm.VmWorkJobDispatcher.runJob(VmWorkJobDispatcher.java:102)
  71. at org.apache.cloudstack.framework.jobs.impl.AsyncJobManagerImpl$5.runInContext(AsyncJobManagerImpl.java:503)
  72. at org.apache.cloudstack.managed.context.ManagedContextRunnable$1.run(ManagedContextRunnable.java:49)
  73. at org.apache.cloudstack.managed.context.impl.DefaultManagedContext$1.call(DefaultManagedContext.java:56)
  74. at org.apache.cloudstack.managed.context.impl.DefaultManagedContext.callWithContext(DefaultManagedContext.java:103)
  75. at org.apache.cloudstack.managed.context.impl.DefaultManagedContext.runWithContext(DefaultManagedContext.java:53)
  76. at org.apache.cloudstack.managed.context.ManagedContextRunnable.run(ManagedContextRunnable.java:46)
  77. at org.apache.cloudstack.framework.jobs.impl.AsyncJobManagerImpl$5.run(AsyncJobManagerImpl.java:460)
  78. at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
  79. at java.util.concurrent.FutureTask.run(FutureTask.java:262)
  80. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
  81. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
  82. at java.lang.Thread.run(Thread.java:745)
  83. INFO [o.a.c.f.j.i.AsyncJobMonitor] (Work-Job-Executor-61:ctx-3f930f76 job-11381/job-11382) Remove job-11382 from job monitoring
  84. WARN [c.c.u.d.Merovingian2] (API-Job-Executor-34:ctx-7da15416 job-11381 ctx-59d48f9d) Was unable to find lock for the key vm_instance1202 and thread id 1332628904
  85. INFO [o.a.c.f.j.i.AsyncJobMonitor] (Work-Job-Executor-62:ctx-6af6e67a job-11381/job-11384) Add job-11384 into job monitoring
  86. INFO [c.c.n.o.OvsTunnelManagerImpl] (Work-Job-Executor-62:ctx-6af6e67a job-11381/job-11384 ctx-9a1f23dd) Create GRE tunnel from 35 to 33 succeeded.SUCCESS:t833-35-33(hostIP:10.10.10.130, remoteIP:10.10.10.133, bridge:xapi10,greKey:833, portName:t833-35-33)
  87. INFO [c.c.n.o.OvsTunnelManagerImpl] (Work-Job-Executor-62:ctx-6af6e67a job-11381/job-11384 ctx-9a1f23dd) Create GRE tunnel from 33 to 35 succeeded.SUCCESS:t833-33-35(hostIP:10.10.10.133, remoteIP:10.10.10.130, bridge:xapi10,greKey:833, portName:t833-33-35)
  88. WARN [o.a.c.f.j.i.AsyncJobMonitor] (Timer-1:ctx-5600f87b) Task (job-11381) has been pending for 126 seconds
  89. INFO [o.a.c.s.v.VolumeServiceImpl] (Work-Job-Executor-62:ctx-6af6e67a job-11381/job-11384 ctx-9a1f23dd) lock is acquired for VMTemplateStoragePool 114
  90. WARN [o.a.c.f.j.i.AsyncJobMonitor] (Timer-1:ctx-e1f64462) Task (job-11381) has been pending for 186 seconds
  91. ERROR [c.c.h.x.r.XenServerStorageProcessor] (DirectAgent-186:ctx-f01bf3e8) Failed to forget sr
  92. SR_BACKEND_FAILURE_40The SR scan failed [opterr=['INTERNAL_ERROR', 'Db_exn.Uniqueness_constraint_violation("VDI", "uuid", "820bf2e3-2da9-3cf4-99b6-685bbd0ba801")']]
  93. at com.xensource.xenapi.Types.checkResponse(Types.java:2006)
  94. at com.xensource.xenapi.Connection.dispatch(Connection.java:350)
  95. at com.cloud.hypervisor.xen.resource.XenServerConnectionPool$XenServerConnection.dispatch(XenServerConnectionPool.java:456)
  96. at com.xensource.xenapi.SR.scan(SR.java:1051)
  97. at com.cloud.hypervisor.xen.resource.Xenserver625StorageProcessor.createFileSR(Xenserver625StorageProcessor.java:107)
  98. at com.cloud.hypervisor.xen.resource.Xenserver625StorageProcessor.createFileSr(Xenserver625StorageProcessor.java:133)
  99. at com.cloud.hypervisor.xen.resource.Xenserver625StorageProcessor.copyTemplateToPrimaryStorage(Xenserver625StorageProcessor.java:167)
  100. at com.cloud.storage.resource.StorageSubsystemCommandHandlerBase.execute(StorageSubsystemCommandHandlerBase.java:77)
  101. at com.cloud.storage.resource.StorageSubsystemCommandHandlerBase.handleStorageCommands(StorageSubsystemCommandHandlerBase.java:52)
  102. at com.cloud.hypervisor.xen.resource.CitrixResourceBase.executeRequest(CitrixResourceBase.java:546)
  103. at com.cloud.hypervisor.xen.resource.XenServer56Resource.executeRequest(XenServer56Resource.java:61)
  104. at com.cloud.hypervisor.xen.resource.XenServer610Resource.executeRequest(XenServer610Resource.java:102)
  105. at com.cloud.hypervisor.xen.resource.XenServer620SP1Resource.executeRequest(XenServer620SP1Resource.java:65)
  106. at com.cloud.agent.manager.DirectAgentAttache$Task.runInContext(DirectAgentAttache.java:216)
  107. at org.apache.cloudstack.managed.context.ManagedContextRunnable$1.run(ManagedContextRunnable.java:49)
  108. at org.apache.cloudstack.managed.context.impl.DefaultManagedContext$1.call(DefaultManagedContext.java:56)
  109. at org.apache.cloudstack.managed.context.impl.DefaultManagedContext.callWithContext(DefaultManagedContext.java:103)
  110. at org.apache.cloudstack.managed.context.impl.DefaultManagedContext.runWithContext(DefaultManagedContext.java:53)
  111. at org.apache.cloudstack.managed.context.ManagedContextRunnable.run(ManagedContextRunnable.java:46)
  112. at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
  113. at java.util.concurrent.FutureTask.run(FutureTask.java:262)
  114. at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:178)
  115. at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:292)
  116. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
  117. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
  118. at java.lang.Thread.run(Thread.java:745)
  119. WARN [c.c.h.x.r.XenServerStorageProcessor] (DirectAgent-186:ctx-f01bf3e8) createFileSR failed! due to SR_BACKEND_FAILURE_40The SR scan failed [opterr=['INTERNAL_ERROR', 'Db_exn.Uniqueness_constraint_violation("VDI", "uuid", "820bf2e3-2da9-3cf4-99b6-685bbd0ba801")']]
  120. SR_BACKEND_FAILURE_40The SR scan failed [opterr=['INTERNAL_ERROR', 'Db_exn.Uniqueness_constraint_violation("VDI", "uuid", "820bf2e3-2da9-3cf4-99b6-685bbd0ba801")']]
  121. at com.xensource.xenapi.Types.checkResponse(Types.java:2006)
  122. at com.xensource.xenapi.Connection.dispatch(Connection.java:350)
  123. at com.cloud.hypervisor.xen.resource.XenServerConnectionPool$XenServerConnection.dispatch(XenServerConnectionPool.java:456)
  124. at com.xensource.xenapi.SR.scan(SR.java:1051)
  125. at com.cloud.hypervisor.xen.resource.Xenserver625StorageProcessor.createFileSR(Xenserver625StorageProcessor.java:107)
  126. at com.cloud.hypervisor.xen.resource.Xenserver625StorageProcessor.createFileSr(Xenserver625StorageProcessor.java:133)
  127. at com.cloud.hypervisor.xen.resource.Xenserver625StorageProcessor.copyTemplateToPrimaryStorage(Xenserver625StorageProcessor.java:167)
  128. at com.cloud.storage.resource.StorageSubsystemCommandHandlerBase.execute(StorageSubsystemCommandHandlerBase.java:77)
  129. at com.cloud.storage.resource.StorageSubsystemCommandHandlerBase.handleStorageCommands(StorageSubsystemCommandHandlerBase.java:52)
  130. at com.cloud.hypervisor.xen.resource.CitrixResourceBase.executeRequest(CitrixResourceBase.java:546)
  131. at com.cloud.hypervisor.xen.resource.XenServer56Resource.executeRequest(XenServer56Resource.java:61)
  132. at com.cloud.hypervisor.xen.resource.XenServer610Resource.executeRequest(XenServer610Resource.java:102)
  133. at com.cloud.hypervisor.xen.resource.XenServer620SP1Resource.executeRequest(XenServer620SP1Resource.java:65)
  134. at com.cloud.agent.manager.DirectAgentAttache$Task.runInContext(DirectAgentAttache.java:216)
  135. at org.apache.cloudstack.managed.context.ManagedContextRunnable$1.run(ManagedContextRunnable.java:49)
  136. at org.apache.cloudstack.managed.context.impl.DefaultManagedContext$1.call(DefaultManagedContext.java:56)
  137. at org.apache.cloudstack.managed.context.impl.DefaultManagedContext.callWithContext(DefaultManagedContext.java:103)
  138. at org.apache.cloudstack.managed.context.impl.DefaultManagedContext.runWithContext(DefaultManagedContext.java:53)
  139. at org.apache.cloudstack.managed.context.ManagedContextRunnable.run(ManagedContextRunnable.java:46)
  140. at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
  141. at java.util.concurrent.FutureTask.run(FutureTask.java:262)
  142. at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:178)
  143. at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:292)
  144. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
  145. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
  146. at java.lang.Thread.run(Thread.java:745)
  147. WARN [c.c.h.x.r.XenServerStorageProcessor] (DirectAgent-186:ctx-f01bf3e8) Catch Exception com.cloud.utils.exception.CloudRuntimeException for template due to com.cloud.utils.exception.CloudRuntimeException: createFileSR failed! due to SR_BACKEND_FAILURE_40The SR scan failed [opterr=['INTERNAL_ERROR', 'Db_exn.Uniqueness_constraint_violation("VDI", "uuid", "820bf2e3-2da9-3cf4-99b6-685bbd0ba801")']]
  148. com.cloud.utils.exception.CloudRuntimeException: createFileSR failed! due to SR_BACKEND_FAILURE_40The SR scan failed [opterr=['INTERNAL_ERROR', 'Db_exn.Uniqueness_constraint_violation("VDI", "uuid", "820bf2e3-2da9-3cf4-99b6-685bbd0ba801")']]
  149. at com.cloud.hypervisor.xen.resource.Xenserver625StorageProcessor.createFileSR(Xenserver625StorageProcessor.java:126)
  150. at com.cloud.hypervisor.xen.resource.Xenserver625StorageProcessor.createFileSr(Xenserver625StorageProcessor.java:133)
  151. at com.cloud.hypervisor.xen.resource.Xenserver625StorageProcessor.copyTemplateToPrimaryStorage(Xenserver625StorageProcessor.java:167)
  152. at com.cloud.storage.resource.StorageSubsystemCommandHandlerBase.execute(StorageSubsystemCommandHandlerBase.java:77)
  153. at com.cloud.storage.resource.StorageSubsystemCommandHandlerBase.handleStorageCommands(StorageSubsystemCommandHandlerBase.java:52)
  154. at com.cloud.hypervisor.xen.resource.CitrixResourceBase.executeRequest(CitrixResourceBase.java:546)
  155. at com.cloud.hypervisor.xen.resource.XenServer56Resource.executeRequest(XenServer56Resource.java:61)
  156. at com.cloud.hypervisor.xen.resource.XenServer610Resource.executeRequest(XenServer610Resource.java:102)
  157. at com.cloud.hypervisor.xen.resource.XenServer620SP1Resource.executeRequest(XenServer620SP1Resource.java:65)
  158. at com.cloud.agent.manager.DirectAgentAttache$Task.runInContext(DirectAgentAttache.java:216)
  159. at org.apache.cloudstack.managed.context.ManagedContextRunnable$1.run(ManagedContextRunnable.java:49)
  160. at org.apache.cloudstack.managed.context.impl.DefaultManagedContext$1.call(DefaultManagedContext.java:56)
  161. at org.apache.cloudstack.managed.context.impl.DefaultManagedContext.callWithContext(DefaultManagedContext.java:103)
  162. at org.apache.cloudstack.managed.context.impl.DefaultManagedContext.runWithContext(DefaultManagedContext.java:53)
  163. at org.apache.cloudstack.managed.context.ManagedContextRunnable.run(ManagedContextRunnable.java:46)
  164. at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
  165. at java.util.concurrent.FutureTask.run(FutureTask.java:262)
  166. at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:178)
  167. at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:292)
  168. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
  169. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
  170. at java.lang.Thread.run(Thread.java:745)
  171. Caused by: SR_BACKEND_FAILURE_40The SR scan failed [opterr=['INTERNAL_ERROR', 'Db_exn.Uniqueness_constraint_violation("VDI", "uuid", "820bf2e3-2da9-3cf4-99b6-685bbd0ba801")']]
  172. at com.xensource.xenapi.Types.checkResponse(Types.java:2006)
  173. at com.xensource.xenapi.Connection.dispatch(Connection.java:350)
  174. at com.cloud.hypervisor.xen.resource.XenServerConnectionPool$XenServerConnection.dispatch(XenServerConnectionPool.java:456)
  175. at com.xensource.xenapi.SR.scan(SR.java:1051)
  176. at com.cloud.hypervisor.xen.resource.Xenserver625StorageProcessor.createFileSR(Xenserver625StorageProcessor.java:107)
  177. ... 21 more
  178. INFO [o.a.c.s.v.VolumeServiceImpl] (Work-Job-Executor-62:ctx-6af6e67a job-11381/job-11384 ctx-9a1f23dd) releasing lock for VMTemplateStoragePool 114
  179. WARN [c.c.u.d.Merovingian2] (Work-Job-Executor-62:ctx-6af6e67a job-11381/job-11384 ctx-9a1f23dd) Was unable to find lock for the key template_spool_ref114 and thread id 1299462181
  180. INFO [c.c.v.VirtualMachineManagerImpl] (Work-Job-Executor-62:ctx-6af6e67a job-11381/job-11384 ctx-9a1f23dd) Unable to contact resource.
  181. com.cloud.exception.StorageUnavailableException: Resource [StoragePool:9] is unreachable: Unable to create Vol[1070|vm=1202|ROOT]:Catch Exception com.cloud.utils.exception.CloudRuntimeException for template due to com.cloud.utils.exception.CloudRuntimeException: createFileSR failed! due to SR_BACKEND_FAILURE_40The SR scan failed [opterr=['INTERNAL_ERROR', 'Db_exn.Uniqueness_constraint_violation("VDI", "uuid", "820bf2e3-2da9-3cf4-99b6-685bbd0ba801")']]
  182. at org.apache.cloudstack.engine.orchestration.VolumeOrchestrator.recreateVolume(VolumeOrchestrator.java:1203)
  183. at org.apache.cloudstack.engine.orchestration.VolumeOrchestrator.prepare(VolumeOrchestrator.java:1255)
  184. at com.cloud.vm.VirtualMachineManagerImpl.orchestrateStart(VirtualMachineManagerImpl.java:987)
  185. at com.cloud.vm.VirtualMachineManagerImpl.orchestrateStart(VirtualMachineManagerImpl.java:5190)
  186. at sun.reflect.GeneratedMethodAccessor507.invoke(Unknown Source)
  187. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
  188. at java.lang.reflect.Method.invoke(Method.java:606)
  189. at com.cloud.vm.VmWorkJobHandlerProxy.handleVmWorkJob(VmWorkJobHandlerProxy.java:107)
  190. at com.cloud.vm.VirtualMachineManagerImpl.handleVmWorkJob(VirtualMachineManagerImpl.java:5335)
  191. at com.cloud.vm.VmWorkJobDispatcher.runJob(VmWorkJobDispatcher.java:102)
  192. at org.apache.cloudstack.framework.jobs.impl.AsyncJobManagerImpl$5.runInContext(AsyncJobManagerImpl.java:503)
  193. at org.apache.cloudstack.managed.context.ManagedContextRunnable$1.run(ManagedContextRunnable.java:49)
  194. at org.apache.cloudstack.managed.context.impl.DefaultManagedContext$1.call(DefaultManagedContext.java:56)
  195. at org.apache.cloudstack.managed.context.impl.DefaultManagedContext.callWithContext(DefaultManagedContext.java:103)
  196. at org.apache.cloudstack.managed.context.impl.DefaultManagedContext.runWithContext(DefaultManagedContext.java:53)
  197. at org.apache.cloudstack.managed.context.ManagedContextRunnable.run(ManagedContextRunnable.java:46)
  198. at org.apache.cloudstack.framework.jobs.impl.AsyncJobManagerImpl$5.run(AsyncJobManagerImpl.java:460)
  199. at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
  200. at java.util.concurrent.FutureTask.run(FutureTask.java:262)
  201. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
  202. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
  203. at java.lang.Thread.run(Thread.java:745)
  204. INFO [c.c.h.x.r.CitrixResourceBase] (DirectAgent-127:ctx-e88f09b1) VM does not exist on XenServerc299d66d-ee90-43ef-aee2-7dbcc0426f15
  205. INFO [c.c.n.o.OvsTunnelManagerImpl] (Work-Job-Executor-62:ctx-6af6e67a job-11381/job-11384 ctx-9a1f23dd) Create GRE tunnel from 32 to 33 succeeded.SUCCESS:t833-32-33(hostIP:10.10.10.131, remoteIP:10.10.10.133, bridge:xapi10,greKey:833, portName:t833-32-33)
  206. INFO [c.c.n.o.OvsTunnelManagerImpl] (Work-Job-Executor-62:ctx-6af6e67a job-11381/job-11384 ctx-9a1f23dd) Create GRE tunnel from 33 to 32 succeeded.SUCCESS:t833-33-32(hostIP:10.10.10.133, remoteIP:10.10.10.131, bridge:xapi10,greKey:833, portName:t833-33-32)
  207. INFO [c.c.h.HighAvailabilityManagerImpl] (HA-1:ctx-74e3555e) checking health of usage server
  208. INFO [o.a.c.s.v.VolumeServiceImpl] (Work-Job-Executor-62:ctx-6af6e67a job-11381/job-11384 ctx-9a1f23dd) lock is acquired for VMTemplateStoragePool 115
  209. ERROR [c.c.h.x.r.XenServerStorageProcessor] (DirectAgent-389:ctx-63ed0240) Failed to forget sr
  210. SR_BACKEND_FAILURE_40The SR scan failed [opterr=['INTERNAL_ERROR', 'Db_exn.Uniqueness_constraint_violation("VDI", "uuid", "820bf2e3-2da9-3cf4-99b6-685bbd0ba801")']]
  211. at com.xensource.xenapi.Types.checkResponse(Types.java:2006)
  212. at com.xensource.xenapi.Connection.dispatch(Connection.java:350)
  213. at com.cloud.hypervisor.xen.resource.XenServerConnectionPool$XenServerConnection.dispatch(XenServerConnectionPool.java:456)
  214. at com.xensource.xenapi.SR.scan(SR.java:1051)
  215. at com.cloud.hypervisor.xen.resource.Xenserver625StorageProcessor.createFileSR(Xenserver625StorageProcessor.java:107)
  216. at com.cloud.hypervisor.xen.resource.Xenserver625StorageProcessor.createFileSr(Xenserver625StorageProcessor.java:133)
  217. at com.cloud.hypervisor.xen.resource.Xenserver625StorageProcessor.copyTemplateToPrimaryStorage(Xenserver625StorageProcessor.java:167)
  218. at com.cloud.storage.resource.StorageSubsystemCommandHandlerBase.execute(StorageSubsystemCommandHandlerBase.java:77)
  219. at com.cloud.storage.resource.StorageSubsystemCommandHandlerBase.handleStorageCommands(StorageSubsystemCommandHandlerBase.java:52)
  220. at com.cloud.hypervisor.xen.resource.CitrixResourceBase.executeRequest(CitrixResourceBase.java:546)
  221. at com.cloud.hypervisor.xen.resource.XenServer56Resource.executeRequest(XenServer56Resource.java:61)
  222. at com.cloud.hypervisor.xen.resource.XenServer610Resource.executeRequest(XenServer610Resource.java:102)
  223. at com.cloud.hypervisor.xen.resource.XenServer620SP1Resource.executeRequest(XenServer620SP1Resource.java:65)
  224. at com.cloud.agent.manager.DirectAgentAttache$Task.runInContext(DirectAgentAttache.java:216)
  225. at org.apache.cloudstack.managed.context.ManagedContextRunnable$1.run(ManagedContextRunnable.java:49)
  226. at org.apache.cloudstack.managed.context.impl.DefaultManagedContext$1.call(DefaultManagedContext.java:56)
  227. at org.apache.cloudstack.managed.context.impl.DefaultManagedContext.callWithContext(DefaultManagedContext.java:103)
  228. at org.apache.cloudstack.managed.context.impl.DefaultManagedContext.runWithContext(DefaultManagedContext.java:53)
  229. at org.apache.cloudstack.managed.context.ManagedContextRunnable.run(ManagedContextRunnable.java:46)
  230. at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
  231. at java.util.concurrent.FutureTask.run(FutureTask.java:262)
  232. at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:178)
  233. at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:292)
  234. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
  235. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
  236. at java.lang.Thread.run(Thread.java:745)
  237. WARN [c.c.h.x.r.XenServerStorageProcessor] (DirectAgent-389:ctx-63ed0240) createFileSR failed! due to SR_BACKEND_FAILURE_40The SR scan failed [opterr=['INTERNAL_ERROR', 'Db_exn.Uniqueness_constraint_violation("VDI", "uuid", "820bf2e3-2da9-3cf4-99b6-685bbd0ba801")']]
  238. SR_BACKEND_FAILURE_40The SR scan failed [opterr=['INTERNAL_ERROR', 'Db_exn.Uniqueness_constraint_violation("VDI", "uuid", "820bf2e3-2da9-3cf4-99b6-685bbd0ba801")']]
  239. at com.xensource.xenapi.Types.checkResponse(Types.java:2006)
  240. at com.xensource.xenapi.Connection.dispatch(Connection.java:350)
  241. at com.cloud.hypervisor.xen.resource.XenServerConnectionPool$XenServerConnection.dispatch(XenServerConnectionPool.java:456)
  242. at com.xensource.xenapi.SR.scan(SR.java:1051)
  243. at com.cloud.hypervisor.xen.resource.Xenserver625StorageProcessor.createFileSR(Xenserver625StorageProcessor.java:107)
  244. at com.cloud.hypervisor.xen.resource.Xenserver625StorageProcessor.createFileSr(Xenserver625StorageProcessor.java:133)
  245. at com.cloud.hypervisor.xen.resource.Xenserver625StorageProcessor.copyTemplateToPrimaryStorage(Xenserver625StorageProcessor.java:167)
  246. at com.cloud.storage.resource.StorageSubsystemCommandHandlerBase.execute(StorageSubsystemCommandHandlerBase.java:77)
  247. at com.cloud.storage.resource.StorageSubsystemCommandHandlerBase.handleStorageCommands(StorageSubsystemCommandHandlerBase.java:52)
  248. at com.cloud.hypervisor.xen.resource.CitrixResourceBase.executeRequest(CitrixResourceBase.java:546)
  249. at com.cloud.hypervisor.xen.resource.XenServer56Resource.executeRequest(XenServer56Resource.java:61)
  250. at com.cloud.hypervisor.xen.resource.XenServer610Resource.executeRequest(XenServer610Resource.java:102)
  251. at com.cloud.hypervisor.xen.resource.XenServer620SP1Resource.executeRequest(XenServer620SP1Resource.java:65)
  252. at com.cloud.agent.manager.DirectAgentAttache$Task.runInContext(DirectAgentAttache.java:216)
  253. at org.apache.cloudstack.managed.context.ManagedContextRunnable$1.run(ManagedContextRunnable.java:49)
  254. at org.apache.cloudstack.managed.context.impl.DefaultManagedContext$1.call(DefaultManagedContext.java:56)
  255. at org.apache.cloudstack.managed.context.impl.DefaultManagedContext.callWithContext(DefaultManagedContext.java:103)
  256. at org.apache.cloudstack.managed.context.impl.DefaultManagedContext.runWithContext(DefaultManagedContext.java:53)
  257. at org.apache.cloudstack.managed.context.ManagedContextRunnable.run(ManagedContextRunnable.java:46)
  258. at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
  259. at java.util.concurrent.FutureTask.run(FutureTask.java:262)
  260. at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:178)
  261. at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:292)
  262. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
  263. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
  264. at java.lang.Thread.run(Thread.java:745)
  265. WARN [c.c.h.x.r.XenServerStorageProcessor] (DirectAgent-389:ctx-63ed0240) Catch Exception com.cloud.utils.exception.CloudRuntimeException for template due to com.cloud.utils.exception.CloudRuntimeException: createFileSR failed! due to SR_BACKEND_FAILURE_40The SR scan failed [opterr=['INTERNAL_ERROR', 'Db_exn.Uniqueness_constraint_violation("VDI", "uuid", "820bf2e3-2da9-3cf4-99b6-685bbd0ba801")']]
  266. com.cloud.utils.exception.CloudRuntimeException: createFileSR failed! due to SR_BACKEND_FAILURE_40The SR scan failed [opterr=['INTERNAL_ERROR', 'Db_exn.Uniqueness_constraint_violation("VDI", "uuid", "820bf2e3-2da9-3cf4-99b6-685bbd0ba801")']]
  267. at com.cloud.hypervisor.xen.resource.Xenserver625StorageProcessor.createFileSR(Xenserver625StorageProcessor.java:126)
  268. at com.cloud.hypervisor.xen.resource.Xenserver625StorageProcessor.createFileSr(Xenserver625StorageProcessor.java:133)
  269. at com.cloud.hypervisor.xen.resource.Xenserver625StorageProcessor.copyTemplateToPrimaryStorage(Xenserver625StorageProcessor.java:167)
  270. at com.cloud.storage.resource.StorageSubsystemCommandHandlerBase.execute(StorageSubsystemCommandHandlerBase.java:77)
  271. at com.cloud.storage.resource.StorageSubsystemCommandHandlerBase.handleStorageCommands(StorageSubsystemCommandHandlerBase.java:52)
  272. at com.cloud.hypervisor.xen.resource.CitrixResourceBase.executeRequest(CitrixResourceBase.java:546)
  273. at com.cloud.hypervisor.xen.resource.XenServer56Resource.executeRequest(XenServer56Resource.java:61)
  274. at com.cloud.hypervisor.xen.resource.XenServer610Resource.executeRequest(XenServer610Resource.java:102)
  275. at com.cloud.hypervisor.xen.resource.XenServer620SP1Resource.executeRequest(XenServer620SP1Resource.java:65)
  276. at com.cloud.agent.manager.DirectAgentAttache$Task.runInContext(DirectAgentAttache.java:216)
  277. at org.apache.cloudstack.managed.context.ManagedContextRunnable$1.run(ManagedContextRunnable.java:49)
  278. at org.apache.cloudstack.managed.context.impl.DefaultManagedContext$1.call(DefaultManagedContext.java:56)
  279. at org.apache.cloudstack.managed.context.impl.DefaultManagedContext.callWithContext(DefaultManagedContext.java:103)
  280. at org.apache.cloudstack.managed.context.impl.DefaultManagedContext.runWithContext(DefaultManagedContext.java:53)
  281. at org.apache.cloudstack.managed.context.ManagedContextRunnable.run(ManagedContextRunnable.java:46)
  282. at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
  283. at java.util.concurrent.FutureTask.run(FutureTask.java:262)
  284. at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:178)
  285. at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:292)
  286. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
  287. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
  288. at java.lang.Thread.run(Thread.java:745)
  289. Caused by: SR_BACKEND_FAILURE_40The SR scan failed [opterr=['INTERNAL_ERROR', 'Db_exn.Uniqueness_constraint_violation("VDI", "uuid", "820bf2e3-2da9-3cf4-99b6-685bbd0ba801")']]
  290. at com.xensource.xenapi.Types.checkResponse(Types.java:2006)
  291. at com.xensource.xenapi.Connection.dispatch(Connection.java:350)
  292. at com.cloud.hypervisor.xen.resource.XenServerConnectionPool$XenServerConnection.dispatch(XenServerConnectionPool.java:456)
  293. at com.xensource.xenapi.SR.scan(SR.java:1051)
  294. at com.cloud.hypervisor.xen.resource.Xenserver625StorageProcessor.createFileSR(Xenserver625StorageProcessor.java:107)
  295. ... 21 more
  296. INFO [o.a.c.s.v.VolumeServiceImpl] (Work-Job-Executor-62:ctx-6af6e67a job-11381/job-11384 ctx-9a1f23dd) releasing lock for VMTemplateStoragePool 115
  297. WARN [c.c.u.d.Merovingian2] (Work-Job-Executor-62:ctx-6af6e67a job-11381/job-11384 ctx-9a1f23dd) Was unable to find lock for the key template_spool_ref115 and thread id 1299462181
  298. INFO [c.c.v.VirtualMachineManagerImpl] (Work-Job-Executor-62:ctx-6af6e67a job-11381/job-11384 ctx-9a1f23dd) Unable to contact resource.
  299. com.cloud.exception.StorageUnavailableException: Resource [StoragePool:9] is unreachable: Unable to create Vol[1070|vm=1202|ROOT]:Catch Exception com.cloud.utils.exception.CloudRuntimeException for template due to com.cloud.utils.exception.CloudRuntimeException: createFileSR failed! due to SR_BACKEND_FAILURE_40The SR scan failed [opterr=['INTERNAL_ERROR', 'Db_exn.Uniqueness_constraint_violation("VDI", "uuid", "820bf2e3-2da9-3cf4-99b6-685bbd0ba801")']]
  300. at org.apache.cloudstack.engine.orchestration.VolumeOrchestrator.recreateVolume(VolumeOrchestrator.java:1203)
  301. at org.apache.cloudstack.engine.orchestration.VolumeOrchestrator.prepare(VolumeOrchestrator.java:1255)
  302. at com.cloud.vm.VirtualMachineManagerImpl.orchestrateStart(VirtualMachineManagerImpl.java:987)
  303. at com.cloud.vm.VirtualMachineManagerImpl.orchestrateStart(VirtualMachineManagerImpl.java:5190)
  304. at sun.reflect.GeneratedMethodAccessor507.invoke(Unknown Source)
  305. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
  306. at java.lang.reflect.Method.invoke(Method.java:606)
  307. at com.cloud.vm.VmWorkJobHandlerProxy.handleVmWorkJob(VmWorkJobHandlerProxy.java:107)
  308. at com.cloud.vm.VirtualMachineManagerImpl.handleVmWorkJob(VirtualMachineManagerImpl.java:5335)
  309. at com.cloud.vm.VmWorkJobDispatcher.runJob(VmWorkJobDispatcher.java:102)
  310. at org.apache.cloudstack.framework.jobs.impl.AsyncJobManagerImpl$5.runInContext(AsyncJobManagerImpl.java:503)
  311. at org.apache.cloudstack.managed.context.ManagedContextRunnable$1.run(ManagedContextRunnable.java:49)
  312. at org.apache.cloudstack.managed.context.impl.DefaultManagedContext$1.call(DefaultManagedContext.java:56)
  313. at org.apache.cloudstack.managed.context.impl.DefaultManagedContext.callWithContext(DefaultManagedContext.java:103)
  314. at org.apache.cloudstack.managed.context.impl.DefaultManagedContext.runWithContext(DefaultManagedContext.java:53)
  315. at org.apache.cloudstack.managed.context.ManagedContextRunnable.run(ManagedContextRunnable.java:46)
  316. at org.apache.cloudstack.framework.jobs.impl.AsyncJobManagerImpl$5.run(AsyncJobManagerImpl.java:460)
  317. at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
  318. at java.util.concurrent.FutureTask.run(FutureTask.java:262)
  319. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
  320. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
  321. at java.lang.Thread.run(Thread.java:745)
  322. INFO [c.c.h.x.r.CitrixResourceBase] (DirectAgent-46:ctx-8b222988) VM does not exist on XenServerf54ebb75-690b-43b0-a123-5281513ad621
  323. WARN [o.a.c.f.j.i.AsyncJobMonitor] (Timer-1:ctx-0f8625f8) Task (job-11384) has been pending for 127 seconds
  324. WARN [o.a.c.f.j.i.AsyncJobMonitor] (Timer-1:ctx-0f8625f8) Task (job-11381) has been pending for 246 seconds
  325. INFO [o.a.c.s.v.VolumeServiceImpl] (Work-Job-Executor-62:ctx-6af6e67a job-11381/job-11384 ctx-9a1f23dd) lock is acquired for VMTemplateStoragePool 116
  326. WARN [o.a.c.h.x.XenServerResourceNewBase] (DirectAgent-122:ctx-b606aacb) No event for task OpaqueRef:39c3bc04-a3d5-de53-a577-467aee5ea15a
  327. WARN [c.c.h.x.r.XenServerStorageProcessor] (DirectAgent-122:ctx-b606aacb) Catch Exception java.util.concurrent.TimeoutException for template due to java.util.concurrent.TimeoutException: No event for task OpaqueRef:39c3bc04-a3d5-de53-a577-467aee5ea15a
  328. java.util.concurrent.TimeoutException: No event for task OpaqueRef:39c3bc04-a3d5-de53-a577-467aee5ea15a
  329. at org.apache.cloudstack.hypervisor.xenserver.XenServerResourceNewBase.waitForTask(XenServerResourceNewBase.java:124)
  330. at com.cloud.hypervisor.xen.resource.Xenserver625StorageProcessor.copyTemplateToPrimaryStorage(Xenserver625StorageProcessor.java:233)
  331. at com.cloud.storage.resource.StorageSubsystemCommandHandlerBase.execute(StorageSubsystemCommandHandlerBase.java:77)
  332. at com.cloud.storage.resource.StorageSubsystemCommandHandlerBase.handleStorageCommands(StorageSubsystemCommandHandlerBase.java:52)
  333. at com.cloud.hypervisor.xen.resource.CitrixResourceBase.executeRequest(CitrixResourceBase.java:546)
  334. at com.cloud.hypervisor.xen.resource.XenServer56Resource.executeRequest(XenServer56Resource.java:61)
  335. at com.cloud.hypervisor.xen.resource.XenServer610Resource.executeRequest(XenServer610Resource.java:102)
  336. at com.cloud.hypervisor.xen.resource.XenServer620SP1Resource.executeRequest(XenServer620SP1Resource.java:65)
  337. at com.cloud.agent.manager.DirectAgentAttache$Task.runInContext(DirectAgentAttache.java:216)
  338. at org.apache.cloudstack.managed.context.ManagedContextRunnable$1.run(ManagedContextRunnable.java:49)
  339. at org.apache.cloudstack.managed.context.impl.DefaultManagedContext$1.call(DefaultManagedContext.java:56)
  340. at org.apache.cloudstack.managed.context.impl.DefaultManagedContext.callWithContext(DefaultManagedContext.java:103)
  341. at org.apache.cloudstack.managed.context.impl.DefaultManagedContext.runWithContext(DefaultManagedContext.java:53)
  342. at org.apache.cloudstack.managed.context.ManagedContextRunnable.run(ManagedContextRunnable.java:46)
  343. at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
  344. at java.util.concurrent.FutureTask.run(FutureTask.java:262)
  345. at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:178)
  346. at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:292)
  347. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
  348. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
  349. at java.lang.Thread.run(Thread.java:745)
  350. WARN [c.c.h.x.r.CitrixResourceBase] (DirectAgent-122:ctx-b606aacb) Host 10.10.10.132 OpaqueRef:4b3249dd-7510-ca27-5636-aa09007bf1ff: Unable to remove SR
  351. INFO [o.a.c.s.v.VolumeServiceImpl] (Work-Job-Executor-62:ctx-6af6e67a job-11381/job-11384 ctx-9a1f23dd) releasing lock for VMTemplateStoragePool 116
  352. WARN [c.c.u.d.Merovingian2] (Work-Job-Executor-62:ctx-6af6e67a job-11381/job-11384 ctx-9a1f23dd) Was unable to find lock for the key template_spool_ref116 and thread id 1299462181
  353. INFO [c.c.v.VirtualMachineManagerImpl] (Work-Job-Executor-62:ctx-6af6e67a job-11381/job-11384 ctx-9a1f23dd) Unable to contact resource.
  354. com.cloud.exception.StorageUnavailableException: Resource [StoragePool:9] is unreachable: Unable to create Vol[1070|vm=1202|ROOT]:Catch Exception java.util.concurrent.TimeoutException for template due to java.util.concurrent.TimeoutException: No event for task OpaqueRef:39c3bc04-a3d5-de53-a577-467aee5ea15a
  355. at org.apache.cloudstack.engine.orchestration.VolumeOrchestrator.recreateVolume(VolumeOrchestrator.java:1203)
  356. at org.apache.cloudstack.engine.orchestration.VolumeOrchestrator.prepare(VolumeOrchestrator.java:1255)
  357. at com.cloud.vm.VirtualMachineManagerImpl.orchestrateStart(VirtualMachineManagerImpl.java:987)
  358. at com.cloud.vm.VirtualMachineManagerImpl.orchestrateStart(VirtualMachineManagerImpl.java:5190)
  359. at sun.reflect.GeneratedMethodAccessor507.invoke(Unknown Source)
  360. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
  361. at java.lang.reflect.Method.invoke(Method.java:606)
  362. at com.cloud.vm.VmWorkJobHandlerProxy.handleVmWorkJob(VmWorkJobHandlerProxy.java:107)
  363. at com.cloud.vm.VirtualMachineManagerImpl.handleVmWorkJob(VirtualMachineManagerImpl.java:5335)
  364. at com.cloud.vm.VmWorkJobDispatcher.runJob(VmWorkJobDispatcher.java:102)
  365. at org.apache.cloudstack.framework.jobs.impl.AsyncJobManagerImpl$5.runInContext(AsyncJobManagerImpl.java:503)
  366. at org.apache.cloudstack.managed.context.ManagedContextRunnable$1.run(ManagedContextRunnable.java:49)
  367. at org.apache.cloudstack.managed.context.impl.DefaultManagedContext$1.call(DefaultManagedContext.java:56)
  368. at org.apache.cloudstack.managed.context.impl.DefaultManagedContext.callWithContext(DefaultManagedContext.java:103)
  369. at org.apache.cloudstack.managed.context.impl.DefaultManagedContext.runWithContext(DefaultManagedContext.java:53)
  370. at org.apache.cloudstack.managed.context.ManagedContextRunnable.run(ManagedContextRunnable.java:46)
  371. at org.apache.cloudstack.framework.jobs.impl.AsyncJobManagerImpl$5.run(AsyncJobManagerImpl.java:460)
  372. at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
  373. at java.util.concurrent.FutureTask.run(FutureTask.java:262)
  374. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
  375. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
  376. at java.lang.Thread.run(Thread.java:745)
  377. INFO [c.c.h.x.r.CitrixResourceBase] (DirectAgent-54:ctx-7605fbd7) VM does not exist on XenServer2859a585-caeb-43de-904e-f11078f74469
  378. INFO [c.c.n.o.OvsTunnelManagerImpl] (Work-Job-Executor-62:ctx-6af6e67a job-11381/job-11384 ctx-9a1f23dd) Create GRE tunnel from 31 to 33 succeeded.SUCCESS:t833-31-33(hostIP:10.10.10.129, remoteIP:10.10.10.133, bridge:xapi10,greKey:833, portName:t833-31-33)
  379. INFO [c.c.n.o.OvsTunnelManagerImpl] (Work-Job-Executor-62:ctx-6af6e67a job-11381/job-11384 ctx-9a1f23dd) Create GRE tunnel from 33 to 31 succeeded.SUCCESS:t833-33-31(hostIP:10.10.10.133, remoteIP:10.10.10.129, bridge:xapi10,greKey:833, portName:t833-33-31)
  380. INFO [o.a.c.s.v.VolumeServiceImpl] (Work-Job-Executor-62:ctx-6af6e67a job-11381/job-11384 ctx-9a1f23dd) lock is acquired for VMTemplateStoragePool 117
  381. ERROR [c.c.h.x.r.XenServerStorageProcessor] (DirectAgent-220:ctx-8cfaaf99) Failed to forget sr
  382. SR_BACKEND_FAILURE_40The SR scan failed [opterr=['INTERNAL_ERROR', 'Db_exn.Uniqueness_constraint_violation("VDI", "uuid", "820bf2e3-2da9-3cf4-99b6-685bbd0ba801")']]
  383. at com.xensource.xenapi.Types.checkResponse(Types.java:2006)
  384. at com.xensource.xenapi.Connection.dispatch(Connection.java:350)
  385. at com.cloud.hypervisor.xen.resource.XenServerConnectionPool$XenServerConnection.dispatch(XenServerConnectionPool.java:456)
  386. at com.xensource.xenapi.SR.scan(SR.java:1051)
  387. at com.cloud.hypervisor.xen.resource.Xenserver625StorageProcessor.createFileSR(Xenserver625StorageProcessor.java:107)
  388. at com.cloud.hypervisor.xen.resource.Xenserver625StorageProcessor.createFileSr(Xenserver625StorageProcessor.java:133)
  389. at com.cloud.hypervisor.xen.resource.Xenserver625StorageProcessor.copyTemplateToPrimaryStorage(Xenserver625StorageProcessor.java:167)
  390. at com.cloud.storage.resource.StorageSubsystemCommandHandlerBase.execute(StorageSubsystemCommandHandlerBase.java:77)
  391. at com.cloud.storage.resource.StorageSubsystemCommandHandlerBase.handleStorageCommands(StorageSubsystemCommandHandlerBase.java:52)
  392. at com.cloud.hypervisor.xen.resource.CitrixResourceBase.executeRequest(CitrixResourceBase.java:546)
  393. at com.cloud.hypervisor.xen.resource.XenServer56Resource.executeRequest(XenServer56Resource.java:61)
  394. at com.cloud.hypervisor.xen.resource.XenServer610Resource.executeRequest(XenServer610Resource.java:102)
  395. at com.cloud.hypervisor.xen.resource.XenServer620SP1Resource.executeRequest(XenServer620SP1Resource.java:65)
  396. at com.cloud.agent.manager.DirectAgentAttache$Task.runInContext(DirectAgentAttache.java:216)
  397. at org.apache.cloudstack.managed.context.ManagedContextRunnable$1.run(ManagedContextRunnable.java:49)
  398. at org.apache.cloudstack.managed.context.impl.DefaultManagedContext$1.call(DefaultManagedContext.java:56)
  399. at org.apache.cloudstack.managed.context.impl.DefaultManagedContext.callWithContext(DefaultManagedContext.java:103)
  400. at org.apache.cloudstack.managed.context.impl.DefaultManagedContext.runWithContext(DefaultManagedContext.java:53)
  401. at org.apache.cloudstack.managed.context.ManagedContextRunnable.run(ManagedContextRunnable.java:46)
  402. at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
  403. at java.util.concurrent.FutureTask.run(FutureTask.java:262)
  404. at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:178)
  405. at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:292)
  406. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
  407. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
  408. at java.lang.Thread.run(Thread.java:745)
  409. WARN [c.c.h.x.r.XenServerStorageProcessor] (DirectAgent-220:ctx-8cfaaf99) createFileSR failed! due to SR_BACKEND_FAILURE_40The SR scan failed [opterr=['INTERNAL_ERROR', 'Db_exn.Uniqueness_constraint_violation("VDI", "uuid", "820bf2e3-2da9-3cf4-99b6-685bbd0ba801")']]
  410. SR_BACKEND_FAILURE_40The SR scan failed [opterr=['INTERNAL_ERROR', 'Db_exn.Uniqueness_constraint_violation("VDI", "uuid", "820bf2e3-2da9-3cf4-99b6-685bbd0ba801")']]
  411. at com.xensource.xenapi.Types.checkResponse(Types.java:2006)
  412. at com.xensource.xenapi.Connection.dispatch(Connection.java:350)
  413. at com.cloud.hypervisor.xen.resource.XenServerConnectionPool$XenServerConnection.dispatch(XenServerConnectionPool.java:456)
  414. at com.xensource.xenapi.SR.scan(SR.java:1051)
  415. at com.cloud.hypervisor.xen.resource.Xenserver625StorageProcessor.createFileSR(Xenserver625StorageProcessor.java:107)
  416. at com.cloud.hypervisor.xen.resource.Xenserver625StorageProcessor.createFileSr(Xenserver625StorageProcessor.java:133)
  417. at com.cloud.hypervisor.xen.resource.Xenserver625StorageProcessor.copyTemplateToPrimaryStorage(Xenserver625StorageProcessor.java:167)
  418. at com.cloud.storage.resource.StorageSubsystemCommandHandlerBase.execute(StorageSubsystemCommandHandlerBase.java:77)
  419. at com.cloud.storage.resource.StorageSubsystemCommandHandlerBase.handleStorageCommands(StorageSubsystemCommandHandlerBase.java:52)
  420. at com.cloud.hypervisor.xen.resource.CitrixResourceBase.executeRequest(CitrixResourceBase.java:546)
  421. at com.cloud.hypervisor.xen.resource.XenServer56Resource.executeRequest(XenServer56Resource.java:61)
  422. at com.cloud.hypervisor.xen.resource.XenServer610Resource.executeRequest(XenServer610Resource.java:102)
  423. at com.cloud.hypervisor.xen.resource.XenServer620SP1Resource.executeRequest(XenServer620SP1Resource.java:65)
  424. at com.cloud.agent.manager.DirectAgentAttache$Task.runInContext(DirectAgentAttache.java:216)
  425. at org.apache.cloudstack.managed.context.ManagedContextRunnable$1.run(ManagedContextRunnable.java:49)
  426. at org.apache.cloudstack.managed.context.impl.DefaultManagedContext$1.call(DefaultManagedContext.java:56)
  427. at org.apache.cloudstack.managed.context.impl.DefaultManagedContext.callWithContext(DefaultManagedContext.java:103)
  428. at org.apache.cloudstack.managed.context.impl.DefaultManagedContext.runWithContext(DefaultManagedContext.java:53)
  429. at org.apache.cloudstack.managed.context.ManagedContextRunnable.run(ManagedContextRunnable.java:46)
  430. at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
  431. at java.util.concurrent.FutureTask.run(FutureTask.java:262)
  432. at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:178)
  433. at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:292)
  434. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
  435. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
  436. at java.lang.Thread.run(Thread.java:745)
  437. WARN [c.c.h.x.r.XenServerStorageProcessor] (DirectAgent-220:ctx-8cfaaf99) Catch Exception com.cloud.utils.exception.CloudRuntimeException for template due to com.cloud.utils.exception.CloudRuntimeException: createFileSR failed! due to SR_BACKEND_FAILURE_40The SR scan failed [opterr=['INTERNAL_ERROR', 'Db_exn.Uniqueness_constraint_violation("VDI", "uuid", "820bf2e3-2da9-3cf4-99b6-685bbd0ba801")']]
  438. com.cloud.utils.exception.CloudRuntimeException: createFileSR failed! due to SR_BACKEND_FAILURE_40The SR scan failed [opterr=['INTERNAL_ERROR', 'Db_exn.Uniqueness_constraint_violation("VDI", "uuid", "820bf2e3-2da9-3cf4-99b6-685bbd0ba801")']]
  439. at com.cloud.hypervisor.xen.resource.Xenserver625StorageProcessor.createFileSR(Xenserver625StorageProcessor.java:126)
  440. at com.cloud.hypervisor.xen.resource.Xenserver625StorageProcessor.createFileSr(Xenserver625StorageProcessor.java:133)
  441. at com.cloud.hypervisor.xen.resource.Xenserver625StorageProcessor.copyTemplateToPrimaryStorage(Xenserver625StorageProcessor.java:167)
  442. at com.cloud.storage.resource.StorageSubsystemCommandHandlerBase.execute(StorageSubsystemCommandHandlerBase.java:77)
  443. at com.cloud.storage.resource.StorageSubsystemCommandHandlerBase.handleStorageCommands(StorageSubsystemCommandHandlerBase.java:52)
  444. at com.cloud.hypervisor.xen.resource.CitrixResourceBase.executeRequest(CitrixResourceBase.java:546)
  445. at com.cloud.hypervisor.xen.resource.XenServer56Resource.executeRequest(XenServer56Resource.java:61)
  446. at com.cloud.hypervisor.xen.resource.XenServer610Resource.executeRequest(XenServer610Resource.java:102)
  447. at com.cloud.hypervisor.xen.resource.XenServer620SP1Resource.executeRequest(XenServer620SP1Resource.java:65)
  448. at com.cloud.agent.manager.DirectAgentAttache$Task.runInContext(DirectAgentAttache.java:216)
  449. at org.apache.cloudstack.managed.context.ManagedContextRunnable$1.run(ManagedContextRunnable.java:49)
  450. at org.apache.cloudstack.managed.context.impl.DefaultManagedContext$1.call(DefaultManagedContext.java:56)
  451. at org.apache.cloudstack.managed.context.impl.DefaultManagedContext.callWithContext(DefaultManagedContext.java:103)
  452. at org.apache.cloudstack.managed.context.impl.DefaultManagedContext.runWithContext(DefaultManagedContext.java:53)
  453. at org.apache.cloudstack.managed.context.ManagedContextRunnable.run(ManagedContextRunnable.java:46)
  454. at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
  455. at java.util.concurrent.FutureTask.run(FutureTask.java:262)
  456. at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:178)
  457. at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:292)
  458. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
  459. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
  460. at java.lang.Thread.run(Thread.java:745)
  461. Caused by: SR_BACKEND_FAILURE_40The SR scan failed [opterr=['INTERNAL_ERROR', 'Db_exn.Uniqueness_constraint_violation("VDI", "uuid", "820bf2e3-2da9-3cf4-99b6-685bbd0ba801")']]
  462. at com.xensource.xenapi.Types.checkResponse(Types.java:2006)
  463. at com.xensource.xenapi.Connection.dispatch(Connection.java:350)
  464. at com.cloud.hypervisor.xen.resource.XenServerConnectionPool$XenServerConnection.dispatch(XenServerConnectionPool.java:456)
  465. at com.xensource.xenapi.SR.scan(SR.java:1051)
  466. at com.cloud.hypervisor.xen.resource.Xenserver625StorageProcessor.createFileSR(Xenserver625StorageProcessor.java:107)
  467. ... 21 more
  468. INFO [o.a.c.s.v.VolumeServiceImpl] (Work-Job-Executor-62:ctx-6af6e67a job-11381/job-11384 ctx-9a1f23dd) releasing lock for VMTemplateStoragePool 117
  469. WARN [c.c.u.d.Merovingian2] (Work-Job-Executor-62:ctx-6af6e67a job-11381/job-11384 ctx-9a1f23dd) Was unable to find lock for the key template_spool_ref117 and thread id 1299462181
  470. INFO [c.c.v.VirtualMachineManagerImpl] (Work-Job-Executor-62:ctx-6af6e67a job-11381/job-11384 ctx-9a1f23dd) Unable to contact resource.
  471. com.cloud.exception.StorageUnavailableException: Resource [StoragePool:9] is unreachable: Unable to create Vol[1070|vm=1202|ROOT]:Catch Exception com.cloud.utils.exception.CloudRuntimeException for template due to com.cloud.utils.exception.CloudRuntimeException: createFileSR failed! due to SR_BACKEND_FAILURE_40The SR scan failed [opterr=['INTERNAL_ERROR', 'Db_exn.Uniqueness_constraint_violation("VDI", "uuid", "820bf2e3-2da9-3cf4-99b6-685bbd0ba801")']]
  472. at org.apache.cloudstack.engine.orchestration.VolumeOrchestrator.recreateVolume(VolumeOrchestrator.java:1203)
  473. at org.apache.cloudstack.engine.orchestration.VolumeOrchestrator.prepare(VolumeOrchestrator.java:1255)
  474. at com.cloud.vm.VirtualMachineManagerImpl.orchestrateStart(VirtualMachineManagerImpl.java:987)
  475. at com.cloud.vm.VirtualMachineManagerImpl.orchestrateStart(VirtualMachineManagerImpl.java:5190)
  476. at sun.reflect.GeneratedMethodAccessor507.invoke(Unknown Source)
  477. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
  478. at java.lang.reflect.Method.invoke(Method.java:606)
  479. at com.cloud.vm.VmWorkJobHandlerProxy.handleVmWorkJob(VmWorkJobHandlerProxy.java:107)
  480. at com.cloud.vm.VirtualMachineManagerImpl.handleVmWorkJob(VirtualMachineManagerImpl.java:5335)
  481. at com.cloud.vm.VmWorkJobDispatcher.runJob(VmWorkJobDispatcher.java:102)
  482. at org.apache.cloudstack.framework.jobs.impl.AsyncJobManagerImpl$5.runInContext(AsyncJobManagerImpl.java:503)
  483. at org.apache.cloudstack.managed.context.ManagedContextRunnable$1.run(ManagedContextRunnable.java:49)
  484. at org.apache.cloudstack.managed.context.impl.DefaultManagedContext$1.call(DefaultManagedContext.java:56)
  485. at org.apache.cloudstack.managed.context.impl.DefaultManagedContext.callWithContext(DefaultManagedContext.java:103)
  486. at org.apache.cloudstack.managed.context.impl.DefaultManagedContext.runWithContext(DefaultManagedContext.java:53)
  487. at org.apache.cloudstack.managed.context.ManagedContextRunnable.run(ManagedContextRunnable.java:46)
  488. at org.apache.cloudstack.framework.jobs.impl.AsyncJobManagerImpl$5.run(AsyncJobManagerImpl.java:460)
  489. at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
  490. at java.util.concurrent.FutureTask.run(FutureTask.java:262)
  491. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
  492. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
  493. at java.lang.Thread.run(Thread.java:745)
  494. INFO [c.c.h.x.r.CitrixResourceBase] (DirectAgent-53:ctx-98b563c7) VM does not exist on XenServer88b9dd9b-9c98-4d63-867e-7bd43b0a8553
  495. INFO [c.c.n.o.OvsTunnelManagerImpl] (Work-Job-Executor-62:ctx-6af6e67a job-11381/job-11384 ctx-9a1f23dd) Create GRE tunnel from 34 to 33 succeeded.SUCCESS:t833-34-33(hostIP:10.10.10.132, remoteIP:10.10.10.133, bridge:xapi10,greKey:833, portName:t833-34-33)
  496. INFO [c.c.n.o.OvsTunnelManagerImpl] (Work-Job-Executor-62:ctx-6af6e67a job-11381/job-11384 ctx-9a1f23dd) Create GRE tunnel from 33 to 34 succeeded.SUCCESS:t833-33-34(hostIP:10.10.10.133, remoteIP:10.10.10.132, bridge:xapi10,greKey:833, portName:t833-33-34)
  497. WARN [o.a.c.f.j.i.AsyncJobMonitor] (Timer-1:ctx-d72bff6c) Task (job-11384) has been pending for 187 seconds
  498. WARN [o.a.c.f.j.i.AsyncJobMonitor] (Timer-1:ctx-d72bff6c) Task (job-11381) has been pending for 306 seconds
  499. INFO [o.a.c.s.v.VolumeServiceImpl] (Work-Job-Executor-62:ctx-6af6e67a job-11381/job-11384 ctx-9a1f23dd) lock is acquired for VMTemplateStoragePool 118
  500. INFO [o.a.c.s.v.VolumeServiceImpl] (Work-Job-Executor-62:ctx-6af6e67a job-11381/job-11384 ctx-9a1f23dd) releasing lock for VMTemplateStoragePool 118
  501. WARN [c.c.u.d.Merovingian2] (Work-Job-Executor-62:ctx-6af6e67a job-11381/job-11384 ctx-9a1f23dd) Was unable to find lock for the key template_spool_ref118 and thread id 1299462181
  502. INFO [c.c.v.VirtualMachineManagerImpl] (Work-Job-Executor-62:ctx-6af6e67a job-11381/job-11384 ctx-9a1f23dd) Unable to contact resource.
  503. com.cloud.exception.StorageUnavailableException: Resource [StoragePool:9] is unreachable: Unable to create Vol[1070|vm=1202|ROOT]:Can't find template VDI under: 10.10.10.137:/vol/cloud_1_sec_vol_01092014_170509/template/tmpl/1/5
  504. at org.apache.cloudstack.engine.orchestration.VolumeOrchestrator.recreateVolume(VolumeOrchestrator.java:1203)
  505. at org.apache.cloudstack.engine.orchestration.VolumeOrchestrator.prepare(VolumeOrchestrator.java:1255)
  506. at com.cloud.vm.VirtualMachineManagerImpl.orchestrateStart(VirtualMachineManagerImpl.java:987)
  507. at com.cloud.vm.VirtualMachineManagerImpl.orchestrateStart(VirtualMachineManagerImpl.java:5190)
  508. at sun.reflect.GeneratedMethodAccessor507.invoke(Unknown Source)
  509. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
  510. at java.lang.reflect.Method.invoke(Method.java:606)
  511. at com.cloud.vm.VmWorkJobHandlerProxy.handleVmWorkJob(VmWorkJobHandlerProxy.java:107)
  512. at com.cloud.vm.VirtualMachineManagerImpl.handleVmWorkJob(VirtualMachineManagerImpl.java:5335)
  513. at com.cloud.vm.VmWorkJobDispatcher.runJob(VmWorkJobDispatcher.java:102)
  514. at org.apache.cloudstack.framework.jobs.impl.AsyncJobManagerImpl$5.runInContext(AsyncJobManagerImpl.java:503)
  515. at org.apache.cloudstack.managed.context.ManagedContextRunnable$1.run(ManagedContextRunnable.java:49)
  516. at org.apache.cloudstack.managed.context.impl.DefaultManagedContext$1.call(DefaultManagedContext.java:56)
  517. at org.apache.cloudstack.managed.context.impl.DefaultManagedContext.callWithContext(DefaultManagedContext.java:103)
  518. at org.apache.cloudstack.managed.context.impl.DefaultManagedContext.runWithContext(DefaultManagedContext.java:53)
  519. at org.apache.cloudstack.managed.context.ManagedContextRunnable.run(ManagedContextRunnable.java:46)
  520. at org.apache.cloudstack.framework.jobs.impl.AsyncJobManagerImpl$5.run(AsyncJobManagerImpl.java:460)
  521. at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
  522. at java.util.concurrent.FutureTask.run(FutureTask.java:262)
  523. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
  524. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
  525. at java.lang.Thread.run(Thread.java:745)
  526. INFO [c.c.h.x.r.CitrixResourceBase] (DirectAgent-252:ctx-b74eed1e) VM does not exist on XenServer5f20e70d-3a85-4e53-911d-ea493ee06e26
  527. ERROR [c.c.v.VmWorkJobHandlerProxy] (Work-Job-Executor-62:ctx-6af6e67a job-11381/job-11384 ctx-9a1f23dd) Invocation exception, caused by: com.cloud.exception.InsufficientServerCapacityException: Unable to create a deployment for VM[User|i-6-1202-VM]Scope=interface com.cloud.dc.DataCenter; id=7
  528. INFO [c.c.v.VmWorkJobHandlerProxy] (Work-Job-Executor-62:ctx-6af6e67a job-11381/job-11384 ctx-9a1f23dd) Rethrow exception com.cloud.exception.InsufficientServerCapacityException: Unable to create a deployment for VM[User|i-6-1202-VM]Scope=interface com.cloud.dc.DataCenter; id=7
  529. ERROR [c.c.v.VmWorkJobDispatcher] (Work-Job-Executor-62:ctx-6af6e67a job-11381/job-11384) Unable to complete AsyncJobVO {id:11384, userId: 5, accountId: 6, instanceType: null, instanceId: null, cmd: com.cloud.vm.VmWorkStart, cmdInfo: rO0ABXNyABhjb20uY2xvdWQudm0uVm1Xb3JrU3RhcnR9cMGsvxz73gIAC0oABGRjSWRMAAZhdm9pZHN0ADBMY29tL2Nsb3VkL2RlcGxveS9EZXBsb3ltZW50UGxhbm5lciRFeGNsdWRlTGlzdDtMAAljbHVzdGVySWR0ABBMamF2YS9sYW5nL0xvbmc7TAAGaG9zdElkcQB-AAJMAAtqb3VybmFsTmFtZXQAEkxqYXZhL2xhbmcvU3RyaW5nO0wAEXBoeXNpY2FsTmV0d29ya0lkcQB-AAJMAAdwbGFubmVycQB-AANMAAVwb2RJZHEAfgACTAAGcG9vbElkcQB-AAJMAAlyYXdQYXJhbXN0AA9MamF2YS91dGlsL01hcDtMAA1yZXNlcnZhdGlvbklkcQB-AAN4cgATY29tLmNsb3VkLnZtLlZtV29ya5-ZtlbwJWdrAgAESgAJYWNjb3VudElkSgAGdXNlcklkSgAEdm1JZEwAC2hhbmRsZXJOYW1lcQB-AAN4cAAAAAAAAAAGAAAAAAAAAAUAAAAAAAAEsnQAGVZpcnR1YWxNYWNoaW5lTWFuYWdlckltcGwAAAAAAAAAAHBwcHBwcHBwc3IAEWphdmEudXRpbC5IYXNoTWFwBQfawcMWYNEDAAJGAApsb2FkRmFjdG9ySQAJdGhyZXNob2xkeHA_QAAAAAAADHcIAAAAEAAAAAF0AApWbVBhc3N3b3JkdAAcck8wQUJYUUFEbk5oZG1Wa1gzQmhjM04zYjNKa3hw, cmdVersion: 0, status: IN_PROGRESS, processStatus: 0, resultCode: 0, result: null, initMsid: 253503110386408, completeMsid: null, lastUpdated: null, lastPolled: null, created: Tue Sep 02 11:45:29 EEST 2014}, job origin:11381
  530. com.cloud.exception.InsufficientServerCapacityException: Unable to create a deployment for VM[User|i-6-1202-VM]Scope=interface com.cloud.dc.DataCenter; id=7
  531. at com.cloud.vm.VirtualMachineManagerImpl.orchestrateStart(VirtualMachineManagerImpl.java:946)
  532. at com.cloud.vm.VirtualMachineManagerImpl.orchestrateStart(VirtualMachineManagerImpl.java:5190)
  533. at sun.reflect.GeneratedMethodAccessor507.invoke(Unknown Source)
  534. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
  535. at java.lang.reflect.Method.invoke(Method.java:606)
  536. at com.cloud.vm.VmWorkJobHandlerProxy.handleVmWorkJob(VmWorkJobHandlerProxy.java:107)
  537. at com.cloud.vm.VirtualMachineManagerImpl.handleVmWorkJob(VirtualMachineManagerImpl.java:5335)
  538. at com.cloud.vm.VmWorkJobDispatcher.runJob(VmWorkJobDispatcher.java:102)
  539. at org.apache.cloudstack.framework.jobs.impl.AsyncJobManagerImpl$5.runInContext(AsyncJobManagerImpl.java:503)
  540. at org.apache.cloudstack.managed.context.ManagedContextRunnable$1.run(ManagedContextRunnable.java:49)
  541. at org.apache.cloudstack.managed.context.impl.DefaultManagedContext$1.call(DefaultManagedContext.java:56)
  542. at org.apache.cloudstack.managed.context.impl.DefaultManagedContext.callWithContext(DefaultManagedContext.java:103)
  543. at org.apache.cloudstack.managed.context.impl.DefaultManagedContext.runWithContext(DefaultManagedContext.java:53)
  544. at org.apache.cloudstack.managed.context.ManagedContextRunnable.run(ManagedContextRunnable.java:46)
  545. at org.apache.cloudstack.framework.jobs.impl.AsyncJobManagerImpl$5.run(AsyncJobManagerImpl.java:460)
  546. at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
  547. at java.util.concurrent.FutureTask.run(FutureTask.java:262)
  548. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
  549. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
  550. at java.lang.Thread.run(Thread.java:745)
  551. INFO [o.a.c.f.j.i.AsyncJobMonitor] (Work-Job-Executor-62:ctx-6af6e67a job-11381/job-11384) Remove job-11384 from job monitoring
  552. WARN [o.a.c.alerts] (API-Job-Executor-34:ctx-7da15416 job-11381 ctx-59d48f9d) alertType:: 8 // dataCenterId:: 7 // podId:: 7 // clusterId:: null // message:: Failed to deploy Vm with Id: 1202, on Host with Id: null
  553. ERROR [c.c.a.ApiAsyncJobDispatcher] (API-Job-Executor-34:ctx-7da15416 job-11381) Unexpected exception while executing org.apache.cloudstack.api.command.admin.vm.DeployVMCmdByAdmin
  554. java.lang.RuntimeException: Job failed due to exception Unable to create a deployment for VM[User|i-6-1202-VM]
  555. at com.cloud.vm.VmWorkJobDispatcher.runJob(VmWorkJobDispatcher.java:114)
  556. at org.apache.cloudstack.framework.jobs.impl.AsyncJobManagerImpl$5.runInContext(AsyncJobManagerImpl.java:503)
  557. at org.apache.cloudstack.managed.context.ManagedContextRunnable$1.run(ManagedContextRunnable.java:49)
  558. at org.apache.cloudstack.managed.context.impl.DefaultManagedContext$1.call(DefaultManagedContext.java:56)
  559. at org.apache.cloudstack.managed.context.impl.DefaultManagedContext.callWithContext(DefaultManagedContext.java:103)
  560. at org.apache.cloudstack.managed.context.impl.DefaultManagedContext.runWithContext(DefaultManagedContext.java:53)
  561. at org.apache.cloudstack.managed.context.ManagedContextRunnable.run(ManagedContextRunnable.java:46)
  562. at org.apache.cloudstack.framework.jobs.impl.AsyncJobManagerImpl$5.run(AsyncJobManagerImpl.java:460)
  563. at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
  564. at java.util.concurrent.FutureTask.run(FutureTask.java:262)
  565. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
  566. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
  567. at java.lang.Thread.run(Thread.java:745)
  568. INFO [o.a.c.f.j.i.AsyncJobMonitor] (API-Job-Executor-34:ctx-7da15416 job-11381) Remove job-11381 from job monitoring
Advertisement
Add Comment
Please, Sign In to add comment
Public Pastes
We use cookies for various purposes including analytics. By continuing to use Pastebin, you agree to our use of cookies as described in the Cookies Policy.  OK, I Understand
Not a member of Pastebin yet?
Sign Up, it unlocks many cool features!