API tools faq
paste
Guest User

Untitled

a guest
Oct 24th, 2024
41
0
Never
Add comment
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 28.39 KB | None | 0 0
raw download clone embed print report
  1. {
  2. "date_time": "1729771361.463876",
  3. "thread_name": "",
  4. "thread_id": "266",
  5. "level": "Warning",
  6. "query_id": "",
  7. "logger_name": "signoz_logs.logs_v2 (445efee3-08d2-4db7-bf32-3af1f8b4f2df)",
  8. "message": "Directory covered-by-broken_20241014_50220_50220_0 (to detach to) already exists, but its content looks similar to content of the broken part which we are going to detach. Assuming it was already cloned to detached, will not do it again to avoid redundant copies of broken part.",
  9. "source_file": "src/Storages/MergeTree/DataPartStorageOnDiskBase.cpp; virtual std::optional<String> DB::DataPartStorageOnDiskBase::getRelativePathForPrefix(LoggerPtr, const String &, bool, bool) const",
  10. "source_line": "89"
  11. }
  12. {
  13. "date_time": "1729771361.463953",
  14. "thread_name": "",
  15. "thread_id": "266",
  16. "level": "Warning",
  17. "query_id": "",
  18. "logger_name": "signoz_logs.logs_v2 (445efee3-08d2-4db7-bf32-3af1f8b4f2df)",
  19. "message": "Directory covered-by-broken_20241014_50221_50221_0 (to detach to) already exists, but its content looks similar to content of the broken part which we are going to detach. Assuming it was already cloned to detached, will not do it again to avoid redundant copies of broken part.",
  20. "source_file": "src/Storages/MergeTree/DataPartStorageOnDiskBase.cpp; virtual std::optional<String> DB::DataPartStorageOnDiskBase::getRelativePathForPrefix(LoggerPtr, const String &, bool, bool) const",
  21. "source_line": "89"
  22. }
  23. {
  24. "date_time": "1729771361.464032",
  25. "thread_name": "",
  26. "thread_id": "266",
  27. "level": "Warning",
  28. "query_id": "",
  29. "logger_name": "signoz_logs.logs_v2 (445efee3-08d2-4db7-bf32-3af1f8b4f2df)",
  30. "message": "Directory covered-by-broken_20241014_50222_50222_0 (to detach to) already exists, but its content looks similar to content of the broken part which we are going to detach. Assuming it was already cloned to detached, will not do it again to avoid redundant copies of broken part.",
  31. "source_file": "src/Storages/MergeTree/DataPartStorageOnDiskBase.cpp; virtual std::optional<String> DB::DataPartStorageOnDiskBase::getRelativePathForPrefix(LoggerPtr, const String &, bool, bool) const",
  32. "source_line": "89"
  33. }
  34. {
  35. "date_time": "1729771361.464051",
  36. "thread_name": "",
  37. "thread_id": "266",
  38. "level": "Warning",
  39. "query_id": "",
  40. "logger_name": "signoz_logs.logs_v2 (445efee3-08d2-4db7-bf32-3af1f8b4f2df)",
  41. "message": "Detached 11 parts covered by broken part 20241014_50217_50227_2: 20241014_50217_50222_1, 20241014_50223_50223_0, 20241014_50224_50224_0, 20241014_50225_50225_0, 20241014_50226_50226_0, 20241014_50217_50217_0, 20241014_50218_50218_0, 20241014_50219_50219_0, 20241014_50220_50220_0, 20241014_50221_50221_0, 20241014_50222_50222_0",
  42. "source_file": "src/Storages/StorageReplicatedMergeTree.cpp; void DB::StorageReplicatedMergeTree::removePartAndEnqueueFetch(const String &, bool)",
  43. "source_line": "4186"
  44. }
  45. {
  46. "date_time": "1729771361.467746",
  47. "thread_name": "",
  48. "thread_id": "266",
  49. "level": "Error",
  50. "query_id": "",
  51. "logger_name": "signoz_logs.logs_v2 (ReplicatedMergeTreeRestartingThread)",
  52. "message": "Failed to restart the table. Will try again: Code: 236. DB::Exception: Cannot remove part because shutdown called. (ABORTED), Stack trace (when copying this message, always include the lines below):\n\n0. DB::Exception::Exception(DB::Exception::MessageMasked&&, int, bool) @ 0x000000000db0867b\n1. DB::Exception::Exception(PreformattedMessage&&, int) @ 0x0000000007e5e2ac\n2. DB::Exception::Exception<>(int, FormatStringHelperImpl<>) @ 0x0000000007e76acb\n3. DB::StorageReplicatedMergeTree::removePartAndEnqueueFetch(String const&, bool) @ 0x0000000012570ae8\n4. DB::ReplicatedMergeTreeQueue::createLogEntriesToFetchBrokenParts() @ 0x0000000012c989b0\n5. DB::ReplicatedMergeTreeRestartingThread::tryStartup() @ 0x0000000012cd6550\n6. DB::ReplicatedMergeTreeRestartingThread::run() @ 0x0000000012cd3f42\n7. DB::BackgroundSchedulePool::threadFunction() @ 0x0000000010deb080\n8. void std::__function::__policy_invoker<void ()>::__call_impl<std::__function::__default_alloc_func<ThreadFromGlobalPoolImpl<false, true>::ThreadFromGlobalPoolImpl<DB::BackgroundSchedulePool::BackgroundSchedulePool(unsigned long, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, char const*)::$_0>(DB::BackgroundSchedulePool::BackgroundSchedulePool(unsigned long, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, char const*)::$_0&&)::'lambda'(), void ()>>(std::__function::__policy_storage const*) @ 0x0000000010dec127\n9. void* std::__thread_proxy[abi:v15007]<std::tuple<std::unique_ptr<std::__thread_struct, std::default_delete<std::__thread_struct>>, void ThreadPoolImpl<std::thread>::scheduleImpl<void>(std::function<void ()>, Priority, std::optional<unsigned long>, bool)::'lambda0'()>>(void*) @ 0x000000000dbe0009\n10. ? @ 0x00007ff43aa3cac3\n11. ? @ 0x00007ff43aace850\n (version 24.8.5.115 (official build))",
  53. "source_file": "src/Common/Exception.cpp; void DB::tryLogCurrentExceptionImpl(Poco::Logger *, const std::string &)",
  54. "source_line": "265"
  55. }
  56. {
  57. "date_time": "1729771364.340680",
  58. "thread_name": "TCPServerConnection ([#5])",
  59. "thread_id": "785",
  60. "level": "Error",
  61. "query_id": "9e782618-1b1d-46c1-9616-2effbc42cf60",
  62. "logger_name": "executeQuery",
  63. "message": "Code: 242. DB::Exception: Table is in readonly mode: replica_path=/clickhouse/tables/445efee3-08d2-4db7-bf32-3af1f8b4f2df/0/replicas/chi-signoz-clickhouse-cluster-0-0. (TABLE_IS_READ_ONLY) (version 24.8.5.115 (official build)) (from [::ffff:10.16.94.5]:45232) (in query: INSERT INTO signoz_logs.distributed_logs_v2 ( ts_bucket_start, resource_fingerprint, timestamp, observed_timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, body, attributes_string, attributes_number, attributes_bool, resources_string, scope_name, scope_version, scope_string ) VALUES), Stack trace (when copying this message, always include the lines below):\n\n0. DB::Exception::Exception(DB::Exception::MessageMasked&&, int, bool) @ 0x000000000db0867b\n1. DB::Exception::Exception(PreformattedMessage&&, int) @ 0x0000000007e5e2ac\n2. DB::Exception::Exception<String const&>(int, FormatStringHelperImpl<std::type_identity<String const&>::type>, String const&) @ 0x0000000007e7196b\n3. DB::ReplicatedMergeTreeSinkImpl<false>::commitPart(std::shared_ptr<DB::ZooKeeperWithFaultInjection> const&, std::shared_ptr<DB::IMergeTreeDataPart>&, String const&, unsigned long)::'lambda1'()::operator()() const @ 0x0000000012cfc853\n4. DB::ReplicatedMergeTreeSinkImpl<false>::commitPart(std::shared_ptr<DB::ZooKeeperWithFaultInjection> const&, std::shared_ptr<DB::IMergeTreeDataPart>&, String const&, unsigned long) @ 0x0000000012ced871\n5. DB::ReplicatedMergeTreeSinkImpl<false>::finishDelayedChunk(std::shared_ptr<DB::ZooKeeperWithFaultInjection> const&) @ 0x0000000012cec5c6\n6. DB::ReplicatedMergeTreeSinkImpl<false>::onFinish() @ 0x0000000012ced039\n7. DB::runStep(std::function<void ()>, DB::ThreadStatus*, std::atomic<unsigned long>*) @ 0x00000000132bbb4b\n8. DB::ExceptionKeepingTransform::work() @ 0x00000000132bb44e\n9. DB::ExecutionThreadContext::executeTask() @ 0x00000000130583c7\n10. DB::PipelineExecutor::executeStepImpl(unsigned long, std::atomic<bool>*) @ 0x000000001304cc90\n11. DB::PipelineExecutor::executeStep(std::atomic<bool>*) @ 0x000000001304c6a8\n12. DB::DistributedSink::writeToLocal(DB::Cluster::ShardInfo const&, DB::Block const&, unsigned long) @ 0x00000000128b63ec\n13. DB::DistributedSink::writeAsyncImpl(DB::Block const&, unsigned long) @ 0x00000000128b34ac\n14. DB::DistributedSink::consume(DB::Chunk&) @ 0x00000000128b0326\n15. DB::SinkToStorage::onConsume(DB::Chunk) @ 0x0000000013383da4\n16. void std::__function::__policy_invoker<void ()>::__call_impl<std::__function::__default_alloc_func<DB::ExceptionKeepingTransform::work()::$_1, void ()>>(std::__function::__policy_storage const*) @ 0x00000000132bbe38\n17. DB::runStep(std::function<void ()>, DB::ThreadStatus*, std::atomic<unsigned long>*) @ 0x00000000132bbb4b\n18. DB::ExceptionKeepingTransform::work() @ 0x00000000132bb375\n19. DB::ExecutionThreadContext::executeTask() @ 0x00000000130583c7\n20. DB::PipelineExecutor::executeStepImpl(unsigned long, std::atomic<bool>*) @ 0x000000001304cc90\n21. DB::PipelineExecutor::executeStep(std::atomic<bool>*) @ 0x000000001304c6a8\n22. DB::TCPHandler::processInsertQuery() @ 0x0000000012fd31ec\n23. DB::TCPHandler::runImpl() @ 0x0000000012fc392d\n24. DB::TCPHandler::run() @ 0x0000000012fde538\n25. Poco::Net::TCPServerConnection::start() @ 0x0000000015dcadc7\n26. Poco::Net::TCPServerDispatcher::run() @ 0x0000000015dcb259\n27. Poco::PooledThread::run() @ 0x0000000015d98201\n28. Poco::ThreadImpl::runnableEntry(void*) @ 0x0000000015d967bd\n29. ? @ 0x00007ff43aa3cac3\n30. ? @ 0x00007ff43aace850\n",
  64. "source_file": "src/Interpreters/executeQuery.cpp; void DB::logException(ContextPtr, QueryLogElement &, bool)",
  65. "source_line": "220"
  66. }
  67. {
  68. "date_time": "1729771364.341020",
  69. "thread_name": "TCPServerConnection ([#5])",
  70. "thread_id": "785",
  71. "level": "Error",
  72. "query_id": "9e782618-1b1d-46c1-9616-2effbc42cf60",
  73. "logger_name": "TCPHandler",
  74. "message": "Code: 242. DB::Exception: Table is in readonly mode: replica_path=/clickhouse/tables/445efee3-08d2-4db7-bf32-3af1f8b4f2df/0/replicas/chi-signoz-clickhouse-cluster-0-0. (TABLE_IS_READ_ONLY), Stack trace (when copying this message, always include the lines below):\n\n0. DB::Exception::Exception(DB::Exception::MessageMasked&&, int, bool) @ 0x000000000db0867b\n1. DB::Exception::Exception(PreformattedMessage&&, int) @ 0x0000000007e5e2ac\n2. DB::Exception::Exception<String const&>(int, FormatStringHelperImpl<std::type_identity<String const&>::type>, String const&) @ 0x0000000007e7196b\n3. DB::ReplicatedMergeTreeSinkImpl<false>::commitPart(std::shared_ptr<DB::ZooKeeperWithFaultInjection> const&, std::shared_ptr<DB::IMergeTreeDataPart>&, String const&, unsigned long)::'lambda1'()::operator()() const @ 0x0000000012cfc853\n4. DB::ReplicatedMergeTreeSinkImpl<false>::commitPart(std::shared_ptr<DB::ZooKeeperWithFaultInjection> const&, std::shared_ptr<DB::IMergeTreeDataPart>&, String const&, unsigned long) @ 0x0000000012ced871\n5. DB::ReplicatedMergeTreeSinkImpl<false>::finishDelayedChunk(std::shared_ptr<DB::ZooKeeperWithFaultInjection> const&) @ 0x0000000012cec5c6\n6. DB::ReplicatedMergeTreeSinkImpl<false>::onFinish() @ 0x0000000012ced039\n7. DB::runStep(std::function<void ()>, DB::ThreadStatus*, std::atomic<unsigned long>*) @ 0x00000000132bbb4b\n8. DB::ExceptionKeepingTransform::work() @ 0x00000000132bb44e\n9. DB::ExecutionThreadContext::executeTask() @ 0x00000000130583c7\n10. DB::PipelineExecutor::executeStepImpl(unsigned long, std::atomic<bool>*) @ 0x000000001304cc90\n11. DB::PipelineExecutor::executeStep(std::atomic<bool>*) @ 0x000000001304c6a8\n12. DB::DistributedSink::writeToLocal(DB::Cluster::ShardInfo const&, DB::Block const&, unsigned long) @ 0x00000000128b63ec\n13. DB::DistributedSink::writeAsyncImpl(DB::Block const&, unsigned long) @ 0x00000000128b34ac\n14. DB::DistributedSink::consume(DB::Chunk&) @ 0x00000000128b0326\n15. DB::SinkToStorage::onConsume(DB::Chunk) @ 0x0000000013383da4\n16. void std::__function::__policy_invoker<void ()>::__call_impl<std::__function::__default_alloc_func<DB::ExceptionKeepingTransform::work()::$_1, void ()>>(std::__function::__policy_storage const*) @ 0x00000000132bbe38\n17. DB::runStep(std::function<void ()>, DB::ThreadStatus*, std::atomic<unsigned long>*) @ 0x00000000132bbb4b\n18. DB::ExceptionKeepingTransform::work() @ 0x00000000132bb375\n19. DB::ExecutionThreadContext::executeTask() @ 0x00000000130583c7\n20. DB::PipelineExecutor::executeStepImpl(unsigned long, std::atomic<bool>*) @ 0x000000001304cc90\n21. DB::PipelineExecutor::executeStep(std::atomic<bool>*) @ 0x000000001304c6a8\n22. DB::TCPHandler::processInsertQuery() @ 0x0000000012fd31ec\n23. DB::TCPHandler::runImpl() @ 0x0000000012fc392d\n24. DB::TCPHandler::run() @ 0x0000000012fde538\n25. Poco::Net::TCPServerConnection::start() @ 0x0000000015dcadc7\n26. Poco::Net::TCPServerDispatcher::run() @ 0x0000000015dcb259\n27. Poco::PooledThread::run() @ 0x0000000015d98201\n28. Poco::ThreadImpl::runnableEntry(void*) @ 0x0000000015d967bd\n29. ? @ 0x00007ff43aa3cac3\n30. ? @ 0x00007ff43aace850\n",
  75. "source_file": "src/Server/TCPHandler.cpp; void DB::TCPHandler::runImpl()",
  76. "source_line": "717"
  77. }
  78. {
  79. "date_time": "1729771371.467802",
  80. "thread_name": "",
  81. "thread_id": "380",
  82. "level": "Warning",
  83. "query_id": "",
  84. "logger_name": "signoz_logs.logs_v2 (ReplicatedMergeTreeRestartingThread)",
  85. "message": "Table was in readonly mode. Will try to activate it.",
  86. "source_file": "src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp; bool DB::ReplicatedMergeTreeRestartingThread::runImpl()",
  87. "source_line": "112"
  88. }
  89. {When i deleted those directories, i now get this
  90.  
  91. {
  92. "date_time": "1729771361.463876",
  93. "thread_name": "",
  94. "thread_id": "266",
  95. "level": "Warning",
  96. "query_id": "",
  97. "logger_name": "signoz_logs.logs_v2 (445efee3-08d2-4db7-bf32-3af1f8b4f2df)",
  98. "message": "Directory covered-by-broken_20241014_50220_50220_0 (to detach to) already exists, but its content looks similar to content of the broken part which we are going to detach. Assuming it was already cloned to detached, will not do it again to avoid redundant copies of broken part.",
  99. "source_file": "src/Storages/MergeTree/DataPartStorageOnDiskBase.cpp; virtual std::optional<String> DB::DataPartStorageOnDiskBase::getRelativePathForPrefix(LoggerPtr, const String &, bool, bool) const",
  100. "source_line": "89"
  101. }
  102. {
  103. "date_time": "1729771361.463953",
  104. "thread_name": "",
  105. "thread_id": "266",
  106. "level": "Warning",
  107. "query_id": "",
  108. "logger_name": "signoz_logs.logs_v2 (445efee3-08d2-4db7-bf32-3af1f8b4f2df)",
  109. "message": "Directory covered-by-broken_20241014_50221_50221_0 (to detach to) already exists, but its content looks similar to content of the broken part which we are going to detach. Assuming it was already cloned to detached, will not do it again to avoid redundant copies of broken part.",
  110. "source_file": "src/Storages/MergeTree/DataPartStorageOnDiskBase.cpp; virtual std::optional<String> DB::DataPartStorageOnDiskBase::getRelativePathForPrefix(LoggerPtr, const String &, bool, bool) const",
  111. "source_line": "89"
  112. }
  113. {
  114. "date_time": "1729771361.464032",
  115. "thread_name": "",
  116. "thread_id": "266",
  117. "level": "Warning",
  118. "query_id": "",
  119. "logger_name": "signoz_logs.logs_v2 (445efee3-08d2-4db7-bf32-3af1f8b4f2df)",
  120. "message": "Directory covered-by-broken_20241014_50222_50222_0 (to detach to) already exists, but its content looks similar to content of the broken part which we are going to detach. Assuming it was already cloned to detached, will not do it again to avoid redundant copies of broken part.",
  121. "source_file": "src/Storages/MergeTree/DataPartStorageOnDiskBase.cpp; virtual std::optional<String> DB::DataPartStorageOnDiskBase::getRelativePathForPrefix(LoggerPtr, const String &, bool, bool) const",
  122. "source_line": "89"
  123. }
  124. {
  125. "date_time": "1729771361.464051",
  126. "thread_name": "",
  127. "thread_id": "266",
  128. "level": "Warning",
  129. "query_id": "",
  130. "logger_name": "signoz_logs.logs_v2 (445efee3-08d2-4db7-bf32-3af1f8b4f2df)",
  131. "message": "Detached 11 parts covered by broken part 20241014_50217_50227_2: 20241014_50217_50222_1, 20241014_50223_50223_0, 20241014_50224_50224_0, 20241014_50225_50225_0, 20241014_50226_50226_0, 20241014_50217_50217_0, 20241014_50218_50218_0, 20241014_50219_50219_0, 20241014_50220_50220_0, 20241014_50221_50221_0, 20241014_50222_50222_0",
  132. "source_file": "src/Storages/StorageReplicatedMergeTree.cpp; void DB::StorageReplicatedMergeTree::removePartAndEnqueueFetch(const String &, bool)",
  133. "source_line": "4186"
  134. }
  135. {
  136. "date_time": "1729771361.467746",
  137. "thread_name": "",
  138. "thread_id": "266",
  139. "level": "Error",
  140. "query_id": "",
  141. "logger_name": "signoz_logs.logs_v2 (ReplicatedMergeTreeRestartingThread)",
  142. "message": "Failed to restart the table. Will try again: Code: 236. DB::Exception: Cannot remove part because shutdown called. (ABORTED), Stack trace (when copying this message, always include the lines below):\n\n0. DB::Exception::Exception(DB::Exception::MessageMasked&&, int, bool) @ 0x000000000db0867b\n1. DB::Exception::Exception(PreformattedMessage&&, int) @ 0x0000000007e5e2ac\n2. DB::Exception::Exception<>(int, FormatStringHelperImpl<>) @ 0x0000000007e76acb\n3. DB::StorageReplicatedMergeTree::removePartAndEnqueueFetch(String const&, bool) @ 0x0000000012570ae8\n4. DB::ReplicatedMergeTreeQueue::createLogEntriesToFetchBrokenParts() @ 0x0000000012c989b0\n5. DB::ReplicatedMergeTreeRestartingThread::tryStartup() @ 0x0000000012cd6550\n6. DB::ReplicatedMergeTreeRestartingThread::run() @ 0x0000000012cd3f42\n7. DB::BackgroundSchedulePool::threadFunction() @ 0x0000000010deb080\n8. void std::__function::__policy_invoker<void ()>::__call_impl<std::__function::__default_alloc_func<ThreadFromGlobalPoolImpl<false, true>::ThreadFromGlobalPoolImpl<DB::BackgroundSchedulePool::BackgroundSchedulePool(unsigned long, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, char const*)::$_0>(DB::BackgroundSchedulePool::BackgroundSchedulePool(unsigned long, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, StrongTypedef<unsigned long, CurrentMetrics::MetricTag>, char const*)::$_0&&)::'lambda'(), void ()>>(std::__function::__policy_storage const*) @ 0x0000000010dec127\n9. void* std::__thread_proxy[abi:v15007]<std::tuple<std::unique_ptr<std::__thread_struct, std::default_delete<std::__thread_struct>>, void ThreadPoolImpl<std::thread>::scheduleImpl<void>(std::function<void ()>, Priority, std::optional<unsigned long>, bool)::'lambda0'()>>(void*) @ 0x000000000dbe0009\n10. ? @ 0x00007ff43aa3cac3\n11. ? @ 0x00007ff43aace850\n (version 24.8.5.115 (official build))",
  143. "source_file": "src/Common/Exception.cpp; void DB::tryLogCurrentExceptionImpl(Poco::Logger *, const std::string &)",
  144. "source_line": "265"
  145. }
  146. {
  147. "date_time": "1729771364.340680",
  148. "thread_name": "TCPServerConnection ([#5])",
  149. "thread_id": "785",
  150. "level": "Error",
  151. "query_id": "9e782618-1b1d-46c1-9616-2effbc42cf60",
  152. "logger_name": "executeQuery",
  153. "message": "Code: 242. DB::Exception: Table is in readonly mode: replica_path=/clickhouse/tables/445efee3-08d2-4db7-bf32-3af1f8b4f2df/0/replicas/chi-signoz-clickhouse-cluster-0-0. (TABLE_IS_READ_ONLY) (version 24.8.5.115 (official build)) (from [::ffff:10.16.94.5]:45232) (in query: INSERT INTO signoz_logs.distributed_logs_v2 ( ts_bucket_start, resource_fingerprint, timestamp, observed_timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, body, attributes_string, attributes_number, attributes_bool, resources_string, scope_name, scope_version, scope_string ) VALUES), Stack trace (when copying this message, always include the lines below):\n\n0. DB::Exception::Exception(DB::Exception::MessageMasked&&, int, bool) @ 0x000000000db0867b\n1. DB::Exception::Exception(PreformattedMessage&&, int) @ 0x0000000007e5e2ac\n2. DB::Exception::Exception<String const&>(int, FormatStringHelperImpl<std::type_identity<String const&>::type>, String const&) @ 0x0000000007e7196b\n3. DB::ReplicatedMergeTreeSinkImpl<false>::commitPart(std::shared_ptr<DB::ZooKeeperWithFaultInjection> const&, std::shared_ptr<DB::IMergeTreeDataPart>&, String const&, unsigned long)::'lambda1'()::operator()() const @ 0x0000000012cfc853\n4. DB::ReplicatedMergeTreeSinkImpl<false>::commitPart(std::shared_ptr<DB::ZooKeeperWithFaultInjection> const&, std::shared_ptr<DB::IMergeTreeDataPart>&, String const&, unsigned long) @ 0x0000000012ced871\n5. DB::ReplicatedMergeTreeSinkImpl<false>::finishDelayedChunk(std::shared_ptr<DB::ZooKeeperWithFaultInjection> const&) @ 0x0000000012cec5c6\n6. DB::ReplicatedMergeTreeSinkImpl<false>::onFinish() @ 0x0000000012ced039\n7. DB::runStep(std::function<void ()>, DB::ThreadStatus*, std::atomic<unsigned long>*) @ 0x00000000132bbb4b\n8. DB::ExceptionKeepingTransform::work() @ 0x00000000132bb44e\n9. DB::ExecutionThreadContext::executeTask() @ 0x00000000130583c7\n10. DB::PipelineExecutor::executeStepImpl(unsigned long, std::atomic<bool>*) @ 0x000000001304cc90\n11. DB::PipelineExecutor::executeStep(std::atomic<bool>*) @ 0x000000001304c6a8\n12. DB::DistributedSink::writeToLocal(DB::Cluster::ShardInfo const&, DB::Block const&, unsigned long) @ 0x00000000128b63ec\n13. DB::DistributedSink::writeAsyncImpl(DB::Block const&, unsigned long) @ 0x00000000128b34ac\n14. DB::DistributedSink::consume(DB::Chunk&) @ 0x00000000128b0326\n15. DB::SinkToStorage::onConsume(DB::Chunk) @ 0x0000000013383da4\n16. void std::__function::__policy_invoker<void ()>::__call_impl<std::__function::__default_alloc_func<DB::ExceptionKeepingTransform::work()::$_1, void ()>>(std::__function::__policy_storage const*) @ 0x00000000132bbe38\n17. DB::runStep(std::function<void ()>, DB::ThreadStatus*, std::atomic<unsigned long>*) @ 0x00000000132bbb4b\n18. DB::ExceptionKeepingTransform::work() @ 0x00000000132bb375\n19. DB::ExecutionThreadContext::executeTask() @ 0x00000000130583c7\n20. DB::PipelineExecutor::executeStepImpl(unsigned long, std::atomic<bool>*) @ 0x000000001304cc90\n21. DB::PipelineExecutor::executeStep(std::atomic<bool>*) @ 0x000000001304c6a8\n22. DB::TCPHandler::processInsertQuery() @ 0x0000000012fd31ec\n23. DB::TCPHandler::runImpl() @ 0x0000000012fc392d\n24. DB::TCPHandler::run() @ 0x0000000012fde538\n25. Poco::Net::TCPServerConnection::start() @ 0x0000000015dcadc7\n26. Poco::Net::TCPServerDispatcher::run() @ 0x0000000015dcb259\n27. Poco::PooledThread::run() @ 0x0000000015d98201\n28. Poco::ThreadImpl::runnableEntry(void*) @ 0x0000000015d967bd\n29. ? @ 0x00007ff43aa3cac3\n30. ? @ 0x00007ff43aace850\n",
  154. "source_file": "src/Interpreters/executeQuery.cpp; void DB::logException(ContextPtr, QueryLogElement &, bool)",
  155. "source_line": "220"
  156. }
  157. {
  158. "date_time": "1729771364.341020",
  159. "thread_name": "TCPServerConnection ([#5])",
  160. "thread_id": "785",
  161. "level": "Error",
  162. "query_id": "9e782618-1b1d-46c1-9616-2effbc42cf60",
  163. "logger_name": "TCPHandler",
  164. "message": "Code: 242. DB::Exception: Table is in readonly mode: replica_path=/clickhouse/tables/445efee3-08d2-4db7-bf32-3af1f8b4f2df/0/replicas/chi-signoz-clickhouse-cluster-0-0. (TABLE_IS_READ_ONLY), Stack trace (when copying this message, always include the lines below):\n\n0. DB::Exception::Exception(DB::Exception::MessageMasked&&, int, bool) @ 0x000000000db0867b\n1. DB::Exception::Exception(PreformattedMessage&&, int) @ 0x0000000007e5e2ac\n2. DB::Exception::Exception<String const&>(int, FormatStringHelperImpl<std::type_identity<String const&>::type>, String const&) @ 0x0000000007e7196b\n3. DB::ReplicatedMergeTreeSinkImpl<false>::commitPart(std::shared_ptr<DB::ZooKeeperWithFaultInjection> const&, std::shared_ptr<DB::IMergeTreeDataPart>&, String const&, unsigned long)::'lambda1'()::operator()() const @ 0x0000000012cfc853\n4. DB::ReplicatedMergeTreeSinkImpl<false>::commitPart(std::shared_ptr<DB::ZooKeeperWithFaultInjection> const&, std::shared_ptr<DB::IMergeTreeDataPart>&, String const&, unsigned long) @ 0x0000000012ced871\n5. DB::ReplicatedMergeTreeSinkImpl<false>::finishDelayedChunk(std::shared_ptr<DB::ZooKeeperWithFaultInjection> const&) @ 0x0000000012cec5c6\n6. DB::ReplicatedMergeTreeSinkImpl<false>::onFinish() @ 0x0000000012ced039\n7. DB::runStep(std::function<void ()>, DB::ThreadStatus*, std::atomic<unsigned long>*) @ 0x00000000132bbb4b\n8. DB::ExceptionKeepingTransform::work() @ 0x00000000132bb44e\n9. DB::ExecutionThreadContext::executeTask() @ 0x00000000130583c7\n10. DB::PipelineExecutor::executeStepImpl(unsigned long, std::atomic<bool>*) @ 0x000000001304cc90\n11. DB::PipelineExecutor::executeStep(std::atomic<bool>*) @ 0x000000001304c6a8\n12. DB::DistributedSink::writeToLocal(DB::Cluster::ShardInfo const&, DB::Block const&, unsigned long) @ 0x00000000128b63ec\n13. DB::DistributedSink::writeAsyncImpl(DB::Block const&, unsigned long) @ 0x00000000128b34ac\n14. DB::DistributedSink::consume(DB::Chunk&) @ 0x00000000128b0326\n15. DB::SinkToStorage::onConsume(DB::Chunk) @ 0x0000000013383da4\n16. void std::__function::__policy_invoker<void ()>::__call_impl<std::__function::__default_alloc_func<DB::ExceptionKeepingTransform::work()::$_1, void ()>>(std::__function::__policy_storage const*) @ 0x00000000132bbe38\n17. DB::runStep(std::function<void ()>, DB::ThreadStatus*, std::atomic<unsigned long>*) @ 0x00000000132bbb4b\n18. DB::ExceptionKeepingTransform::work() @ 0x00000000132bb375\n19. DB::ExecutionThreadContext::executeTask() @ 0x00000000130583c7\n20. DB::PipelineExecutor::executeStepImpl(unsigned long, std::atomic<bool>*) @ 0x000000001304cc90\n21. DB::PipelineExecutor::executeStep(std::atomic<bool>*) @ 0x000000001304c6a8\n22. DB::TCPHandler::processInsertQuery() @ 0x0000000012fd31ec\n23. DB::TCPHandler::runImpl() @ 0x0000000012fc392d\n24. DB::TCPHandler::run() @ 0x0000000012fde538\n25. Poco::Net::TCPServerConnection::start() @ 0x0000000015dcadc7\n26. Poco::Net::TCPServerDispatcher::run() @ 0x0000000015dcb259\n27. Poco::PooledThread::run() @ 0x0000000015d98201\n28. Poco::ThreadImpl::runnableEntry(void*) @ 0x0000000015d967bd\n29. ? @ 0x00007ff43aa3cac3\n30. ? @ 0x00007ff43aace850\n",
  165. "source_file": "src/Server/TCPHandler.cpp; void DB::TCPHandler::runImpl()",
  166. "source_line": "717"
  167. }
  168. {
  169. "date_time": "1729771371.467802",
  170. "thread_name": "",
  171. "thread_id": "380",
  172. "level": "Warning",
  173. "query_id": "",
  174. "logger_name": "signoz_logs.logs_v2 (ReplicatedMergeTreeRestartingThread)",
  175. "message": "Table was in readonly mode. Will try to activate it.",
  176. "source_file": "src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp; bool DB::ReplicatedMergeTreeRestartingThread::runImpl()",
  177. "source_line": "112"
  178. }
  179. {
  180. "date_time": "1729771371.480485",
  181. "thread_name": "",
  182. "thread_id": "380",
  183. "level": "Warning",
  184. "query_id": "",
  185. "logger_name": "signoz_logs.logs_v2 (445efee3-08d2-4db7-bf32-3af1f8b4f2df)",
  186. "message": "Directory covered-by-broken_20241014_50217_50222_1 (to detach to) already exists, but its content looks similar to content of the broken part which we are going to detach. Assuming it was already cloned to detached, will not do it again to avoid redundant copies of broken part.",
  187. "source_file": "src/Storages/MergeTree/DataPartStorageOnDiskBase.cpp; virtual std::optional<String> DB::DataPartStorageOnDiskBase::getRelativePathForPrefix(LoggerPtr, const String &, bool, bool) const",
  188. "source_line": "89"
  189. }
  190. {
  191. "date_time": "1729771371.480607",
  192. "thread_name": "",
  193. "thread_id": "380",
  194. "level": "Warning",
  195. "query_id": "",
  196. "logger_name": "signoz_logs.logs_v2 (445efee3-08d2-4db7-bf32-3af1f8b4f2df)",
  197. "message": "Directory covered-by-broken_20241014_50223_50223_0 (to detach to) already exists, but its content looks similar to content of the broken part which we are going to detach. Assuming it was already cloned to detached, will not do it again to avoid redundant copies of broken part.",
  198. "source_file": "src/Storages/MergeTree/DataPartStorageOnDiskBase.cpp; virtual std::optional<String> DB::DataPartStorageOnDiskBase::getRelativePathForPrefix(LoggerPtr, const String &, bool, bool) const",
  199. "source_line": "89"
  200. }
  201. "date_time": "1729771371.480485",
  202. "thread_name": "",
  203. "thread_id": "380",
  204. "level": "Warning",
  205. "query_id": "",
  206. "logger_name": "signoz_logs.logs_v2 (445efee3-08d2-4db7-bf32-3af1f8b4f2df)",
  207. "message": "Directory covered-by-broken_20241014_50217_50222_1 (to detach to) already exists, but its content looks similar to content of the broken part which we are going to detach. Assuming it was already cloned to detached, will not do it again to avoid redundant copies of broken part.",
  208. "source_file": "src/Storages/MergeTree/DataPartStorageOnDiskBase.cpp; virtual std::optional<String> DB::DataPartStorageOnDiskBase::getRelativePathForPrefix(LoggerPtr, const String &, bool, bool) const",
  209. "source_line": "89"
  210. }
  211. {
  212. "date_time": "1729771371.480607",
  213. "thread_name": "",
  214. "thread_id": "380",
  215. "level": "Warning",
  216. "query_id": "",
  217. "logger_name": "signoz_logs.logs_v2 (445efee3-08d2-4db7-bf32-3af1f8b4f2df)",
  218. "message": "Directory covered-by-broken_20241014_50223_50223_0 (to detach to) already exists, but its content looks similar to content of the broken part which we are going to detach. Assuming it was already cloned to detached, will not do it again to avoid redundant copies of broken part.",
  219. "source_file": "src/Storages/MergeTree/DataPartStorageOnDiskBase.cpp; virtual std::optional<String> DB::DataPartStorageOnDiskBase::getRelativePathForPrefix(LoggerPtr, const String &, bool, bool) const",
  220. "source_line": "89"
  221. }
  222.  
Advertisement
Add Comment
Please, Sign In to add comment
Public Pastes
We use cookies for various purposes including analytics. By continuing to use Pastebin, you agree to our use of cookies as described in the Cookies Policy.  OK, I Understand
Not a member of Pastebin yet?
Sign Up, it unlocks many cool features!