API tools faq
paste
Guest User

build_fix.patch

a guest
Feb 12th, 2025
54
0
Never
Add comment
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 101.25 KB | None | 0 0
raw download clone embed print report
  1. diff --git a/contest/grader/grader.cpp b/contest/grader/grader.cpp
  2. index 7c309179..99676f64 100644
  3. --- a/contest/grader/grader.cpp
  4. +++ b/contest/grader/grader.cpp
  5. @@ -38,6 +38,8 @@ class ContestGrader : public td::actor::Actor {
  6. }
  7.  
  8. void start_up() override {
  9. + vm::detail::push_thread_pool::tpool_init();
  10. +
  11. vm::init_vm().ensure();
  12. scan_tests_dir();
  13. run_next_test();
  14. @@ -262,4 +264,4 @@ int main(int argc, char* argv[]) {
  15. }
  16.  
  17. return 0;
  18. -}
  19. \ No newline at end of file
  20. +}
  21. diff --git a/contest/solution/contest-validate-query.cpp b/contest/solution/contest-validate-query.cpp
  22. index 3559f64d..c57533b3 100644
  23. --- a/contest/solution/contest-validate-query.cpp
  24. +++ b/contest/solution/contest-validate-query.cpp
  25. @@ -1799,7 +1799,11 @@ bool ContestValidateQuery::unpack_precheck_value_flow(Ref<vm::Cell> value_flow_r
  26. return reject_query("ValueFlow of block "s + id_.to_str() +
  27. " is invalid (non-zero fees_imported in a non-masterchain block)");
  28. }
  29. - auto accounts_extra = ps_.account_dict_->get_root_extra();
  30. + Ref<vm::CellSlice> accounts_extra;
  31. + {
  32. + WriteLockGuard<NonRecursiveRWLock> lock(account_dict_mtx);
  33. + accounts_extra = ps_.account_dict_->get_root_extra();
  34. + }
  35. block::CurrencyCollection cc;
  36. if (!(accounts_extra.write().advance(5) && cc.unpack(std::move(accounts_extra)))) {
  37. return reject_query("cannot unpack CurrencyCollection from the root of old accounts dictionary");
  38. @@ -1848,8 +1852,11 @@ bool ContestValidateQuery::compute_minted_amount(block::CurrencyCollection& to_m
  39. bool ContestValidateQuery::postcheck_one_account_update(td::ConstBitPtr acc_id, Ref<vm::CellSlice> old_value,
  40. Ref<vm::CellSlice> new_value) {
  41. LOG(DEBUG) << "checking update of account " << acc_id.to_hex(256);
  42. - old_value = ps_.account_dict_->extract_value(std::move(old_value));
  43. - new_value = ns_.account_dict_->extract_value(std::move(new_value));
  44. + {
  45. + //WriteLockGuard<NonRecursiveRWLock> lock(account_dict_mtx);
  46. + old_value = ps_.account_dict_->extract_value(std::move(old_value));
  47. + new_value = ns_.account_dict_->extract_value(std::move(new_value));
  48. + }
  49. auto acc_blk_root = account_blocks_dict_->lookup(acc_id, 256);
  50. if (acc_blk_root.is_null()) {
  51. return reject_query("the state of account "s + acc_id.to_hex(256) +
  52. @@ -1896,16 +1903,19 @@ bool ContestValidateQuery::postcheck_one_account_update(td::ConstBitPtr acc_id,
  53. bool ContestValidateQuery::postcheck_account_updates() {
  54. LOG(INFO) << "pre-checking all Account updates between the old and the new state";
  55. try {
  56. - CHECK(ps_.account_dict_ && ns_.account_dict_);
  57. - if (!ps_.account_dict_->scan_diff(
  58. - *ns_.account_dict_,
  59. - [this](td::ConstBitPtr key, int key_len, Ref<vm::CellSlice> old_val_extra,
  60. - Ref<vm::CellSlice> new_val_extra) {
  61. - CHECK(key_len == 256);
  62. - return postcheck_one_account_update(key, std::move(old_val_extra), std::move(new_val_extra));
  63. - },
  64. - 2 /* check augmentation of changed nodes in the new dict */)) {
  65. - return reject_query("invalid ShardAccounts dictionary in the new state");
  66. + {
  67. + WriteLockGuard<NonRecursiveRWLock> lock(account_dict_mtx);
  68. + CHECK(ps_.account_dict_ && ns_.account_dict_);
  69. + if (!ps_.account_dict_->scan_diff(
  70. + *ns_.account_dict_,
  71. + [this](td::ConstBitPtr key, int key_len, Ref<vm::CellSlice> old_val_extra,
  72. + Ref<vm::CellSlice> new_val_extra) {
  73. + CHECK(key_len == 256);
  74. + return postcheck_one_account_update(key, std::move(old_val_extra), std::move(new_val_extra));
  75. + },
  76. + 2 /* check augmentation of changed nodes in the new dict */)) {
  77. + return reject_query("invalid ShardAccounts dictionary in the new state");
  78. + }
  79. }
  80. } catch (vm::VmError& err) {
  81. return reject_query("invalid ShardAccount dictionary difference between the old and the new state: "s +
  82. @@ -2021,8 +2031,11 @@ bool ContestValidateQuery::precheck_one_account_block(td::ConstBitPtr acc_id, Re
  83. acc_blk.account_addr.to_hex());
  84. }
  85. block::tlb::ShardAccount::Record old_state;
  86. - if (!old_state.unpack(ps_.account_dict_->lookup(acc_id, 256))) {
  87. - return reject_query("cannot extract Account from the ShardAccount of "s + acc_id.to_hex(256));
  88. + {
  89. + WriteLockGuard<NonRecursiveRWLock> lock(account_dict_mtx);
  90. + if (!old_state.unpack(ps_.account_dict_->lookup(acc_id, 256))) {
  91. + return reject_query("cannot extract Account from the ShardAccount of "s + acc_id.to_hex(256));
  92. + }
  93. }
  94. if (hash_upd.old_hash != old_state.account->get_hash().bits()) {
  95. return reject_query("(HASH_UPDATE Account) from the AccountBlock of "s + acc_id.to_hex(256) +
  96. @@ -4459,8 +4472,16 @@ std::unique_ptr<block::Account> ContestValidateQuery::make_account_from(td::Cons
  97. * Returns nullptr if an error occured.
  98. */
  99. std::unique_ptr<block::Account> ContestValidateQuery::unpack_account(td::ConstBitPtr addr) {
  100. - auto dict_entry = ps_.account_dict_->lookup_extra(addr, 256);
  101. - auto new_acc = make_account_from(addr, std::move(dict_entry.first));
  102. + Ref<vm::CellSlice> efirst;
  103. + {
  104. + std::pair<Ref<vm::CellSlice>, Ref<vm::CellSlice>> dict_entry;
  105. +
  106. + WriteLockGuard<NonRecursiveRWLock> lock(account_dict_mtx);
  107. + dict_entry = ps_.account_dict_->lookup_extra(addr, 256);
  108. + efirst = std::move(dict_entry.first);
  109. + }
  110. +
  111. + auto new_acc = make_account_from(addr, efirst);
  112. if (!new_acc) {
  113. reject_query("cannot load state of account "s + addr.to_hex(256) + " from previous shardchain state");
  114. return {};
  115. @@ -4473,6 +4494,19 @@ std::unique_ptr<block::Account> ContestValidateQuery::unpack_account(td::ConstBi
  116. return new_acc;
  117. }
  118.  
  119. +
  120. +
  121. +/*void ContestValidateQuery::check_one_transaction_wrapper(block::Account& account, ton::LogicalTime lt, Ref<vm::Cell> trans_root,
  122. + bool is_first, bool is_last)
  123. +{
  124. + //here set flag when finished, set "error" flag is check_one_transaction returned false
  125. +
  126. + if(check_one_transaction(account, lt, trans_root, is_first, is_last) == false)
  127. + {
  128. +
  129. + }
  130. +}*/
  131. +
  132. /**
  133. * Checks the validity of a single transaction for a given account.
  134. * Performs transaction execution.
  135. @@ -4505,18 +4539,18 @@ bool ContestValidateQuery::check_one_transaction(block::Account& account, ton::L
  136. if (in_msg_root.not_null()) {
  137. auto in_descr_cs = in_msg_dict_->lookup(in_msg_root->get_hash().as_bitslice());
  138. if (in_descr_cs.is_null()) {
  139. - return reject_query(PSTRING() << "inbound message with hash " << in_msg_root->get_hash().to_hex()
  140. + return false;/*return reject_query(PSTRING() << "inbound message with hash " << in_msg_root->get_hash().to_hex()
  141. << " of transaction " << lt << " of account " << addr.to_hex()
  142. - << " does not have a corresponding InMsg record");
  143. + << " does not have a corresponding InMsg record");*/
  144. }
  145. auto in_msg_tag = block::gen::t_InMsg.get_tag(*in_descr_cs);
  146. if (in_msg_tag != block::gen::InMsg::msg_import_ext && in_msg_tag != block::gen::InMsg::msg_import_fin &&
  147. in_msg_tag != block::gen::InMsg::msg_import_imm && in_msg_tag != block::gen::InMsg::msg_import_ihr &&
  148. in_msg_tag != block::gen::InMsg::msg_import_deferred_fin) {
  149. - return reject_query(PSTRING() << "inbound message with hash " << in_msg_root->get_hash().to_hex()
  150. + return false;/*return reject_query(PSTRING() << "inbound message with hash " << in_msg_root->get_hash().to_hex()
  151. << " of transaction " << lt << " of account " << addr.to_hex()
  152. << " has an invalid InMsg record (not one of msg_import_ext, msg_import_fin, "
  153. - "msg_import_imm, msg_import_ihr or msg_import_deferred_fin)");
  154. + "msg_import_imm, msg_import_ihr or msg_import_deferred_fin)");*/
  155. }
  156. is_special_tx = is_special_in_msg(*in_descr_cs);
  157. // once we know there is a InMsg with correct hash, we already know that it contains a message with this hash (by the verification of InMsg), so it is our message
  158. @@ -4532,18 +4566,18 @@ bool ContestValidateQuery::check_one_transaction(block::Account& account, ton::L
  159. block::gen::CommonMsgInfo::Record_int_msg_info info;
  160. CHECK(tlb::unpack_cell_inexact(in_msg_root, info));
  161. if (info.created_lt >= lt) {
  162. - return reject_query(PSTRING() << "transaction " << lt << " of " << addr.to_hex()
  163. + return false;/*return reject_query(PSTRING() << "transaction " << lt << " of " << addr.to_hex()
  164. << " processed inbound message created later at logical time "
  165. - << info.created_lt);
  166. + << info.created_lt);*/
  167. }
  168. LogicalTime emitted_lt = info.created_lt; // See ContestValidateQuery::check_message_processing_order
  169. if (in_msg_tag == block::gen::InMsg::msg_import_imm || in_msg_tag == block::gen::InMsg::msg_import_fin ||
  170. in_msg_tag == block::gen::InMsg::msg_import_deferred_fin) {
  171. block::tlb::MsgEnvelope::Record_std msg_env;
  172. if (!block::tlb::unpack_cell(in_descr_cs->prefetch_ref(), msg_env)) {
  173. - return reject_query(PSTRING() << "InMsg record for inbound message with hash "
  174. + return false;/*return reject_query(PSTRING() << "InMsg record for inbound message with hash "
  175. << in_msg_root->get_hash().to_hex() << " of transaction " << lt
  176. - << " of account " << addr.to_hex() << " does not have a valid MsgEnvelope");
  177. + << " of account " << addr.to_hex() << " does not have a valid MsgEnvelope");*/
  178. }
  179. in_msg_metadata = std::move(msg_env.metadata);
  180. if (msg_env.emitted_lt) {
  181. @@ -4565,15 +4599,15 @@ bool ContestValidateQuery::check_one_transaction(block::Account& account, ton::L
  182. StdSmcAddress d_addr;
  183. CHECK(block::tlb::t_MsgAddressInt.extract_std_address(dest, d_wc, d_addr));
  184. if (d_wc != workchain() || d_addr != addr) {
  185. - return reject_query(PSTRING() << "inbound message of transaction " << lt << " of account " << addr.to_hex()
  186. - << " has a different destination address " << d_wc << ":" << d_addr.to_hex());
  187. + return false;/*return reject_query(PSTRING() << "inbound message of transaction " << lt << " of account " << addr.to_hex()
  188. + << " has a different destination address " << d_wc << ":" << d_addr.to_hex());*/
  189. }
  190. auto in_msg_trans = in_descr_cs->prefetch_ref(1); // trans:^Transaction
  191. CHECK(in_msg_trans.not_null());
  192. if (in_msg_trans->get_hash() != trans_root->get_hash()) {
  193. - return reject_query(PSTRING() << "InMsg record for inbound message with hash " << in_msg_root->get_hash().to_hex()
  194. + return false;/*return reject_query(PSTRING() << "InMsg record for inbound message with hash " << in_msg_root->get_hash().to_hex()
  195. << " of transaction " << lt << " of account " << addr.to_hex()
  196. - << " refers to a different processing transaction");
  197. + << " refers to a different processing transaction");*/
  198. }
  199. }
  200. // check output messages
  201. @@ -4590,20 +4624,25 @@ bool ContestValidateQuery::check_one_transaction(block::Account& account, ton::L
  202. for (int i = 0; i < trans.outmsg_cnt; i++) {
  203. auto out_msg_root = out_dict.lookup_ref(td::BitArray<15>{i});
  204. CHECK(out_msg_root.not_null()); // we have pre-checked this
  205. - auto out_descr_cs = out_msg_dict_->lookup(out_msg_root->get_hash().as_bitslice());
  206. +
  207. + Ref<vm::CellSlice> out_descr_cs;
  208. + {
  209. + WriteLockGuard<NonRecursiveRWLock> lock(out_msg_dict_mtx);
  210. + out_descr_cs = out_msg_dict_->lookup(out_msg_root->get_hash().as_bitslice());
  211. + }
  212. if (out_descr_cs.is_null()) {
  213. - return reject_query(PSTRING() << "outbound message #" << i + 1 << " with hash "
  214. + return false;/*return reject_query(PSTRING() << "outbound message #" << i + 1 << " with hash "
  215. << out_msg_root->get_hash().to_hex() << " of transaction " << lt << " of account "
  216. - << addr.to_hex() << " does not have a corresponding OutMsg record");
  217. + << addr.to_hex() << " does not have a corresponding OutMsg record");*/
  218. }
  219. auto tag = block::gen::t_OutMsg.get_tag(*out_descr_cs);
  220. if (tag != block::gen::OutMsg::msg_export_ext && tag != block::gen::OutMsg::msg_export_new &&
  221. tag != block::gen::OutMsg::msg_export_imm && tag != block::gen::OutMsg::msg_export_new_defer) {
  222. - return reject_query(PSTRING() << "outbound message #" << i + 1 << " with hash "
  223. + return false;/*return reject_query(PSTRING() << "outbound message #" << i + 1 << " with hash "
  224. << out_msg_root->get_hash().to_hex() << " of transaction " << lt << " of account "
  225. << addr.to_hex()
  226. << " has an invalid OutMsg record (not one of msg_export_ext, msg_export_new, "
  227. - "msg_export_imm or msg_export_new_defer)");
  228. + "msg_export_imm or msg_export_new_defer)");*/
  229. }
  230. // once we know there is an OutMsg with correct hash, we already know that it contains a message with this hash
  231. // (by the verification of OutMsg), so it is our message
  232. @@ -4631,46 +4670,46 @@ bool ContestValidateQuery::check_one_transaction(block::Account& account, ton::L
  233. CHECK(msg_export_value.is_valid());
  234. money_exported += msg_export_value;
  235. if (msg_env.metadata != new_msg_metadata) {
  236. - return reject_query(PSTRING() << "outbound message #" << i + 1 << " with hash "
  237. + return false;/*return reject_query(PSTRING() << "outbound message #" << i + 1 << " with hash "
  238. << out_msg_root->get_hash().to_hex() << " of transaction " << lt << " of account "
  239. << addr.to_hex() << " has invalid metadata in an OutMsg record: expected "
  240. << (new_msg_metadata ? new_msg_metadata.value().to_str() : "<none>") << ", found "
  241. - << (msg_env.metadata ? msg_env.metadata.value().to_str() : "<none>"));
  242. + << (msg_env.metadata ? msg_env.metadata.value().to_str() : "<none>"));*/
  243. }
  244. }
  245. WorkchainId s_wc;
  246. StdSmcAddress ss_addr; // s_addr is some macros in Windows
  247. CHECK(block::tlb::t_MsgAddressInt.extract_std_address(src, s_wc, ss_addr));
  248. if (s_wc != workchain() || ss_addr != addr) {
  249. - return reject_query(PSTRING() << "outbound message #" << i + 1 << " of transaction " << lt << " of account "
  250. + return false;/*return reject_query(PSTRING() << "outbound message #" << i + 1 << " of transaction " << lt << " of account "
  251. << addr.to_hex() << " has a different source address " << s_wc << ":"
  252. - << ss_addr.to_hex());
  253. + << ss_addr.to_hex());*/
  254. }
  255. auto out_msg_trans = out_descr_cs->prefetch_ref(1); // trans:^Transaction
  256. CHECK(out_msg_trans.not_null());
  257. if (out_msg_trans->get_hash() != trans_root->get_hash()) {
  258. - return reject_query(PSTRING() << "OutMsg record for outbound message #" << i + 1 << " with hash "
  259. + return false;/*return reject_query(PSTRING() << "OutMsg record for outbound message #" << i + 1 << " with hash "
  260. << out_msg_root->get_hash().to_hex() << " of transaction " << lt << " of account "
  261. - << addr.to_hex() << " refers to a different processing transaction");
  262. + << addr.to_hex() << " refers to a different processing transaction");*/
  263. }
  264. if (tag != block::gen::OutMsg::msg_export_ext) {
  265. bool is_deferred = tag == block::gen::OutMsg::msg_export_new_defer;
  266. if (account_expected_defer_all_messages_.count(ss_addr) && !is_deferred) {
  267. - return reject_query(
  268. + return false;/*return reject_query(
  269. PSTRING() << "outbound message #" << i + 1 << " on account " << workchain() << ":" << ss_addr.to_hex()
  270. - << " must be deferred because this account has earlier messages in DispatchQueue");
  271. + << " must be deferred because this account has earlier messages in DispatchQueue");*/
  272. }
  273. if (is_deferred) {
  274. LOG(INFO) << "message from account " << workchain() << ":" << ss_addr.to_hex() << " with lt " << message_lt
  275. << " was deferred";
  276. if (!deferring_messages_enabled_ && !account_expected_defer_all_messages_.count(ss_addr)) {
  277. - return reject_query(PSTRING() << "outbound message #" << i + 1 << " on account " << workchain() << ":"
  278. - << ss_addr.to_hex() << " is deferred, but deferring messages is disabled");
  279. + return false;/*return reject_query(PSTRING() << "outbound message #" << i + 1 << " on account " << workchain() << ":"
  280. + << ss_addr.to_hex() << " is deferred, but deferring messages is disabled");*/
  281. }
  282. if (i == 0 && !account_expected_defer_all_messages_.count(ss_addr)) {
  283. - return reject_query(PSTRING() << "outbound message #1 on account " << workchain() << ":" << ss_addr.to_hex()
  284. + return false;/*return reject_query(PSTRING() << "outbound message #1 on account " << workchain() << ":" << ss_addr.to_hex()
  285. << " must not be deferred (the first message cannot be deferred unless some "
  286. - "prevoius messages are deferred)");
  287. + "prevoius messages are deferred)");*/
  288. }
  289. account_expected_defer_all_messages_.insert(ss_addr);
  290. }
  291. @@ -4686,45 +4725,45 @@ bool ContestValidateQuery::check_one_transaction(block::Account& account, ton::L
  292. bool split = (tag == block::gen::TransactionDescr::trans_split_prepare ||
  293. tag == block::gen::TransactionDescr::trans_split_install);
  294. if (split && !before_split_) {
  295. - return reject_query(PSTRING() << "transaction " << lt << " of account " << addr.to_hex()
  296. - << " is a split prepare/install transaction, but this block is not before a split");
  297. + return false;/*return reject_query(PSTRING() << "transaction " << lt << " of account " << addr.to_hex()
  298. + << " is a split prepare/install transaction, but this block is not before a split");*/
  299. }
  300. if (split && !is_last) {
  301. - return reject_query(PSTRING() << "transaction " << lt << " of account " << addr.to_hex()
  302. + return false;/*return reject_query(PSTRING() << "transaction " << lt << " of account " << addr.to_hex()
  303. << " is a split prepare/install transaction, but it is not the last transaction "
  304. - "for this account in this block");
  305. + "for this account in this block");*/
  306. }
  307. if (!split && !after_merge_) {
  308. - return reject_query(
  309. + return false;/*return reject_query(
  310. PSTRING() << "transaction " << lt << " of account " << addr.to_hex()
  311. - << " is a merge prepare/install transaction, but this block is not immediately after a merge");
  312. + << " is a merge prepare/install transaction, but this block is not immediately after a merge");*/
  313. }
  314. if (!split && !is_first) {
  315. - return reject_query(PSTRING() << "transaction " << lt << " of account " << addr.to_hex()
  316. + return false;/*return reject_query(PSTRING() << "transaction " << lt << " of account " << addr.to_hex()
  317. << " is a merge prepare/install transaction, but it is not the first transaction "
  318. - "for this account in this block");
  319. + "for this account in this block");*/
  320. }
  321. // check later a global configuration flag in config_.global_flags_
  322. // (for now, split/merge transactions are always globally disabled)
  323. - return reject_query(PSTRING() << "transaction " << lt << " of account " << addr.to_hex()
  324. - << " is a split/merge prepare/install transaction, which are globally disabled");
  325. + return false;/*return reject_query(PSTRING() << "transaction " << lt << " of account " << addr.to_hex()
  326. + << " is a split/merge prepare/install transaction, which are globally disabled");*/
  327. }
  328. if (tag == block::gen::TransactionDescr::trans_tick_tock) {
  329. - return reject_query(PSTRING() << "transaction " << lt << " of account " << addr.to_hex()
  330. - << " is a tick-tock transaction, which is impossible outside a masterchain block");
  331. + return false;/*return reject_query(PSTRING() << "transaction " << lt << " of account " << addr.to_hex()
  332. + << " is a tick-tock transaction, which is impossible outside a masterchain block");*/
  333. }
  334. if (tag == block::gen::TransactionDescr::trans_storage && !is_first) {
  335. - return reject_query(
  336. + return false;/*return reject_query(
  337. PSTRING() << "transaction " << lt << " of account " << addr.to_hex()
  338. - << " is a storage transaction, but it is not the first transaction for this account in this block");
  339. + << " is a storage transaction, but it is not the first transaction for this account in this block");*/
  340. }
  341. // check that the original account state has correct hash
  342. CHECK(account.total_state.not_null());
  343. if (hash_upd.old_hash != account.total_state->get_hash().bits()) {
  344. - return reject_query(PSTRING() << "transaction " << lt << " of account " << addr.to_hex()
  345. + return false;/*return reject_query(PSTRING() << "transaction " << lt << " of account " << addr.to_hex()
  346. << " claims that the original account state hash must be "
  347. << hash_upd.old_hash.to_hex() << " but the actual value is "
  348. - << account.total_state->get_hash().to_hex());
  349. + << account.total_state->get_hash().to_hex());*/
  350. }
  351. // some type-specific checks
  352. int trans_type = block::transaction::Transaction::tr_none;
  353. @@ -4732,8 +4771,8 @@ bool ContestValidateQuery::check_one_transaction(block::Account& account, ton::L
  354. case block::gen::TransactionDescr::trans_ord: {
  355. trans_type = block::transaction::Transaction::tr_ord;
  356. if (in_msg_root.is_null()) {
  357. - return reject_query(PSTRING() << "ordinary transaction " << lt << " of account " << addr.to_hex()
  358. - << " has no inbound message");
  359. + return false;/*return reject_query(PSTRING() << "ordinary transaction " << lt << " of account " << addr.to_hex()
  360. + << " has no inbound message");*/
  361. }
  362. need_credit_phase = !external;
  363. break;
  364. @@ -4741,78 +4780,78 @@ bool ContestValidateQuery::check_one_transaction(block::Account& account, ton::L
  365. case block::gen::TransactionDescr::trans_storage: {
  366. trans_type = block::transaction::Transaction::tr_storage;
  367. if (in_msg_root.not_null()) {
  368. - return reject_query(PSTRING() << "storage transaction " << lt << " of account " << addr.to_hex()
  369. - << " has an inbound message");
  370. + return false;/*return reject_query(PSTRING() << "storage transaction " << lt << " of account " << addr.to_hex()
  371. + << " has an inbound message");*/
  372. }
  373. if (trans.outmsg_cnt) {
  374. - return reject_query(PSTRING() << "storage transaction " << lt << " of account " << addr.to_hex()
  375. - << " has at least one outbound message");
  376. + return false;/*return reject_query(PSTRING() << "storage transaction " << lt << " of account " << addr.to_hex()
  377. + << " has at least one outbound message");*/
  378. }
  379. // FIXME
  380. - return reject_query(PSTRING() << "unable to verify storage transaction " << lt << " of account "
  381. - << addr.to_hex());
  382. + return false;/*return reject_query(PSTRING() << "unable to verify storage transaction " << lt << " of account "
  383. + << addr.to_hex());*/
  384. break;
  385. }
  386. case block::gen::TransactionDescr::trans_tick_tock: {
  387. bool is_tock = (td_cs.prefetch_ulong(4) & 1);
  388. trans_type = is_tock ? block::transaction::Transaction::tr_tock : block::transaction::Transaction::tr_tick;
  389. if (in_msg_root.not_null()) {
  390. - return reject_query(PSTRING() << (is_tock ? "tock" : "tick") << " transaction " << lt << " of account "
  391. - << addr.to_hex() << " has an inbound message");
  392. + return false;/*return reject_query(PSTRING() << (is_tock ? "tock" : "tick") << " transaction " << lt << " of account "
  393. + << addr.to_hex() << " has an inbound message");*/
  394. }
  395. break;
  396. }
  397. case block::gen::TransactionDescr::trans_merge_prepare: {
  398. trans_type = block::transaction::Transaction::tr_merge_prepare;
  399. if (in_msg_root.not_null()) {
  400. - return reject_query(PSTRING() << "merge prepare transaction " << lt << " of account " << addr.to_hex()
  401. - << " has an inbound message");
  402. + return false;/*return reject_query(PSTRING() << "merge prepare transaction " << lt << " of account " << addr.to_hex()
  403. + << " has an inbound message");*/
  404. }
  405. if (trans.outmsg_cnt != 1) {
  406. - return reject_query(PSTRING() << "merge prepare transaction " << lt << " of account " << addr.to_hex()
  407. - << " must have exactly one outbound message");
  408. + return false;/*return reject_query(PSTRING() << "merge prepare transaction " << lt << " of account " << addr.to_hex()
  409. + << " must have exactly one outbound message");*/
  410. }
  411. // FIXME
  412. - return reject_query(PSTRING() << "unable to verify merge prepare transaction " << lt << " of account "
  413. - << addr.to_hex());
  414. + return false;/*return reject_query(PSTRING() << "unable to verify merge prepare transaction " << lt << " of account "
  415. + << addr.to_hex());*/
  416. break;
  417. }
  418. case block::gen::TransactionDescr::trans_merge_install: {
  419. trans_type = block::transaction::Transaction::tr_merge_install;
  420. if (in_msg_root.is_null()) {
  421. - return reject_query(PSTRING() << "merge install transaction " << lt << " of account " << addr.to_hex()
  422. - << " has no inbound message");
  423. + return false;/*return reject_query(PSTRING() << "merge install transaction " << lt << " of account " << addr.to_hex()
  424. + << " has no inbound message");*/
  425. }
  426. need_credit_phase = true;
  427. // FIXME
  428. - return reject_query(PSTRING() << "unable to verify merge install transaction " << lt << " of account "
  429. - << addr.to_hex());
  430. + return false;/*return reject_query(PSTRING() << "unable to verify merge install transaction " << lt << " of account "
  431. + << addr.to_hex());*/
  432. break;
  433. }
  434. case block::gen::TransactionDescr::trans_split_prepare: {
  435. trans_type = block::transaction::Transaction::tr_split_prepare;
  436. if (in_msg_root.not_null()) {
  437. - return reject_query(PSTRING() << "split prepare transaction " << lt << " of account " << addr.to_hex()
  438. - << " has an inbound message");
  439. + return false;/*return reject_query(PSTRING() << "split prepare transaction " << lt << " of account " << addr.to_hex()
  440. + << " has an inbound message");*/
  441. }
  442. if (trans.outmsg_cnt > 1) {
  443. - return reject_query(PSTRING() << "split prepare transaction " << lt << " of account " << addr.to_hex()
  444. - << " must have exactly one outbound message");
  445. + return false;/*return reject_query(PSTRING() << "split prepare transaction " << lt << " of account " << addr.to_hex()
  446. + << " must have exactly one outbound message");*/
  447. }
  448. // FIXME
  449. - return reject_query(PSTRING() << "unable to verify split prepare transaction " << lt << " of account "
  450. - << addr.to_hex());
  451. + return false;/*return reject_query(PSTRING() << "unable to verify split prepare transaction " << lt << " of account "
  452. + << addr.to_hex());*/
  453. break;
  454. }
  455. case block::gen::TransactionDescr::trans_split_install: {
  456. trans_type = block::transaction::Transaction::tr_split_install;
  457. if (in_msg_root.is_null()) {
  458. - return reject_query(PSTRING() << "split install transaction " << lt << " of account " << addr.to_hex()
  459. - << " has no inbound message");
  460. + return false;/*return reject_query(PSTRING() << "split install transaction " << lt << " of account " << addr.to_hex()
  461. + << " has no inbound message");*/
  462. }
  463. // FIXME
  464. - return reject_query(PSTRING() << "unable to verify split install transaction " << lt << " of account "
  465. - << addr.to_hex());
  466. + return false;/*return reject_query(PSTRING() << "unable to verify split install transaction " << lt << " of account "
  467. + << addr.to_hex());*/
  468. break;
  469. }
  470. }
  471. @@ -4825,61 +4864,67 @@ bool ContestValidateQuery::check_one_transaction(block::Account& account, ton::L
  472. if (in_msg_root.not_null()) {
  473. if (!trs->unpack_input_msg(ihr_delivered, &action_phase_cfg_)) {
  474. // inbound external message was not accepted
  475. - return reject_query(PSTRING() << "could not unpack inbound " << (external ? "external" : "internal")
  476. + return false;/*return reject_query(PSTRING() << "could not unpack inbound " << (external ? "external" : "internal")
  477. << " message processed by ordinary transaction " << lt << " of account "
  478. - << addr.to_hex());
  479. + << addr.to_hex());*/
  480. }
  481. }
  482. if (trs->bounce_enabled) {
  483. - if (!trs->prepare_storage_phase(storage_phase_cfg_, true)) {
  484. - return reject_query(PSTRING() << "cannot re-create storage phase of transaction " << lt << " for smart contract "
  485. - << addr.to_hex());
  486. + {
  487. + WriteLockGuard<NonRecursiveRWLock> lock(strg_cfg_mtx);
  488. + if (!trs->prepare_storage_phase(storage_phase_cfg_, true)) {
  489. + return false;//return reject_query(PSTRING() << "cannot re-create storage phase of transaction " << lt << " for smart contract " << addr.to_hex());
  490. + }
  491. }
  492. if (need_credit_phase && !trs->prepare_credit_phase()) {
  493. - return reject_query(PSTRING() << "cannot create re-credit phase of transaction " << lt << " for smart contract "
  494. - << addr.to_hex());
  495. + return false;/*reject_query(PSTRING() << "cannot create re-credit phase of transaction " << lt << " for smart contract "
  496. + << addr.to_hex());*/
  497. }
  498. } else {
  499. if (need_credit_phase && !trs->prepare_credit_phase()) {
  500. - return reject_query(PSTRING() << "cannot re-create credit phase of transaction " << lt << " for smart contract "
  501. - << addr.to_hex());
  502. + return false;/*return false;return reject_query(PSTRING() << "cannot re-create credit phase of transaction " << lt << " for smart contract "
  503. + << addr.to_hex());*/
  504. }
  505. - if (!trs->prepare_storage_phase(storage_phase_cfg_, true, need_credit_phase)) {
  506. - return reject_query(PSTRING() << "cannot re-create storage phase of transaction " << lt << " for smart contract "
  507. - << addr.to_hex());
  508. + {
  509. + WriteLockGuard<NonRecursiveRWLock> lock(strg_cfg_mtx);
  510. + if (!trs->prepare_storage_phase(storage_phase_cfg_, true, need_credit_phase)) {
  511. + return false;/*return reject_query(PSTRING() << "cannot re-create storage phase of transaction " << lt << " for smart contract "
  512. + << addr.to_hex());*/
  513. + }
  514. }
  515. }
  516. - if (!trs->prepare_compute_phase(compute_phase_cfg_)) {
  517. - return reject_query(PSTRING() << "cannot re-create compute phase of transaction " << lt << " for smart contract "
  518. - << addr.to_hex());
  519. + {
  520. + WriteLockGuard<NonRecursiveRWLock> lock(comp_cfg_mtx);
  521. + if (!trs->prepare_compute_phase(compute_phase_cfg_)) {
  522. + return false;/*return reject_query(PSTRING() << "cannot re-create compute phase of transaction " << lt << " for smart contract "
  523. + << addr.to_hex());*/
  524. + }
  525. }
  526. if (!trs->compute_phase->accepted) {
  527. if (external) {
  528. - return reject_query(PSTRING() << "inbound external message claimed to be processed by ordinary transaction " << lt
  529. - << " of account " << addr.to_hex()
  530. - << " was in fact rejected (such transaction cannot appear in valid blocks)");
  531. + return false;//return reject_query(PSTRING() << "inbound external message claimed to be processed by ordinary transaction " << lt << " of account " << addr.to_hex() << " was in fact rejected (such transaction cannot appear in valid blocks)");
  532. } else if (trs->compute_phase->skip_reason == block::ComputePhase::sk_none) {
  533. - return reject_query(PSTRING() << "inbound internal message processed by ordinary transaction " << lt
  534. - << " of account " << addr.to_hex() << " was not processed without any reason");
  535. + return false;//return reject_query(PSTRING() << "inbound internal message processed by ordinary transaction " << lt << " of account " << addr.to_hex() << " was not processed without any reason");
  536. }
  537. }
  538. - if (trs->compute_phase->success && !trs->prepare_action_phase(action_phase_cfg_)) {
  539. - return reject_query(PSTRING() << "cannot re-create action phase of transaction " << lt << " for smart contract "
  540. - << addr.to_hex());
  541. - }
  542. - if (trs->bounce_enabled &&
  543. - (!trs->compute_phase->success || trs->action_phase->state_exceeds_limits || trs->action_phase->bounce) &&
  544. - !trs->prepare_bounce_phase(action_phase_cfg_)) {
  545. - return reject_query(PSTRING() << "cannot re-create bounce phase of transaction " << lt << " for smart contract "
  546. - << addr.to_hex());
  547. +
  548. + {
  549. + WriteLockGuard<NonRecursiveRWLock> lock(actp_cfg_mtx);
  550. + if (trs->compute_phase->success && !trs->prepare_action_phase(action_phase_cfg_)) {
  551. + return false;//return reject_query(PSTRING() << "cannot re-create action phase of transaction " << lt << " for smart contract " << addr.to_hex());
  552. + }
  553. + if (trs->bounce_enabled &&
  554. + (!trs->compute_phase->success || trs->action_phase->state_exceeds_limits || trs->action_phase->bounce) &&
  555. + !trs->prepare_bounce_phase(action_phase_cfg_)) {
  556. + return false;//return reject_query(PSTRING() << "cannot re-create bounce phase of transaction " << lt << " for smart contract " << addr.to_hex());
  557. + }
  558. }
  559. if (!trs->serialize()) {
  560. - return reject_query(PSTRING() << "cannot re-create the serialization of transaction " << lt
  561. - << " for smart contract " << addr.to_hex());
  562. + return false;//return reject_query(PSTRING() << "cannot re-create the serialization of transaction " << lt << " for smart contract " << addr.to_hex());
  563. }
  564. if (!trs->update_limits(*block_limit_status_, /* with_gas = */ false, /* with_size = */ false)) {
  565. - return fatal_error(PSTRING() << "cannot update block limit status to include transaction " << lt << " of account "
  566. - << addr.to_hex());
  567. + return false;/*return fatal_error(PSTRING() << "cannot update block limit status to include transaction " << lt << " of account "
  568. + << addr.to_hex());*/
  569. }
  570.  
  571. // Collator should stop if total gas usage exceeds limits, including transactions on special accounts, but without
  572. @@ -4888,24 +4933,30 @@ bool ContestValidateQuery::check_one_transaction(block::Account& account, ton::L
  573. if (!is_special_tx && !trs->gas_limit_overridden && trans_type == block::transaction::Transaction::tr_ord) {
  574. (account.is_special ? total_special_gas_used_ : total_gas_used_) += trs->gas_used();
  575. }
  576. - if (total_gas_used_ > block_limits_->gas.hard() + compute_phase_cfg_.gas_limit) {
  577. - return reject_query(PSTRING() << "gas block limits are exceeded: total_gas_used > gas_limit_hard + trx_gas_limit ("
  578. - << "total_gas_used=" << total_gas_used_
  579. - << ", gas_limit_hard=" << block_limits_->gas.hard()
  580. - << ", trx_gas_limit=" << compute_phase_cfg_.gas_limit << ")");
  581. + {
  582. + WriteLockGuard<NonRecursiveRWLock> lock(comp_cfg_mtx);
  583. + if (total_gas_used_ > block_limits_->gas.hard() + compute_phase_cfg_.gas_limit) {
  584. + return false;/*return reject_query(PSTRING() << "gas block limits are exceeded: total_gas_used > gas_limit_hard + trx_gas_limit ("
  585. + << "total_gas_used=" << total_gas_used_
  586. + << ", gas_limit_hard=" << block_limits_->gas.hard()
  587. + << ", trx_gas_limit=" << compute_phase_cfg_.gas_limit << ")");*/
  588. + }
  589. }
  590. - if (total_special_gas_used_ > block_limits_->gas.hard() + compute_phase_cfg_.special_gas_limit) {
  591. - return reject_query(
  592. - PSTRING() << "gas block limits are exceeded: total_special_gas_used > gas_limit_hard + special_gas_limit ("
  593. - << "total_special_gas_used=" << total_special_gas_used_
  594. - << ", gas_limit_hard=" << block_limits_->gas.hard()
  595. - << ", special_gas_limit=" << compute_phase_cfg_.special_gas_limit << ")");
  596. + {
  597. + WriteLockGuard<NonRecursiveRWLock> lock(comp_cfg_mtx);
  598. + if (total_special_gas_used_ > block_limits_->gas.hard() + compute_phase_cfg_.special_gas_limit) {
  599. + return false;/*return reject_query(
  600. + PSTRING() << "gas block limits are exceeded: total_special_gas_used > gas_limit_hard + special_gas_limit ("
  601. + << "total_special_gas_used=" << total_special_gas_used_
  602. + << ", gas_limit_hard=" << block_limits_->gas.hard()
  603. + << ", special_gas_limit=" << compute_phase_cfg_.special_gas_limit << ")");*/
  604. + }
  605. }
  606.  
  607. auto trans_root2 = trs->commit(account);
  608. if (trans_root2.is_null()) {
  609. - return reject_query(PSTRING() << "the re-created transaction " << lt << " for smart contract " << addr.to_hex()
  610. - << " could not be committed");
  611. + return false;/*return reject_query(PSTRING() << "the re-created transaction " << lt << " for smart contract " << addr.to_hex()
  612. + << " could not be committed");*/
  613. }
  614. // now compare the re-created transaction with the one we have
  615. if (trans_root2->get_hash() != trans_root->get_hash()) {
  616. @@ -4915,58 +4966,69 @@ bool ContestValidateQuery::check_one_transaction(block::Account& account, ton::L
  617. std::cerr << "re-created transaction " << lt << " of " << addr.to_hex() << ": ";
  618. block::gen::t_Transaction.print_ref(std::cerr, trans_root2);
  619. }
  620. - return reject_query(PSTRING() << "the transaction " << lt << " of " << addr.to_hex() << " has hash "
  621. + return false;/*return reject_query(PSTRING() << "the transaction " << lt << " of " << addr.to_hex() << " has hash "
  622. << trans_root->get_hash().to_hex()
  623. << " different from that of the recreated transaction "
  624. - << trans_root2->get_hash().to_hex());
  625. + << trans_root2->get_hash().to_hex());*/
  626. }
  627. block::gen::Transaction::Record trans2;
  628. block::gen::HASH_UPDATE::Record hash_upd2;
  629. if (!(tlb::unpack_cell(trans_root2, trans2) &&
  630. tlb::type_unpack_cell(std::move(trans2.state_update), block::gen::t_HASH_UPDATE_Account, hash_upd2))) {
  631. - return fatal_error(PSTRING() << "cannot unpack the re-created transaction " << lt << " of " << addr.to_hex());
  632. + return false;/*return fatal_error(PSTRING() << "cannot unpack the re-created transaction " << lt << " of " << addr.to_hex());*/
  633. }
  634. if (hash_upd2.old_hash != hash_upd.old_hash) {
  635. - return fatal_error(PSTRING() << "the re-created transaction " << lt << " of " << addr.to_hex()
  636. - << " is invalid: it starts from account state with different hash");
  637. + return false;/*return fatal_error(PSTRING() << "the re-created transaction " << lt << " of " << addr.to_hex()
  638. + << " is invalid: it starts from account state with different hash");*/
  639. }
  640. if (hash_upd2.new_hash != account.total_state->get_hash().bits()) {
  641. - return fatal_error(
  642. + return false;/*return fatal_error(
  643. PSTRING() << "the re-created transaction " << lt << " of " << addr.to_hex()
  644. - << " is invalid: its claimed new account hash differs from the actual new account state");
  645. + << " is invalid: its claimed new account hash differs from the actual new account state");*/
  646. }
  647. if (hash_upd.new_hash != account.total_state->get_hash().bits()) {
  648. - return reject_query(PSTRING() << "transaction " << lt << " of " << addr.to_hex()
  649. + return false;/*return reject_query(PSTRING() << "transaction " << lt << " of " << addr.to_hex()
  650. << " is invalid: it claims that the new account state hash is "
  651. << hash_upd.new_hash.to_hex() << " but the re-computed value is "
  652. - << hash_upd2.new_hash.to_hex());
  653. + << hash_upd2.new_hash.to_hex());*/
  654. }
  655. if (!trans.r1.out_msgs->contents_equal(*trans2.r1.out_msgs)) {
  656. - return reject_query(
  657. + return false;/*return reject_query(
  658. PSTRING()
  659. << "transaction " << lt << " of " << addr.to_hex()
  660. - << " is invalid: it has produced a set of outbound messages different from that listed in the transaction");
  661. + << " is invalid: it has produced a set of outbound messages different from that listed in the transaction");*/
  662. }
  663. total_burned_ += trs->blackhole_burned;
  664. // check new balance and value flow
  665. auto new_balance = account.get_balance();
  666. block::CurrencyCollection total_fees;
  667. if (!total_fees.validate_unpack(trans.total_fees)) {
  668. - return reject_query(PSTRING() << "transaction " << lt << " of " << addr.to_hex()
  669. - << " has an invalid total_fees value");
  670. + return false;/*return reject_query(PSTRING() << "transaction " << lt << " of " << addr.to_hex()
  671. + << " has an invalid total_fees value");*/
  672. }
  673. if (old_balance + money_imported != new_balance + money_exported + total_fees + trs->blackhole_burned) {
  674. - return reject_query(
  675. + return false;/*return reject_query(
  676. PSTRING() << "transaction " << lt << " of " << addr.to_hex()
  677. << " violates the currency flow condition: old balance=" << old_balance.to_str()
  678. << " + imported=" << money_imported.to_str() << " does not equal new balance=" << new_balance.to_str()
  679. << " + exported=" << money_exported.to_str() << " + total_fees=" << total_fees.to_str()
  680. << (trs->blackhole_burned.is_zero() ? ""
  681. - : PSTRING() << " burned=" << trs->blackhole_burned.to_str()));
  682. + : PSTRING() << " burned=" << trs->blackhole_burned.to_str()));*/
  683. }
  684. return true;
  685. }
  686.  
  687. +
  688. +/*void ContestValidateQuery::check_account_transactions_wrapper(const StdSmcAddress& acc_addr, Ref<vm::CellSlice> acc_blk_root)
  689. +{
  690. + if(check_account_transactions(acc_addr, acc_blk_root) == false)
  691. + {
  692. + flag_encountered_invalid.store(1, std::memory_order_relaxed);
  693. + }
  694. +
  695. + accs_count_processed.fetch_add(1, std::memory_order_relaxed);
  696. +}*/
  697. +
  698. /**
  699. * Checks the validity of transactions for a given account block.
  700. * NB: may be run in parallel for different accounts
  701. @@ -4978,10 +5040,16 @@ bool ContestValidateQuery::check_one_transaction(block::Account& account, ton::L
  702. */
  703. bool ContestValidateQuery::check_account_transactions(const StdSmcAddress& acc_addr, Ref<vm::CellSlice> acc_blk_root) {
  704. block::gen::AccountBlock::Record acc_blk;
  705. - CHECK(tlb::csr_unpack(std::move(acc_blk_root), acc_blk) && acc_blk.account_addr == acc_addr);
  706. +
  707. + {
  708. + std::lock_guard<std::mutex> unpacklock(unpack_mtx);
  709. + CHECK(tlb::csr_unpack(std::move(acc_blk_root), acc_blk));
  710. + CHECK(acc_blk.account_addr == acc_addr);
  711. + }
  712. auto account_p = unpack_account(acc_addr.cbits());
  713. if (!account_p) {
  714. - return reject_query("cannot unpack old state of account "s + acc_addr.to_hex());
  715. + printf("1\n");
  716. + return false;//return reject_query("cannot unpack old state of account "s + acc_addr.to_hex());
  717. }
  718. auto& account = *account_p;
  719. CHECK(account.addr == acc_addr);
  720. @@ -4998,7 +5066,8 @@ bool ContestValidateQuery::check_account_transactions(const StdSmcAddress& acc_a
  721. extra.clear();
  722. return check_one_transaction(account, lt, value->prefetch_ref(), lt == min_trans_lt, lt == max_trans_lt);
  723. })) {
  724. - return reject_query("at least one Transaction of account "s + acc_addr.to_hex() + " is invalid");
  725. + printf("2\n");
  726. + return false;//return reject_query("at least one Transaction of account "s + acc_addr.to_hex() + " is invalid");
  727. }
  728.  
  729. // See Collator::combine_account_trabsactions
  730. @@ -5008,12 +5077,19 @@ bool ContestValidateQuery::check_account_transactions(const StdSmcAddress& acc_a
  731. // account created
  732. CHECK(account.status != block::Account::acc_nonexist);
  733. vm::CellBuilder cb;
  734. - if (!(cb.store_ref_bool(account.total_state) // account_descr$_ account:^Account
  735. - && cb.store_bits_bool(account.last_trans_hash_) // last_trans_hash:bits256
  736. - && cb.store_long_bool(account.last_trans_lt_, 64) // last_trans_lt:uint64
  737. - && ns_.account_dict_->set_builder(account.addr, cb, vm::Dictionary::SetMode::Add))) {
  738. - return fatal_error(std::string{"cannot add newly-created account "} + account.addr.to_hex() +
  739. - " into ShardAccounts");
  740. + bool write_res = (cb.store_ref_bool(account.total_state) // account_descr$_ account:^Account
  741. + && cb.store_bits_bool(account.last_trans_hash_) // last_trans_hash:bits256
  742. + && cb.store_long_bool(account.last_trans_lt_, 64)); // last_trans_lt:uint64
  743. +
  744. + if(write_res)
  745. + {
  746. + WriteLockGuard<NonRecursiveRWLock> lock(account_dict_mtx);
  747. + write_res = ns_.account_dict_->set_builder(account.addr, cb, vm::Dictionary::SetMode::Add);
  748. + }
  749. +
  750. + if (!write_res) {
  751. + printf("3\n");
  752. + return false;//return fatal_error(std::string{"cannot add newly-created account "} + account.addr.to_hex() + " into ShardAccounts");
  753. }
  754. } else if (account.status == block::Account::acc_nonexist) {
  755. // account deleted
  756. @@ -5021,8 +5097,13 @@ bool ContestValidateQuery::check_account_transactions(const StdSmcAddress& acc_a
  757. std::cerr << "deleting account " << account.addr.to_hex() << " with empty new value ";
  758. block::gen::t_Account.print_ref(std::cerr, account.total_state);
  759. }
  760. - if (ns_.account_dict_->lookup_delete(account.addr).is_null()) {
  761. - return fatal_error(std::string{"cannot delete account "} + account.addr.to_hex() + " from ShardAccounts");
  762. +
  763. + {
  764. + WriteLockGuard<NonRecursiveRWLock> lock(account_dict_mtx);
  765. + if (ns_.account_dict_->lookup_delete(account.addr).is_null()) {
  766. + printf("4\n");
  767. + return false;//return fatal_error(std::string{"cannot delete account "} + account.addr.to_hex() + " from ShardAccounts");
  768. + }
  769. }
  770. } else {
  771. // existing account modified
  772. @@ -5031,32 +5112,47 @@ bool ContestValidateQuery::check_account_transactions(const StdSmcAddress& acc_a
  773. block::gen::t_Account.print_ref(std::cerr, account.total_state);
  774. }
  775. vm::CellBuilder cb;
  776. - if (!(cb.store_ref_bool(account.total_state) // account_descr$_ account:^Account
  777. - && cb.store_bits_bool(account.last_trans_hash_) // last_trans_hash:bits256
  778. - && cb.store_long_bool(account.last_trans_lt_, 64) // last_trans_lt:uint64
  779. - && ns_.account_dict_->set_builder(account.addr, cb, vm::Dictionary::SetMode::Replace))) {
  780. - return fatal_error(std::string{"cannot modify existing account "} + account.addr.to_hex() +
  781. - " in ShardAccounts");
  782. +
  783. + bool write_res = (cb.store_ref_bool(account.total_state) // account_descr$_ account:^Account
  784. + && cb.store_bits_bool(account.last_trans_hash_) // last_trans_hash:bits256
  785. + && cb.store_long_bool(account.last_trans_lt_, 64)); // last_trans_lt:uint64
  786. +
  787. + if(write_res)
  788. + {
  789. + WriteLockGuard<NonRecursiveRWLock> lock(account_dict_mtx);
  790. + write_res = ns_.account_dict_->set_builder(account.addr, cb, vm::Dictionary::SetMode::Replace);
  791. + }
  792. +
  793. + if (!write_res) {
  794. + printf("5\n");
  795. + return false;//return fatal_error(std::string{"cannot modify existing account "} + account.addr.to_hex() + " in ShardAccounts");
  796. }
  797. }
  798. }
  799.  
  800. block::gen::HASH_UPDATE::Record hash_upd;
  801. if (!tlb::type_unpack_cell(std::move(acc_blk.state_update), block::gen::t_HASH_UPDATE_Account, hash_upd)) {
  802. - return reject_query("cannot extract (HASH_UPDATE Account) from the AccountBlock of "s + account.addr.to_hex());
  803. + printf("6\n");
  804. + return false;//return reject_query("cannot extract (HASH_UPDATE Account) from the AccountBlock of "s + account.addr.to_hex());
  805. }
  806. block::tlb::ShardAccount::Record old_state, new_state;
  807. - if (!(old_state.unpack(ps_.account_dict_->lookup(account.addr)) &&
  808. - new_state.unpack(ns_.account_dict_->lookup(account.addr)))) {
  809. - return reject_query("cannot extract Account from the ShardAccount of "s + account.addr.to_hex());
  810. +
  811. + {
  812. + WriteLockGuard<NonRecursiveRWLock> lock(account_dict_mtx);
  813. + if (!(old_state.unpack(ps_.account_dict_->lookup(account.addr)) &&
  814. + new_state.unpack(ns_.account_dict_->lookup(account.addr)))) {
  815. + printf("7\n");
  816. + return false;//return reject_query("cannot extract Account from the ShardAccount of "s + account.addr.to_hex());
  817. + }
  818. }
  819. +
  820. if (hash_upd.old_hash != old_state.account->get_hash().bits()) {
  821. - return reject_query("(HASH_UPDATE Account) from the AccountBlock of "s + account.addr.to_hex() +
  822. - " has incorrect old hash");
  823. + printf("8\n");
  824. + return false;//return reject_query("(HASH_UPDATE Account) from the AccountBlock of "s + account.addr.to_hex() + " has incorrect old hash");
  825. }
  826. if (hash_upd.new_hash != new_state.account->get_hash().bits()) {
  827. - return reject_query("(HASH_UPDATE Account) from the AccountBlock of "s + account.addr.to_hex() +
  828. - " has incorrect new hash");
  829. + printf("9\n");
  830. + return false;//return reject_query("(HASH_UPDATE Account) from the AccountBlock of "s + account.addr.to_hex() + " has incorrect new hash");
  831. }
  832.  
  833. return true;
  834. @@ -5069,15 +5165,75 @@ bool ContestValidateQuery::check_account_transactions(const StdSmcAddress& acc_a
  835. */
  836. bool ContestValidateQuery::check_transactions() {
  837. LOG(INFO) << "checking all transactions";
  838. - ns_.account_dict_ =
  839. - std::make_unique<vm::AugmentedDictionary>(ps_.account_dict_->get_root(), 256, block::tlb::aug_ShardAccounts);
  840. + {
  841. + WriteLockGuard<NonRecursiveRWLock> lock(account_dict_mtx);
  842. + ns_.account_dict_ = std::make_unique<vm::AugmentedDictionary>(ps_.account_dict_->get_root(), 256, block::tlb::aug_ShardAccounts);
  843. + }
  844. +
  845. + //int pushed_jobs_cnt = 0; //account_dict_
  846. +
  847. + //this needs to be reset or something as after 1 thread works in slot it always is marked as done
  848. + vm::detail::thread_pool_out_data t_out_data[8];
  849. + uint8_t t_thread_was_used[8];
  850. + memset(t_thread_was_used, 0, 8);
  851. + flag_encountered_invalid.store(0, std::memory_order_relaxed);
  852. +
  853. bool ok = account_blocks_dict_->check_for_each_extra(
  854. - [this](Ref<vm::CellSlice> value, Ref<vm::CellSlice> extra, td::ConstBitPtr key, int key_len) {
  855. + [this, &t_out_data, &t_thread_was_used](Ref<vm::CellSlice> value, Ref<vm::CellSlice> extra, td::ConstBitPtr key, int key_len) {
  856. + //++pushed_jobs_cnt;
  857. CHECK(key_len == 256);
  858. - return check_account_transactions(key, std::move(value));
  859. +
  860. + //waiting_for_count_proc.fetch_add(1, std::memory_order_relaxed);
  861. +
  862. + std::array<unsigned char, 32> key_array;
  863. + std::memcpy(key_array.data(), key.ptr, 32);
  864. +
  865. + auto value_copy = Ref<vm::CellSlice>{true, value->clone()};
  866. + std::function<void()> proc_func = [this, value = std::move(value_copy), key_array = key_array](){
  867. +
  868. + auto value_arg = value;
  869. + if(check_account_transactions(td::ConstBitPtr(key_array.data()), std::move(value_arg)) == false)
  870. + {
  871. + printf("setting encountered invalid\n");
  872. + flag_encountered_invalid.store(1, std::memory_order_relaxed);
  873. + }
  874. +
  875. + //accs_count_processed.fetch_add(1, std::memory_order_relaxed);
  876. + };
  877. +
  878. + //uint32_t pushed_thread_idx = 0xFF; //for single threaded test
  879. + uint32_t pushed_thread_idx = vm::detail::push_thread_pool::try_find_and_set_thread_work(proc_func, t_out_data);
  880. + if(pushed_thread_idx == 0xFF) //pool is busy
  881. + {
  882. + proc_func();
  883. + }
  884. + else
  885. + {
  886. + t_thread_was_used[pushed_thread_idx] = 1;
  887. + }
  888. + return true;//return check_account_transactions(key, std::move(value));
  889. });
  890. +
  891. + //waiting_for_count_proc.store(pushed_jobs_cnt, std::memory_order_release);
  892. + vm::detail::push_thread_pool::tpool_wait_for_all_threads(t_out_data, t_thread_was_used);//waits for all sems / flags
  893. + //here we're done
  894. +
  895. + //here we wait sem and check counter
  896. +
  897. + bool res = false;
  898. + int flag_val = flag_encountered_invalid.load(std::memory_order_acquire);
  899. +
  900. + if(flag_val)
  901. + {
  902. + printf("returning false\n");
  903. + res = false;
  904. + }
  905. + else
  906. + {
  907. + res = true;
  908. + }
  909.  
  910. - return ok;
  911. + return res;
  912. }
  913.  
  914. /**
  915. @@ -5177,7 +5333,11 @@ bool ContestValidateQuery::check_new_state() {
  916. * @returns True if the value flow is valid, False otherwise.
  917. */
  918. bool ContestValidateQuery::postcheck_value_flow() {
  919. - auto accounts_extra = ns_.account_dict_->get_root_extra();
  920. + Ref<vm::CellSlice> accounts_extra;
  921. + {
  922. + WriteLockGuard<NonRecursiveRWLock> lock(account_dict_mtx);
  923. + accounts_extra = ns_.account_dict_->get_root_extra();
  924. + }
  925. block::CurrencyCollection cc;
  926. if (!(accounts_extra.write().advance(5) && cc.unpack(std::move(accounts_extra)))) {
  927. return reject_query("cannot unpack CurrencyCollection from the root of new accounts dictionary");
  928. @@ -5359,27 +5519,38 @@ bool ContestValidateQuery::build_state_update() {
  929. vm::CellBuilder cb, cb2;
  930.  
  931. // See Collator::create_shard_state
  932. - if (!(cb.store_long_bool(0x9023afe2, 32) // shard_state#9023afe2
  933. - && cb.store_long_bool(global_id_, 32) // global_id:int32
  934. - && block::ShardId{shard_}.serialize(cb) // shard_id:ShardIdent
  935. - && cb.store_long_bool(id_.seqno(), 32) // seq_no:uint32
  936. - && cb.store_long_bool(vert_seqno_, 32) // vert_seq_no:#
  937. - && cb.store_long_bool(now_, 32) // gen_utime:uint32
  938. - && cb.store_long_bool(ns_.lt_, 64) // gen_lt:uint64
  939. - && cb.store_long_bool(ns_.min_ref_mc_seqno_, 32) // min_ref_mc_seqno:uint32
  940. - && cb.store_ref_bool(msg_q_info) // out_msg_queue_info:^OutMsgQueueInfo
  941. - && cb.store_long_bool(before_split_, 1) // before_split:Bool
  942. - && ns_.account_dict_->append_dict_to_bool(cb2) // accounts:^ShardAccounts
  943. - && cb.store_ref_bool(cb2.finalize()) // ...
  944. - && cb2.store_long_bool(ns_.overload_history_, 64) // ^[ overload_history:uint64
  945. - && cb2.store_long_bool(ns_.underload_history_, 64) // underload_history:uint64
  946. - && ns_.total_balance_.store(cb2) // total_balance:CurrencyCollection
  947. - && ns_.total_validator_fees_.store(cb2) // total_validator_fees:CurrencyCollection
  948. - && cb2.store_bool_bool(false) // libraries:(HashmapE 256 LibDescr)
  949. - && cb2.store_bool_bool(true) && store_master_ref(cb2) // master_ref:(Maybe BlkMasterInfo)
  950. - && cb.store_ref_bool(cb2.finalize()) // ]
  951. - && cb.store_bool_bool(false) // custom:(Maybe ^McStateExtra)
  952. - && cb.finalize_to(state_root))) {
  953. + bool shwrite_ok = (cb.store_long_bool(0x9023afe2, 32) // shard_state#9023afe2
  954. + && cb.store_long_bool(global_id_, 32) // global_id:int32
  955. + && block::ShardId{shard_}.serialize(cb) // shard_id:ShardIdent
  956. + && cb.store_long_bool(id_.seqno(), 32) // seq_no:uint32
  957. + && cb.store_long_bool(vert_seqno_, 32) // vert_seq_no:#
  958. + && cb.store_long_bool(now_, 32) // gen_utime:uint32
  959. + && cb.store_long_bool(ns_.lt_, 64) // gen_lt:uint64
  960. + && cb.store_long_bool(ns_.min_ref_mc_seqno_, 32) // min_ref_mc_seqno:uint32
  961. + && cb.store_ref_bool(msg_q_info) // out_msg_queue_info:^OutMsgQueueInfo
  962. + && cb.store_long_bool(before_split_, 1)); // before_split:Bool
  963. +
  964. + if(shwrite_ok){
  965. + shwrite_ok = ns_.account_dict_->append_dict_to_bool(cb2);// accounts:^ShardAccounts
  966. + }
  967. +
  968. + if(shwrite_ok){
  969. + shwrite_ok = (cb.store_ref_bool(cb2.finalize()) // ...
  970. + && cb2.store_long_bool(ns_.overload_history_, 64) // ^[ overload_history:uint64
  971. + && cb2.store_long_bool(ns_.underload_history_, 64) // underload_history:uint64
  972. + && ns_.total_balance_.store(cb2) // total_balance:CurrencyCollection
  973. + && ns_.total_validator_fees_.store(cb2) // total_validator_fees:CurrencyCollection
  974. + && cb2.store_bool_bool(false) // libraries:(HashmapE 256 LibDescr)
  975. + && cb2.store_bool_bool(true) && store_master_ref(cb2) // master_ref:(Maybe BlkMasterInfo)
  976. + && cb.store_ref_bool(cb2.finalize()) // ]
  977. + && cb.store_bool_bool(false) // custom:(Maybe ^McStateExtra)
  978. + && cb.finalize_to(state_root));
  979. + }
  980. +
  981. +
  982. +
  983. + if(!shwrite_ok)
  984. + {
  985. return fatal_error("cannot create new ShardState");
  986. }
  987.  
  988. diff --git a/contest/solution/contest-validate-query.hpp b/contest/solution/contest-validate-query.hpp
  989. index 7a43c244..6dde9baa 100644
  990. --- a/contest/solution/contest-validate-query.hpp
  991. +++ b/contest/solution/contest-validate-query.hpp
  992. @@ -12,6 +12,7 @@
  993. #include <map>
  994. #include "common/global-version.h"
  995. #include "tonlib/tonlib/ExtClient.h"
  996. +//#include "vm/cells/MerkleProof.h"
  997.  
  998. namespace solution {
  999.  
  1000. @@ -81,6 +82,225 @@ inline ErrorCtxSet ErrorCtx::set_guard(std::vector<std::string> str_list) {
  1001. return ErrorCtxSet(*this, std::move(str_list));
  1002. }
  1003.  
  1004. +
  1005. +
  1006. +#ifndef __CPP11OM_RWLOCK_H__
  1007. +#define __CPP11OM_RWLOCK_H__
  1008. +
  1009. +#include <cassert>
  1010. +#include <atomic>
  1011. +//#include <random>
  1012. +#include "bitfield.h"
  1013. +
  1014. +
  1015. +//---------------------------------------------------------
  1016. +// LightweightSemaphore
  1017. +//---------------------------------------------------------
  1018. +class LightweightSemaphore
  1019. +{
  1020. +private:
  1021. + std::atomic<int> m_count;
  1022. + vm::Semaphore m_sema;
  1023. +
  1024. + void waitWithPartialSpinning()
  1025. + {
  1026. + int oldCount;
  1027. + // Is there a better way to set the initial spin count?
  1028. + // If we lower it to 1000, testBenaphore becomes 15x slower on my Core i7-5930K Windows PC,
  1029. + // as threads start hitting the kernel semaphore.
  1030. + int spin = 10000;
  1031. + while (spin--)
  1032. + {
  1033. + oldCount = m_count.load(std::memory_order_relaxed);
  1034. + if ((oldCount > 0) && m_count.compare_exchange_strong(oldCount, oldCount - 1, std::memory_order_acquire))
  1035. + return;
  1036. + std::atomic_signal_fence(std::memory_order_acquire); // Prevent the compiler from collapsing the loop.
  1037. + }
  1038. + oldCount = m_count.fetch_sub(1, std::memory_order_acquire);
  1039. + if (oldCount <= 0)
  1040. + {
  1041. + m_sema.wait();
  1042. + }
  1043. + }
  1044. +
  1045. +public:
  1046. + LightweightSemaphore(int initialCount = 0) : m_count(initialCount)
  1047. + {
  1048. + assert(initialCount >= 0);
  1049. + }
  1050. +
  1051. + bool tryWait()
  1052. + {
  1053. + int oldCount = m_count.load(std::memory_order_relaxed);
  1054. + return (oldCount > 0 && m_count.compare_exchange_strong(oldCount, oldCount - 1, std::memory_order_acquire));
  1055. + }
  1056. +
  1057. + void wait()
  1058. + {
  1059. + if (!tryWait())
  1060. + waitWithPartialSpinning();
  1061. + }
  1062. +
  1063. + void signal(int count = 1)
  1064. + {
  1065. + int oldCount = m_count.fetch_add(count, std::memory_order_release);
  1066. + int toRelease = -oldCount < count ? -oldCount : count;
  1067. + if (toRelease > 0)
  1068. + {
  1069. + m_sema.signal(toRelease);
  1070. + }
  1071. + }
  1072. +};
  1073. +
  1074. +
  1075. +typedef LightweightSemaphore DefaultSemaphoreType;
  1076. +
  1077. +//---------------------------------------------------------
  1078. +// NonRecursiveRWLock
  1079. +//---------------------------------------------------------
  1080. +class NonRecursiveRWLock
  1081. +{
  1082. +private:
  1083. + BEGIN_BITFIELD_TYPE(Status, uint32_t)
  1084. + ADD_BITFIELD_MEMBER(readers, 0, 10)
  1085. + ADD_BITFIELD_MEMBER(waitToRead, 10, 10)
  1086. + ADD_BITFIELD_MEMBER(writers, 20, 10)
  1087. + END_BITFIELD_TYPE()
  1088. +
  1089. + std::atomic<uint32_t> m_status;
  1090. + DefaultSemaphoreType m_readSema;
  1091. + DefaultSemaphoreType m_writeSema;
  1092. +
  1093. +public:
  1094. + NonRecursiveRWLock() : m_status(0) {}
  1095. +
  1096. + void lockReader()
  1097. + {
  1098. + Status oldStatus = m_status.load(std::memory_order_relaxed);
  1099. + Status newStatus;
  1100. + do
  1101. + {
  1102. + newStatus = oldStatus;
  1103. + if (oldStatus.writers > 0)
  1104. + {
  1105. + newStatus.waitToRead++;
  1106. + }
  1107. + else
  1108. + {
  1109. + newStatus.readers++;
  1110. + }
  1111. + // CAS until successful. On failure, oldStatus will be updated with the latest value.
  1112. + }
  1113. + while (!m_status.compare_exchange_weak(oldStatus, newStatus,
  1114. + std::memory_order_acquire, std::memory_order_relaxed));
  1115. +
  1116. + if (oldStatus.writers > 0)
  1117. + {
  1118. + m_readSema.wait();
  1119. + }
  1120. + }
  1121. +
  1122. + void unlockReader()
  1123. + {
  1124. + Status oldStatus = m_status.fetch_sub(Status().readers.one(), std::memory_order_release);
  1125. + assert(oldStatus.readers > 0);
  1126. + if (oldStatus.readers == 1 && oldStatus.writers > 0)
  1127. + {
  1128. + m_writeSema.signal();
  1129. + }
  1130. + }
  1131. +
  1132. + void lockWriter()
  1133. + {
  1134. + Status oldStatus = m_status.fetch_add(Status().writers.one(), std::memory_order_acquire);
  1135. + assert(oldStatus.writers + 1 <= Status().writers.maximum());
  1136. + if (oldStatus.readers > 0 || oldStatus.writers > 0)
  1137. + {
  1138. + m_writeSema.wait();
  1139. + }
  1140. + }
  1141. +
  1142. + void unlockWriter()
  1143. + {
  1144. + Status oldStatus = m_status.load(std::memory_order_relaxed);
  1145. + Status newStatus;
  1146. + uint32_t waitToRead = 0;
  1147. + do
  1148. + {
  1149. + assert(oldStatus.readers == 0);
  1150. + newStatus = oldStatus;
  1151. + newStatus.writers--;
  1152. + waitToRead = oldStatus.waitToRead;
  1153. + if (waitToRead > 0)
  1154. + {
  1155. + newStatus.waitToRead = 0;
  1156. + newStatus.readers = waitToRead;
  1157. + }
  1158. + // CAS until successful. On failure, oldStatus will be updated with the latest value.
  1159. + }
  1160. + while (!m_status.compare_exchange_weak(oldStatus, newStatus,
  1161. + std::memory_order_release, std::memory_order_relaxed));
  1162. +
  1163. + if (waitToRead > 0)
  1164. + {
  1165. + m_readSema.signal(waitToRead);
  1166. + }
  1167. + else if (oldStatus.writers > 1)
  1168. + {
  1169. + m_writeSema.signal();
  1170. + }
  1171. + }
  1172. +};
  1173. +
  1174. +
  1175. +//---------------------------------------------------------
  1176. +// ReadLockGuard
  1177. +//---------------------------------------------------------
  1178. +template <class LockType>
  1179. +class ReadLockGuard
  1180. +{
  1181. +private:
  1182. + LockType& m_lock;
  1183. +
  1184. +public:
  1185. + ReadLockGuard(LockType& lock) : m_lock(lock)
  1186. + {
  1187. + m_lock.lockReader();
  1188. + }
  1189. +
  1190. + ~ReadLockGuard()
  1191. + {
  1192. + m_lock.unlockReader();
  1193. + }
  1194. +};
  1195. +
  1196. +
  1197. +//---------------------------------------------------------
  1198. +// WriteLockGuard
  1199. +//---------------------------------------------------------
  1200. +template <class LockType>
  1201. +class WriteLockGuard
  1202. +{
  1203. +private:
  1204. + LockType& m_lock;
  1205. +
  1206. +public:
  1207. + WriteLockGuard(LockType& lock) : m_lock(lock)
  1208. + {
  1209. + m_lock.lockWriter();
  1210. + }
  1211. +
  1212. + ~WriteLockGuard()
  1213. + {
  1214. + m_lock.unlockWriter();
  1215. + }
  1216. +};
  1217. +
  1218. +
  1219. +#endif // __CPP11OM_RWLOCK_H__
  1220. +
  1221. +
  1222. +
  1223. class ContestValidateQuery : public td::actor::Actor {
  1224. static constexpr int supported_version() {
  1225. return SUPPORTED_VERSION;
  1226. @@ -113,6 +333,8 @@ class ContestValidateQuery : public td::actor::Actor {
  1227. bool prev_key_block_exists_{false};
  1228. bool debug_checks_{false};
  1229. bool outq_cleanup_partial_{false};
  1230. + std::mutex unpack_mtx;
  1231. + NonRecursiveRWLock account_dict_mtx;
  1232. BlockSeqno prev_key_seqno_{~0u};
  1233. int stage_{0};
  1234. td::BitArray<64> shard_pfx_;
  1235. @@ -166,6 +388,9 @@ class ContestValidateQuery : public td::actor::Actor {
  1236. std::vector<block::StoragePrices> storage_prices_;
  1237. block::StoragePhaseConfig storage_phase_cfg_{&storage_prices_};
  1238. block::ComputePhaseConfig compute_phase_cfg_;
  1239. + NonRecursiveRWLock comp_cfg_mtx;
  1240. + NonRecursiveRWLock strg_cfg_mtx;
  1241. + NonRecursiveRWLock actp_cfg_mtx;
  1242. block::ActionPhaseConfig action_phase_cfg_;
  1243. td::RefInt256 masterchain_create_fee_, basechain_create_fee_;
  1244.  
  1245. @@ -181,7 +406,13 @@ class ContestValidateQuery : public td::actor::Actor {
  1246. std::map<td::Bits256, int> block_create_count_;
  1247. unsigned block_create_total_{0};
  1248.  
  1249. + //std::atomic<int> accs_count_processed = {0};
  1250. + std::atomic<int> flag_encountered_invalid = {0};
  1251. + //std::atomic<int> waiting_for_count_proc = {0};
  1252. + //std::atomic<int> pushed_all_jobs_flag = {0};
  1253. +
  1254. std::unique_ptr<vm::AugmentedDictionary> in_msg_dict_, out_msg_dict_, account_blocks_dict_;
  1255. + NonRecursiveRWLock out_msg_dict_mtx;
  1256. block::ValueFlow value_flow_;
  1257. block::CurrencyCollection import_created_, transaction_fees_, total_burned_{0}, fees_burned_{0};
  1258. td::RefInt256 import_fees_;
  1259. @@ -338,4 +569,4 @@ class ContestValidateQuery : public td::actor::Actor {
  1260. bool build_state_update();
  1261. };
  1262.  
  1263. -} // namespace solution
  1264. \ No newline at end of file
  1265. +} // namespace solution
  1266. diff --git a/crypto/block/check-proof.cpp b/crypto/block/check-proof.cpp
  1267. index 431a03fe..c0e88924 100644
  1268. --- a/crypto/block/check-proof.cpp
  1269. +++ b/crypto/block/check-proof.cpp
  1270. @@ -612,6 +612,8 @@ td::Status check_block_signatures(const std::vector<ton::ValidatorDescr>& nodes,
  1271. if (signatures.empty()) {
  1272. return td::Status::Error("empty validator signature set");
  1273. }
  1274. +
  1275. + printf("DEBUG: check_block_signatures called\n");
  1276. // compute the string to be signed and its hash
  1277. unsigned char to_sign[68];
  1278. td::as<td::uint32>(to_sign) = 0xc50b6e70; // ton.blockId root_cell_hash:int256 file_hash:int256 = ton.BlockId;
  1279. @@ -626,8 +628,12 @@ td::Status check_block_signatures(const std::vector<ton::ValidatorDescr>& nodes,
  1280. total_weight += nodes[i].weight;
  1281. node_map.emplace_back(compute_node_id_short(nodes[i].key), i);
  1282. }
  1283. +
  1284. + printf("DEBUG: total_weight = %llu\n", total_weight);
  1285. std::sort(node_map.begin(), node_map.end());
  1286. std::vector<unsigned> seen;
  1287. +
  1288. + printf("DEBUG: signatures len: %llu\n", signatures.size());
  1289. for (auto& sig : signatures) {
  1290. // lookup node in validator set
  1291. auto& id = sig.node;
  1292. @@ -641,6 +647,10 @@ td::Status check_block_signatures(const std::vector<ton::ValidatorDescr>& nodes,
  1293. // check one signature
  1294. td::Ed25519::PublicKey pub_key{td::SecureString{nodes.at(i).key.as_slice()}};
  1295. auto res = pub_key.verify_signature(td::Slice{to_sign, 68}, sig.signature.as_slice());
  1296. +
  1297. + td::MutableSlice sigsl = const_cast<td::BufferSlice&>(sig.signature).as_slice();
  1298. + td::StringBuilder sig_strb(sigsl);
  1299. + printf("\tnodeidshort: %s\n\tpubkey: %s\n\tsignature: %s\n\tweight: %llu\n\n\n", id.to_hex().c_str(), nodes.at(i).key._pubkey.to_hex().c_str(), sig_strb.as_cslice().c_str(), nodes[i].weight);
  1300. if (res.is_error()) {
  1301. return res;
  1302. }
  1303. @@ -649,6 +659,8 @@ td::Status check_block_signatures(const std::vector<ton::ValidatorDescr>& nodes,
  1304. break;
  1305. }
  1306. }
  1307. +
  1308. + printf("DEBUG: signed_weight = %llu\n", signed_weight);
  1309. std::sort(seen.begin(), seen.end());
  1310. for (std::size_t i = 1; i < seen.size(); i++) {
  1311. if (seen[i] == seen[i - 1]) {
  1312. @@ -656,6 +668,9 @@ td::Status check_block_signatures(const std::vector<ton::ValidatorDescr>& nodes,
  1313. compute_node_id_short(nodes.at(seen[i]).key).to_hex());
  1314. }
  1315. }
  1316. +
  1317. + printf("DEBUG: checking 3 * %llu <= 2 * %llu \n", signed_weight, total_weight);
  1318. + printf("DEBUG: checking %llu <= %llu \n", 3 * signed_weight, 2 * total_weight);
  1319. if (3 * signed_weight <= 2 * total_weight) {
  1320. return td::Status::Error(PSTRING() << "insufficient total signature weight: only " << signed_weight << " out of "
  1321. << total_weight);
  1322. diff --git a/crypto/vm/cells/CellUsageTree.cpp b/crypto/vm/cells/CellUsageTree.cpp
  1323. index 410b3fcd..eee26c1a 100644
  1324. --- a/crypto/vm/cells/CellUsageTree.cpp
  1325. +++ b/crypto/vm/cells/CellUsageTree.cpp
  1326. @@ -23,6 +23,8 @@ namespace vm {
  1327. // CellUsageTree::NodePtr
  1328. //
  1329. bool CellUsageTree::NodePtr::on_load(const td::Ref<vm::DataCell>& cell) const {
  1330. + //WriteLockGuard<NonRecursiveRWLock> lock(tree_mtx);
  1331. +
  1332. auto tree = tree_weak_.lock();
  1333. if (!tree) {
  1334. return false;
  1335. @@ -32,6 +34,8 @@ bool CellUsageTree::NodePtr::on_load(const td::Ref<vm::DataCell>& cell) const {
  1336. }
  1337.  
  1338. CellUsageTree::NodePtr CellUsageTree::NodePtr::create_child(unsigned ref_id) const {
  1339. + //WriteLockGuard<NonRecursiveRWLock> lock(tree_mtx);
  1340. +
  1341. auto tree = tree_weak_.lock();
  1342. if (!tree) {
  1343. return {};
  1344. @@ -40,6 +44,8 @@ CellUsageTree::NodePtr CellUsageTree::NodePtr::create_child(unsigned ref_id) con
  1345. }
  1346.  
  1347. bool CellUsageTree::NodePtr::is_from_tree(const CellUsageTree* master_tree) const {
  1348. + //WriteLockGuard<NonRecursiveRWLock> lock(tree_mtx);
  1349. +
  1350. DCHECK(master_tree);
  1351. auto tree = tree_weak_.lock();
  1352. if (tree.get() != master_tree) {
  1353. @@ -49,6 +55,8 @@ bool CellUsageTree::NodePtr::is_from_tree(const CellUsageTree* master_tree) cons
  1354. }
  1355.  
  1356. bool CellUsageTree::NodePtr::mark_path(CellUsageTree* master_tree) const {
  1357. + //WriteLockGuard<NonRecursiveRWLock> lock(tree_mtx);
  1358. +
  1359. DCHECK(master_tree);
  1360. auto tree = tree_weak_.lock();
  1361. if (tree.get() != master_tree) {
  1362. @@ -62,25 +70,29 @@ bool CellUsageTree::NodePtr::mark_path(CellUsageTree* master_tree) const {
  1363. // CellUsageTree
  1364. //
  1365. CellUsageTree::NodePtr CellUsageTree::root_ptr() {
  1366. + //WriteLockGuard<NonRecursiveRWLock> lock(nodes_mtx);
  1367. return {shared_from_this(), 1};
  1368. }
  1369.  
  1370. -CellUsageTree::NodeId CellUsageTree::root_id() const {
  1371. +CellUsageTree::NodeId CellUsageTree::root_id() {
  1372. return 1;
  1373. };
  1374.  
  1375. -bool CellUsageTree::is_loaded(NodeId node_id) const {
  1376. +bool CellUsageTree::is_loaded(NodeId node_id) {
  1377. + std::lock_guard<std::recursive_mutex> lock(nodes_mtx);
  1378. if (use_mark_) {
  1379. return nodes_[node_id].has_mark;
  1380. }
  1381. return nodes_[node_id].is_loaded;
  1382. }
  1383.  
  1384. -bool CellUsageTree::has_mark(NodeId node_id) const {
  1385. +bool CellUsageTree::has_mark(NodeId node_id) {
  1386. + std::lock_guard<std::recursive_mutex> lock(nodes_mtx);
  1387. return nodes_[node_id].has_mark;
  1388. }
  1389.  
  1390. void CellUsageTree::set_mark(NodeId node_id, bool mark) {
  1391. + std::lock_guard<std::recursive_mutex> lock(nodes_mtx);
  1392. if (node_id == 0) {
  1393. return;
  1394. }
  1395. @@ -88,6 +100,7 @@ void CellUsageTree::set_mark(NodeId node_id, bool mark) {
  1396. }
  1397.  
  1398. void CellUsageTree::mark_path(NodeId node_id) {
  1399. + std::lock_guard<std::recursive_mutex> lock(nodes_mtx);
  1400. auto cur_node_id = get_parent(node_id);
  1401. while (cur_node_id != 0) {
  1402. if (has_mark(cur_node_id)) {
  1403. @@ -99,19 +112,23 @@ void CellUsageTree::mark_path(NodeId node_id) {
  1404. }
  1405.  
  1406. CellUsageTree::NodeId CellUsageTree::get_parent(NodeId node_id) {
  1407. + std::lock_guard<std::recursive_mutex> lock(nodes_mtx);
  1408. return nodes_[node_id].parent;
  1409. }
  1410.  
  1411. CellUsageTree::NodeId CellUsageTree::get_child(NodeId node_id, unsigned ref_id) {
  1412. + std::lock_guard<std::recursive_mutex> lock(nodes_mtx);
  1413. DCHECK(ref_id < CellTraits::max_refs);
  1414. return nodes_[node_id].children[ref_id];
  1415. }
  1416.  
  1417. void CellUsageTree::set_use_mark_for_is_loaded(bool use_mark) {
  1418. + std::lock_guard<std::recursive_mutex> lock(nodes_mtx);
  1419. use_mark_ = use_mark;
  1420. }
  1421.  
  1422. void CellUsageTree::on_load(NodeId node_id, const td::Ref<vm::DataCell>& cell) {
  1423. + std::lock_guard<std::recursive_mutex> lock(nodes_mtx);
  1424. if (nodes_[node_id].is_loaded) {
  1425. return;
  1426. }
  1427. @@ -122,6 +139,7 @@ void CellUsageTree::on_load(NodeId node_id, const td::Ref<vm::DataCell>& cell) {
  1428. }
  1429.  
  1430. CellUsageTree::NodeId CellUsageTree::create_child(NodeId node_id, unsigned ref_id) {
  1431. + std::lock_guard<std::recursive_mutex> lock(nodes_mtx);
  1432. DCHECK(ref_id < CellTraits::max_refs);
  1433. NodeId res = nodes_[node_id].children[ref_id];
  1434. if (res) {
  1435. @@ -133,6 +151,7 @@ CellUsageTree::NodeId CellUsageTree::create_child(NodeId node_id, unsigned ref_i
  1436. }
  1437.  
  1438. CellUsageTree::NodeId CellUsageTree::create_node(NodeId parent) {
  1439. + std::lock_guard<std::recursive_mutex> lock(nodes_mtx);
  1440. NodeId res = static_cast<NodeId>(nodes_.size());
  1441. nodes_.emplace_back();
  1442. nodes_.back().parent = parent;
  1443. diff --git a/crypto/vm/cells/CellUsageTree.h b/crypto/vm/cells/CellUsageTree.h
  1444. index af0f21f5..03af746e 100644
  1445. --- a/crypto/vm/cells/CellUsageTree.h
  1446. +++ b/crypto/vm/cells/CellUsageTree.h
  1447. @@ -24,8 +24,372 @@
  1448. #include "td/utils/logging.h"
  1449. #include <functional>
  1450.  
  1451. +
  1452. +
  1453. +
  1454. namespace vm {
  1455.  
  1456. +
  1457. +#include <cassert>
  1458. +#include <atomic>
  1459. +//#include <random>
  1460. +//#include "trbitfield.h"
  1461. +
  1462. +//preshing's semaphore
  1463. +
  1464. +#if defined(_WIN32)
  1465. +//---------------------------------------------------------
  1466. +// Semaphore (Windows)
  1467. +//---------------------------------------------------------
  1468. +
  1469. +#include <windows.h>
  1470. +#undef min
  1471. +#undef max
  1472. +
  1473. +class Semaphore
  1474. +{
  1475. +private:
  1476. + HANDLE m_hSema;
  1477. +
  1478. + Semaphore(const Semaphore& other) = delete;
  1479. + Semaphore& operator=(const Semaphore& other) = delete;
  1480. +
  1481. +public:
  1482. + Semaphore(int initialCount = 0)
  1483. + {
  1484. + assert(initialCount >= 0);
  1485. + m_hSema = CreateSemaphore(NULL, initialCount, MAXLONG, NULL);
  1486. + }
  1487. +
  1488. + ~Semaphore()
  1489. + {
  1490. + CloseHandle(m_hSema);
  1491. + }
  1492. +
  1493. + void wait()
  1494. + {
  1495. + WaitForSingleObject(m_hSema, INFINITE);
  1496. + }
  1497. +
  1498. + void signal(int count = 1)
  1499. + {
  1500. + ReleaseSemaphore(m_hSema, count, NULL);
  1501. + }
  1502. +};
  1503. +
  1504. +
  1505. +#elif defined(__MACH__)
  1506. +//---------------------------------------------------------
  1507. +// Semaphore (Apple iOS and OSX)
  1508. +// Can't use POSIX semaphores due to http://lists.apple.com/archives/darwin-kernel/2009/Apr/msg00010.html
  1509. +//---------------------------------------------------------
  1510. +
  1511. +#include <mach/mach.h>
  1512. +
  1513. +class Semaphore
  1514. +{
  1515. +private:
  1516. + semaphore_t m_sema;
  1517. +
  1518. + Semaphore(const Semaphore& other) = delete;
  1519. + Semaphore& operator=(const Semaphore& other) = delete;
  1520. +
  1521. +public:
  1522. + Semaphore(int initialCount = 0)
  1523. + {
  1524. + assert(initialCount >= 0);
  1525. + semaphore_create(mach_task_self(), &m_sema, SYNC_POLICY_FIFO, initialCount);
  1526. + }
  1527. +
  1528. + ~Semaphore()
  1529. + {
  1530. + semaphore_destroy(mach_task_self(), m_sema);
  1531. + }
  1532. +
  1533. + void wait()
  1534. + {
  1535. + semaphore_wait(m_sema);
  1536. + }
  1537. +
  1538. + void signal()
  1539. + {
  1540. + semaphore_signal(m_sema);
  1541. + }
  1542. +
  1543. + void signal(int count)
  1544. + {
  1545. + while (count-- > 0)
  1546. + {
  1547. + semaphore_signal(m_sema);
  1548. + }
  1549. + }
  1550. +};
  1551. +
  1552. +
  1553. +#elif defined(__unix__)
  1554. +//---------------------------------------------------------
  1555. +// Semaphore (POSIX, Linux)
  1556. +//---------------------------------------------------------
  1557. +
  1558. +#include <semaphore.h>
  1559. +
  1560. +class Semaphore
  1561. +{
  1562. +private:
  1563. + sem_t m_sema;
  1564. +
  1565. + Semaphore(const Semaphore& other) = delete;
  1566. + Semaphore& operator=(const Semaphore& other) = delete;
  1567. +
  1568. +public:
  1569. + Semaphore(int initialCount = 0)
  1570. + {
  1571. + assert(initialCount >= 0);
  1572. + sem_init(&m_sema, 0, initialCount);
  1573. + }
  1574. +
  1575. + ~Semaphore()
  1576. + {
  1577. + sem_destroy(&m_sema);
  1578. + }
  1579. +
  1580. + void wait()
  1581. + {
  1582. + // http://stackoverflow.com/questions/2013181/gdb-causes-sem-wait-to-fail-with-eintr-error
  1583. + int rc;
  1584. + do
  1585. + {
  1586. + rc = sem_wait(&m_sema);
  1587. + }
  1588. + while (rc == -1 && errno == EINTR);
  1589. + }
  1590. +
  1591. + void signal()
  1592. + {
  1593. + sem_post(&m_sema);
  1594. + }
  1595. +
  1596. + void signal(int count)
  1597. + {
  1598. + while (count-- > 0)
  1599. + {
  1600. + sem_post(&m_sema);
  1601. + }
  1602. + }
  1603. +};
  1604. +
  1605. +
  1606. +#else
  1607. +
  1608. +#error Unsupported platform!
  1609. +
  1610. +#endif
  1611. +
  1612. +
  1613. +//---------------------------------------------------------
  1614. +// LightweightSemaphore
  1615. +//---------------------------------------------------------
  1616. +class LightweightSemaphore
  1617. +{
  1618. +private:
  1619. + std::atomic<int> m_count;
  1620. + Semaphore m_sema;
  1621. +
  1622. + void waitWithPartialSpinning()
  1623. + {
  1624. + int oldCount;
  1625. + // Is there a better way to set the initial spin count?
  1626. + // If we lower it to 1000, testBenaphore becomes 15x slower on my Core i7-5930K Windows PC,
  1627. + // as threads start hitting the kernel semaphore.
  1628. + int spin = 10000;
  1629. + while (spin--)
  1630. + {
  1631. + oldCount = m_count.load(std::memory_order_relaxed);
  1632. + if ((oldCount > 0) && m_count.compare_exchange_strong(oldCount, oldCount - 1, std::memory_order_acquire))
  1633. + return;
  1634. + std::atomic_signal_fence(std::memory_order_acquire); // Prevent the compiler from collapsing the loop.
  1635. + }
  1636. + oldCount = m_count.fetch_sub(1, std::memory_order_acquire);
  1637. + if (oldCount <= 0)
  1638. + {
  1639. + m_sema.wait();
  1640. + }
  1641. + }
  1642. +
  1643. +public:
  1644. + LightweightSemaphore(int initialCount = 0) : m_count(initialCount)
  1645. + {
  1646. + assert(initialCount >= 0);
  1647. + }
  1648. +
  1649. + bool tryWait()
  1650. + {
  1651. + int oldCount = m_count.load(std::memory_order_relaxed);
  1652. + return (oldCount > 0 && m_count.compare_exchange_strong(oldCount, oldCount - 1, std::memory_order_acquire));
  1653. + }
  1654. +
  1655. + void wait()
  1656. + {
  1657. + if (!tryWait())
  1658. + waitWithPartialSpinning();
  1659. + }
  1660. +
  1661. + void signal(int count = 1)
  1662. + {
  1663. + int oldCount = m_count.fetch_add(count, std::memory_order_release);
  1664. + int toRelease = -oldCount < count ? -oldCount : count;
  1665. + if (toRelease > 0)
  1666. + {
  1667. + m_sema.signal(toRelease);
  1668. + }
  1669. + }
  1670. +};
  1671. +
  1672. +
  1673. +typedef LightweightSemaphore DefaultSemaphoreType;
  1674. +/*
  1675. +//---------------------------------------------------------
  1676. +// NonRecursiveRWLock
  1677. +//---------------------------------------------------------
  1678. +class NonRecursiveRWLock
  1679. +{
  1680. +private:
  1681. + BEGIN_BITFIELD_TYPE(Status, uint32_t)
  1682. + ADD_BITFIELD_MEMBER(readers, 0, 10)
  1683. + ADD_BITFIELD_MEMBER(waitToRead, 10, 10)
  1684. + ADD_BITFIELD_MEMBER(writers, 20, 10)
  1685. + END_BITFIELD_TYPE()
  1686. +
  1687. + std::atomic<uint32_t> m_status;
  1688. + DefaultSemaphoreType m_readSema;
  1689. + DefaultSemaphoreType m_writeSema;
  1690. +
  1691. +public:
  1692. + NonRecursiveRWLock() : m_status(0) {}
  1693. +
  1694. + void lockReader()
  1695. + {
  1696. + Status oldStatus = m_status.load(std::memory_order_relaxed);
  1697. + Status newStatus;
  1698. + do
  1699. + {
  1700. + newStatus = oldStatus;
  1701. + if (oldStatus.writers > 0)
  1702. + {
  1703. + newStatus.waitToRead++;
  1704. + }
  1705. + else
  1706. + {
  1707. + newStatus.readers++;
  1708. + }
  1709. + // CAS until successful. On failure, oldStatus will be updated with the latest value.
  1710. + }
  1711. + while (!m_status.compare_exchange_weak(oldStatus, newStatus,
  1712. + std::memory_order_acquire, std::memory_order_relaxed));
  1713. +
  1714. + if (oldStatus.writers > 0)
  1715. + {
  1716. + m_readSema.wait();
  1717. + }
  1718. + }
  1719. +
  1720. + void unlockReader()
  1721. + {
  1722. + Status oldStatus = m_status.fetch_sub(Status().readers.one(), std::memory_order_release);
  1723. + assert(oldStatus.readers > 0);
  1724. + if (oldStatus.readers == 1 && oldStatus.writers > 0)
  1725. + {
  1726. + m_writeSema.signal();
  1727. + }
  1728. + }
  1729. +
  1730. + void lockWriter()
  1731. + {
  1732. + Status oldStatus = m_status.fetch_add(Status().writers.one(), std::memory_order_acquire);
  1733. + assert(oldStatus.writers + 1 <= Status().writers.maximum());
  1734. + if (oldStatus.readers > 0 || oldStatus.writers > 0)
  1735. + {
  1736. + m_writeSema.wait();
  1737. + }
  1738. + }
  1739. +
  1740. + void unlockWriter()
  1741. + {
  1742. + Status oldStatus = m_status.load(std::memory_order_relaxed);
  1743. + Status newStatus;
  1744. + uint32_t waitToRead = 0;
  1745. + do
  1746. + {
  1747. + assert(oldStatus.readers == 0);
  1748. + newStatus = oldStatus;
  1749. + newStatus.writers--;
  1750. + waitToRead = oldStatus.waitToRead;
  1751. + if (waitToRead > 0)
  1752. + {
  1753. + newStatus.waitToRead = 0;
  1754. + newStatus.readers = waitToRead;
  1755. + }
  1756. + // CAS until successful. On failure, oldStatus will be updated with the latest value.
  1757. + }
  1758. + while (!m_status.compare_exchange_weak(oldStatus, newStatus,
  1759. + std::memory_order_release, std::memory_order_relaxed));
  1760. +
  1761. + if (waitToRead > 0)
  1762. + {
  1763. + m_readSema.signal(waitToRead);
  1764. + }
  1765. + else if (oldStatus.writers > 1)
  1766. + {
  1767. + m_writeSema.signal();
  1768. + }
  1769. + }
  1770. +};
  1771. +
  1772. +
  1773. +//---------------------------------------------------------
  1774. +// ReadLockGuard
  1775. +//---------------------------------------------------------
  1776. +template <class LockType>
  1777. +class ReadLockGuard
  1778. +{
  1779. +private:
  1780. + LockType& m_lock;
  1781. +
  1782. +public:
  1783. + ReadLockGuard(LockType& lock) : m_lock(lock)
  1784. + {
  1785. + m_lock.lockReader();
  1786. + }
  1787. +
  1788. + ~ReadLockGuard()
  1789. + {
  1790. + m_lock.unlockReader();
  1791. + }
  1792. +};
  1793. +
  1794. +
  1795. +//---------------------------------------------------------
  1796. +// WriteLockGuard
  1797. +//---------------------------------------------------------
  1798. +template <class LockType>
  1799. +class WriteLockGuard
  1800. +{
  1801. +private:
  1802. + LockType& m_lock;
  1803. +
  1804. +public:
  1805. + WriteLockGuard(LockType& lock) : m_lock(lock)
  1806. + {
  1807. + m_lock.lockWriter();
  1808. + }
  1809. +
  1810. + ~WriteLockGuard()
  1811. + {
  1812. + m_lock.unlockWriter();
  1813. + }
  1814. +};*/
  1815. +
  1816. +
  1817. class DataCell;
  1818.  
  1819. class CellUsageTree : public std::enable_shared_from_this<CellUsageTree> {
  1820. @@ -53,9 +417,9 @@ class CellUsageTree : public std::enable_shared_from_this<CellUsageTree> {
  1821. };
  1822.  
  1823. NodePtr root_ptr();
  1824. - NodeId root_id() const;
  1825. - bool is_loaded(NodeId node_id) const;
  1826. - bool has_mark(NodeId node_id) const;
  1827. + NodeId root_id();
  1828. + bool is_loaded(NodeId node_id);
  1829. + bool has_mark(NodeId node_id);
  1830. void set_mark(NodeId node_id, bool mark = true);
  1831. void mark_path(NodeId node_id);
  1832. NodeId get_parent(NodeId node_id);
  1833. @@ -77,6 +441,8 @@ class CellUsageTree : public std::enable_shared_from_this<CellUsageTree> {
  1834. bool use_mark_{false};
  1835. std::vector<Node> nodes_{2};
  1836. std::function<void(const td::Ref<vm::DataCell>&)> cell_load_callback_;
  1837. +
  1838. + std::recursive_mutex nodes_mtx;
  1839.  
  1840. void on_load(NodeId node_id, const td::Ref<vm::DataCell>& cell);
  1841. NodeId create_node(NodeId parent);
  1842. diff --git a/crypto/vm/cells/MerkleProof.cpp b/crypto/vm/cells/MerkleProof.cpp
  1843. index 26dff787..8e74ed0f 100644
  1844. --- a/crypto/vm/cells/MerkleProof.cpp
  1845. +++ b/crypto/vm/cells/MerkleProof.cpp
  1846. @@ -23,17 +23,187 @@
  1847.  
  1848. #include "td/utils/HashMap.h"
  1849. #include "td/utils/HashSet.h"
  1850. +#include <mutex>
  1851. +#include <queue>
  1852. +#include <thread>
  1853. +
  1854. +//#define PUSH_THREAD_POOL_DBG_ENABLED
  1855. +
  1856. +class ContestValidateQuery;
  1857.  
  1858. namespace vm {
  1859. namespace detail {
  1860. -class MerkleProofImpl {
  1861. - public:
  1862. - explicit MerkleProofImpl(MerkleProof::IsPrunnedFunction is_prunned) : is_prunned_(std::move(is_prunned)) {
  1863. +
  1864. +push_thread_pool_data push_thread_pool::task_pushed_data[8];
  1865. +std::atomic<int> push_thread_pool::thread_ready_bitset = { 0xFF }; //reset to all threads available
  1866. +std::thread push_thread_pool::worker_threads[8];
  1867. +
  1868. +
  1869. +uint32_t push_thread_pool::try_find_and_set_thread_work(std::function<void()> in_data_exec_lambda, thread_pool_out_data* out_data_arr)
  1870. +{
  1871. + int old_bitset = thread_ready_bitset.load(std::memory_order_relaxed);
  1872. +
  1873. + int thread_idx = 0;
  1874. + for(;;) //bit search in a CAS loop
  1875. + {
  1876. + if(old_bitset == 0)
  1877. + {
  1878. + return 0xFF; //no ready threads
  1879. + }
  1880. +
  1881. + thread_idx = __builtin_ctz(old_bitset);
  1882. + int new_bitset = old_bitset & ~( 1 << thread_idx );
  1883. +
  1884. + //acq rel on success to make sure there is no reordering of data modification going after the cas
  1885. + if(thread_ready_bitset.compare_exchange_weak(old_bitset, new_bitset, std::memory_order_acq_rel, std::memory_order_relaxed))
  1886. + {
  1887. + break;
  1888. + }
  1889. +
  1890. + //compare exchange failed, likely because bitset has changed in the meantime, retry operation
  1891. + }
  1892. +
  1893. + //TODO: push data for dfs
  1894. + task_pushed_data[thread_idx].in_data_exec_lambda = in_data_exec_lambda;
  1895. + //task_pushed_data[thread_idx].in_data_cell = in_dat_cell;
  1896. + //task_pushed_data[thread_idx].in_data_merkle_depth = in_dat_merkle_depth;
  1897. +
  1898. + //it's assumed that we won't delete memory it points to until this function is done, which is reasonable
  1899. + task_pushed_data[thread_idx].out_data_memptr = out_data_arr + thread_idx;
  1900. +
  1901. + //now that we pushed data, set flag and notify thread
  1902. +
  1903. + //also reset out flag, store after acts as a barrier for it to not go under
  1904. + out_data_arr[thread_idx].out_data_flag.store(0, std::memory_order_relaxed);
  1905. + task_pushed_data[thread_idx].flag.store(1, std::memory_order_release); //release to make sure data is flushed, no reorder
  1906. + task_pushed_data[thread_idx].sema.signal();
  1907. +
  1908. +#ifdef PUSH_THREAD_POOL_DBG_ENABLED
  1909. + printf("successfully pushed work, t_idx: %d\n", thread_idx);
  1910. +#endif
  1911. + return thread_idx; //success
  1912. +}
  1913. +
  1914. +
  1915. +void push_thread_pool::tpool_thread_main(uint32_t thread_idx)
  1916. +{
  1917. + int flag_expected = 1;
  1918. +
  1919. + while (true) {
  1920. + //TODO: potentially some spinning before wait to reduce switching
  1921. +
  1922. + flag_expected = 1;
  1923. +
  1924. + if(task_pushed_data[thread_idx].flag.compare_exchange_strong(flag_expected, 0, std::memory_order_acquire))
  1925. + {
  1926. + //flag was 1, execute func, because of acquire - release semantics we can be sure that in_data is visible here
  1927. +
  1928. + //ContestValidateQuery* context_obj = task_pushed_data[thread_idx].in_data_context_obj;
  1929. + //Ref<Cell> ret_cell = context_obj->dfs(task_pushed_data[thread_idx].in_data_cell, task_pushed_data[thread_idx].in_data_merkle_depth);
  1930. +
  1931. + std::function<void()> cur_work = task_pushed_data[thread_idx].in_data_exec_lambda;
  1932. + cur_work();
  1933. +
  1934. + thread_pool_out_data* out_data_ptr = task_pushed_data[thread_idx].out_data_memptr;
  1935. + //out_data_ptr->out_data_cell = ret_cell;
  1936. + out_data_ptr->out_data_flag.store(1, std::memory_order_release); //release to make sure data is flushed, no reorder from before to after
  1937. + out_data_ptr->out_data_sema.signal();
  1938. +
  1939. + //TODO: semaphore reinit?
  1940. + }
  1941. + else
  1942. + {
  1943. + //was 0, wait for signal
  1944. + thread_ready_bitset.fetch_or( 1 << thread_idx, std::memory_order_release);
  1945. +
  1946. + task_pushed_data[thread_idx].sema.wait();
  1947. + }
  1948. +
  1949. + }
  1950. +}
  1951. +
  1952. +void push_thread_pool::tpool_init()
  1953. +{
  1954. +#ifdef PUSH_THREAD_POOL_DBG_ENABLED
  1955. + printf("initializing push_thread_pool\n");
  1956. +#endif
  1957. + thread_ready_bitset.store(0xFF, std::memory_order_relaxed);
  1958. +
  1959. + for(int idx = 0; idx < 8; ++idx)
  1960. + {
  1961. + std::thread worker_th(tpool_thread_main, idx);
  1962. +
  1963. + worker_threads[idx] = std::move(worker_th);
  1964. + }
  1965. +}
  1966. +
  1967. +void push_thread_pool::tpool_prepare()
  1968. +{
  1969. +#ifdef PUSH_THREAD_POOL_DBG_ENABLED
  1970. + printf("preparing push_thread_pool for reuse\n");
  1971. +#endif
  1972. + thread_ready_bitset.store(0xFF, std::memory_order_relaxed);
  1973. +
  1974. + for(int idx = 0; idx < 8; ++idx)
  1975. + {
  1976. + //task_pushed_data[idx].in_data_context_obj = 0;
  1977. + //task_pushed_data[idx].in_data_merkle_depth = 0;
  1978. + //task_pushed_data[idx].out_data_memptr = 0;
  1979. + task_pushed_data[idx].flag.store(0, std::memory_order_relaxed);
  1980. + }
  1981. +}
  1982. +
  1983. +
  1984. +//Ref<Cell> out_data_cell;
  1985. +//std::atomic<int> out_data_flag = { 0 };
  1986. +//Semaphore out_data_sema;
  1987. +
  1988. +void push_thread_pool::tpool_thread_wait_for_out_data(thread_pool_out_data* out_data_ptr)
  1989. +{
  1990. + int flag_expected = 1;
  1991. +
  1992. + while (true) {
  1993. + //TODO: initial spin to reduce switches
  1994. +
  1995. + flag_expected = 1;
  1996. +
  1997. + if(out_data_ptr->out_data_flag.compare_exchange_strong(flag_expected, 0, std::memory_order_acquire))
  1998. + {
  1999. + //out data flag was 1, meaning that because of release->acquire sync we definitely got out_data that was written before flag release store
  2000. +
  2001. + return; //out_data_ptr->out_data_cell;
  2002. + }
  2003. + else
  2004. + {
  2005. + //printf("thread is not done..\n");
  2006. + out_data_ptr->out_data_sema.wait();
  2007. + }
  2008. +
  2009. + }
  2010. +}
  2011. +
  2012. +void push_thread_pool::tpool_wait_for_all_threads(thread_pool_out_data* out_data, uint8_t* thread_was_used)
  2013. +{
  2014. + for(int idx = 0; idx < 8; ++idx)
  2015. + {
  2016. + if(thread_was_used[idx])
  2017. + {
  2018. + tpool_thread_wait_for_out_data(out_data + idx);
  2019. + }
  2020. }
  2021. - explicit MerkleProofImpl(CellUsageTree *usage_tree) : usage_tree_(usage_tree) {
  2022. +}
  2023. +
  2024. +
  2025. +
  2026. +
  2027. + MerkleProofImpl::MerkleProofImpl(MerkleProof::IsPrunnedFunction is_prunned) : is_prunned_(std::move(is_prunned)) {
  2028. + push_thread_pool::tpool_prepare();
  2029. + }
  2030. + MerkleProofImpl::MerkleProofImpl(CellUsageTree *usage_tree) : usage_tree_(usage_tree) {
  2031. + push_thread_pool::tpool_prepare();
  2032. }
  2033.  
  2034. - Ref<Cell> create_from(Ref<Cell> cell) {
  2035. + Ref<Cell> MerkleProofImpl::create_from(Ref<Cell> cell) {
  2036. if (!is_prunned_) {
  2037. CHECK(usage_tree_);
  2038. dfs_usage_tree(cell, usage_tree_->root_id());
  2039. @@ -48,28 +218,57 @@ class MerkleProofImpl {
  2040. }
  2041. }
  2042.  
  2043. - private:
  2044. - using Key = std::pair<Cell::Hash, int>;
  2045. - td::HashMap<Key, Ref<Cell>> cells_;
  2046. - td::HashSet<Cell::Hash> visited_cells_;
  2047. - CellUsageTree *usage_tree_{nullptr};
  2048. - MerkleProof::IsPrunnedFunction is_prunned_;
  2049.  
  2050. - void dfs_usage_tree(Ref<Cell> cell, CellUsageTree::NodeId node_id) {
  2051. + void MerkleProofImpl::dfs_usage_tree(Ref<Cell> cell, CellUsageTree::NodeId node_id) {
  2052. if (!usage_tree_->is_loaded(node_id)) {
  2053. return;
  2054. }
  2055. visited_cells_.insert(cell->get_hash());
  2056. CellSlice cs(NoVm(), cell);
  2057. + //printf("dfs_tree_refs: %d\n", cs.size_refs()); //2,3, havent seen more in logs
  2058. for (unsigned i = 0; i < cs.size_refs(); i++) {
  2059. dfs_usage_tree(cs.prefetch_ref(i), usage_tree_->get_child(node_id, i));
  2060. }
  2061. }
  2062.  
  2063. - Ref<Cell> dfs(Ref<Cell> cell, int merkle_depth) {
  2064. +//ORIG DFS
  2065. +Ref<Cell> MerkleProofImpl::dfs(Ref<Cell> cell, int merkle_depth) {
  2066. + CHECK(cell.not_null());
  2067. + Key key{cell->get_hash(), merkle_depth};
  2068. + {
  2069. + auto it = cells_.find(key);
  2070. + if (it != cells_.end()) {
  2071. + CHECK(it->second.not_null());
  2072. + return it->second;
  2073. + }
  2074. + }
  2075. +
  2076. + if (is_prunned_(cell)) {
  2077. + auto res = CellBuilder::create_pruned_branch(cell, merkle_depth + 1);
  2078. + CHECK(res.not_null());
  2079. + cells_.emplace(key, res);
  2080. + return res;
  2081. + }
  2082. + CellSlice cs(NoVm(), cell);
  2083. + int children_merkle_depth = cs.child_merkle_depth(merkle_depth);
  2084. + CellBuilder cb;
  2085. + cb.store_bits(cs.fetch_bits(cs.size()));
  2086. + for (unsigned i = 0; i < cs.size_refs(); i++) {
  2087. + cb.store_ref(dfs(cs.prefetch_ref(i), children_merkle_depth));
  2088. + }
  2089. + auto res = cb.finalize(cs.is_special());
  2090. + CHECK(res.not_null());
  2091. + cells_.emplace(key, res);
  2092. + return res;
  2093. +}
  2094. +//DFS END
  2095. +
  2096. +/*
  2097. + Ref<Cell> MerkleProofImpl::dfs(Ref<Cell> cell, int merkle_depth) {
  2098. CHECK(cell.not_null());
  2099. Key key{cell->get_hash(), merkle_depth};
  2100. {
  2101. + std::lock_guard<std::mutex> clock(cells_hm_mutex);
  2102. auto it = cells_.find(key);
  2103. if (it != cells_.end()) {
  2104. CHECK(it->second.not_null());
  2105. @@ -80,22 +279,82 @@ class MerkleProofImpl {
  2106. if (is_prunned_(cell)) {
  2107. auto res = CellBuilder::create_pruned_branch(cell, merkle_depth + 1);
  2108. CHECK(res.not_null());
  2109. - cells_.emplace(key, res);
  2110. + {
  2111. + std::lock_guard<std::mutex> clock(cells_hm_mutex);
  2112. + cells_.emplace(key, res);
  2113. + }
  2114. return res;
  2115. }
  2116. CellSlice cs(NoVm(), cell);
  2117. int children_merkle_depth = cs.child_merkle_depth(merkle_depth);
  2118. CellBuilder cb;
  2119. cb.store_bits(cs.fetch_bits(cs.size()));
  2120. - for (unsigned i = 0; i < cs.size_refs(); i++) {
  2121. - cb.store_ref(dfs(cs.prefetch_ref(i), children_merkle_depth));
  2122. +
  2123. + //printf("dfs_refs: %d, children_merkle_depth: %d\n", cs.size_refs(), children_merkle_depth);
  2124. + //for (unsigned i = 0; i < cs.size_refs(); i++) {
  2125. + //for (unsigned i = 1; i < cs.size_refs(); i++) {
  2126. + //should be dispatched to other threads
  2127. + //after 0s dfs we put wait for signal from other threads (so after it finishes it waits for other threads)
  2128. +
  2129. + //we stop forking if we reached hardware concurrency, so we're not destroying performance with switches
  2130. + //cb.store_ref(dfs(cs.prefetch_ref(i), children_merkle_depth));
  2131. + //}
  2132. +
  2133. + if(cs.size_refs() > 0)
  2134. + {
  2135. + unsigned forked_cell_futures_len = cs.size_refs() - 1;
  2136. +
  2137. +#ifdef PUSH_THREAD_POOL_DBG_ENABLED
  2138. + printf("creating %u forks\n", forked_cell_futures_len);
  2139. +#endif
  2140. + thread_pool_out_data forked_cell_futures[forked_cell_futures_len];
  2141. + bool forked_cell_futures_success_bools[forked_cell_futures_len];
  2142. +
  2143. + for (unsigned i = 1; i < cs.size_refs(); i++) {
  2144. +#ifdef PUSH_THREAD_POOL_DBG_ENABLED
  2145. + printf("dispatched %u\n", i - 1);
  2146. +#endif
  2147. +
  2148. + forked_cell_futures_success_bools[i - 1] = push_thread_pool::try_find_and_set_thread_work(this, cs.prefetch_ref(i), children_merkle_depth, forked_cell_futures + (i - 1));
  2149. + //if failed to push into pool, no free threads
  2150. + //do it in this thread later
  2151. +
  2152. + //forked_cell_futures[i - 1] = tpool.submit(&MerkleProofImpl::dfs, this, cs.prefetch_ref(i), children_merkle_depth);
  2153. + }
  2154. + cb.store_ref(dfs(cs.prefetch_ref(0), children_merkle_depth));
  2155. + //when we get those tasks back, store refs
  2156. + for(unsigned ix = 0; ix < forked_cell_futures_len; ++ix)
  2157. + {
  2158. + bool has_work_been_delegated = forked_cell_futures_success_bools[ix];
  2159. + if(has_work_been_delegated)
  2160. + {
  2161. +#ifdef PUSH_THREAD_POOL_DBG_ENABLED
  2162. + printf("waiting for result from out_data_memptr[%d]..\n", ix);
  2163. +#endif
  2164. + cb.store_ref(push_thread_pool::tpool_thread_wait_for_out_data(forked_cell_futures + ix));
  2165. +#ifdef PUSH_THREAD_POOL_DBG_ENABLED
  2166. + printf("got result from out_data_memptr %u\n", ix);
  2167. +#endif
  2168. + }
  2169. + else
  2170. + {
  2171. + cb.store_ref(dfs(cs.prefetch_ref(ix + 1), children_merkle_depth));
  2172. +#ifdef PUSH_THREAD_POOL_DBG_ENABLED
  2173. + printf("calculated result in current thread, %u\n", ix);
  2174. +#endif
  2175. + }
  2176. + }
  2177. }
  2178. +
  2179. auto res = cb.finalize(cs.is_special());
  2180. CHECK(res.not_null());
  2181. - cells_.emplace(key, res);
  2182. +
  2183. + {
  2184. + std::lock_guard<std::mutex> clock(cells_hm_mutex);
  2185. + cells_.emplace(key, res);
  2186. + }
  2187. return res;
  2188. - }
  2189. -};
  2190. + }*/
  2191. } // namespace detail
  2192.  
  2193. Ref<Cell> MerkleProof::generate_raw(Ref<Cell> cell, IsPrunnedFunction is_prunned) {
  2194. diff --git a/crypto/vm/cells/MerkleProof.h b/crypto/vm/cells/MerkleProof.h
  2195. index fc2cb6eb..f84bb7df 100644
  2196. --- a/crypto/vm/cells/MerkleProof.h
  2197. +++ b/crypto/vm/cells/MerkleProof.h
  2198. @@ -20,8 +20,12 @@
  2199. #include "vm/cells/Cell.h"
  2200. #include "td/utils/buffer.h"
  2201.  
  2202. +#include "td/utils/HashMap.h"
  2203. +#include "td/utils/HashSet.h"
  2204. +
  2205. #include <utility>
  2206. #include <functional>
  2207. +#include <thread>
  2208.  
  2209. namespace vm {
  2210.  
  2211. @@ -51,6 +55,85 @@ class MerkleProof {
  2212. static Ref<Cell> combine_fast_raw(Ref<Cell> a, Ref<Cell> b);
  2213. };
  2214.  
  2215. +namespace detail {
  2216. +
  2217. +
  2218. +
  2219. +class ContestValidateQuery;
  2220. +
  2221. +class alignas(128) thread_pool_out_data
  2222. +{
  2223. +public:
  2224. + //Ref<Cell> out_data_cell;
  2225. +
  2226. + std::atomic<int> out_data_flag = { 0 };
  2227. + Semaphore out_data_sema;
  2228. +};
  2229. +
  2230. +
  2231. +//64 is based on x86's hardware destructive interference size, should also be 128 for arm
  2232. +class alignas(128) push_thread_pool_data
  2233. +{
  2234. +public:
  2235. +
  2236. + //we know we will do dfs, just store data
  2237. + //ContestValidateQuery* in_data_context_obj = 0;
  2238. + //const StdSmcAddress& in_data_acc_addr;
  2239. + //Ref<vm::CellSlice> in_data_acc_blk_root = 0;
  2240. +
  2241. + thread_pool_out_data* out_data_memptr = 0;
  2242. + std::function<void()> in_data_exec_lambda;
  2243. +
  2244. + std::atomic<int> flag = { 0 };
  2245. + Semaphore sema;
  2246. +};
  2247. +
  2248. +class push_thread_pool
  2249. +{
  2250. +public:
  2251. + static push_thread_pool_data task_pushed_data[8];
  2252. +
  2253. + static std::atomic<int> thread_ready_bitset; //we use this for search from pusher's perspective and we use separated flag values for false sharing prevention
  2254. +
  2255. + //find first bit set
  2256. + //set bit through | 1 << bit_num_from_zero
  2257. + //check bit through __builtin_ctz(bitset), if bitset is 0 it would be ub
  2258. +
  2259. + static std::thread worker_threads[8];
  2260. +
  2261. + static uint32_t try_find_and_set_thread_work(std::function<void()> in_data_exec_lambda, thread_pool_out_data* out_data_arr);
  2262. + static void tpool_thread_main(uint32_t thread_idx);
  2263. +
  2264. + static void tpool_init();
  2265. + static void tpool_prepare(); //for reuse
  2266. +
  2267. + static void tpool_thread_wait_for_out_data(thread_pool_out_data* out_data_ptr);
  2268. + static void tpool_wait_for_all_threads(thread_pool_out_data* out_data, uint8_t* thread_was_used);
  2269. +};
  2270. +
  2271. +class MerkleProofImpl {
  2272. +public:
  2273. + explicit MerkleProofImpl(MerkleProof::IsPrunnedFunction is_prunned);
  2274. + explicit MerkleProofImpl(CellUsageTree *usage_tree);
  2275. +
  2276. + Ref<Cell> create_from(Ref<Cell> cell);
  2277. +
  2278. + using Key = std::pair<Cell::Hash, int>;
  2279. + td::HashMap<Key, Ref<Cell>> cells_;
  2280. + td::HashSet<Cell::Hash> visited_cells_;
  2281. + CellUsageTree *usage_tree_{nullptr};
  2282. + MerkleProof::IsPrunnedFunction is_prunned_;
  2283. +
  2284. + std::mutex cells_hm_mutex;
  2285. + std::mutex visited_cells_mutex;
  2286. + std::mutex usage_tree_mutex;
  2287. +
  2288. + void dfs_usage_tree(Ref<Cell> cell, CellUsageTree::NodeId node_id);
  2289. +
  2290. + Ref<Cell> dfs(Ref<Cell> cell, int merkle_depth);
  2291. +};
  2292. +}
  2293. +
  2294. class MerkleProofBuilder {
  2295. std::shared_ptr<CellUsageTree> usage_tree;
  2296. Ref<vm::Cell> orig_root, usage_root;
  2297.  
Advertisement
Add Comment
Please, Sign In to add comment
Public Pastes
We use cookies for various purposes including analytics. By continuing to use Pastebin, you agree to our use of cookies as described in the Cookies Policy.  OK, I Understand
Not a member of Pastebin yet?
Sign Up, it unlocks many cool features!