Advertisement
Guest User

Untitled

a guest
Nov 21st, 2017
61
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 2.86 KB | None | 0 0
  1. def self.load_thread_posts_with_max_responses_in_interval(fid, h_back =24)
  2.  
  3. title = DB[:forums].filter(siteid:SID,fid:fid).first[:title]
  4. p "----------------FORUM: #{fid} #{title}"
  5.  
  6. from=date_now(h_back)
  7. BCTalkParser.class_variable_set(:@@from_date, from)
  8.  
  9. p " --load_thread_posts_with_max_responses_in_interval fid:#{fid} h_back:#{h_back} start_from:#{from.strftime("%F %H:%M:%S")}"
  10. to=date_now(0)
  11.  
  12. threads_responses = DB[:threads_responses].filter(Sequel.lit("sid=? and fid=? and last_post_date > ?", SID,fid,from))
  13. .select_map([:tid,:responses,:last_post_date])
  14.  
  15. list_threads = []
  16. start_date = date_now(12)
  17.  
  18. ########analz
  19. sorted_thread_stats = threads_responses.group_by{|dd| dd[0]}
  20. .select{|k,v| v.size>1}
  21. .sort_by{|k,tt| dd=tt.map { |el| el[1] }.minmax; dd[1]-dd[0] }
  22. .reverse.take(THREADS_ANALZ_NUM)
  23.  
  24. #sorted_thread_stats.each do |tid, tt|
  25. Parallel.map_with_index(sorted_thread_stats,:in_threads=>1) do |rr,idx|
  26.  
  27. tid = rr[0]
  28. resps = rr[1]
  29. #next if tid!=2198936
  30.  
  31. resps_minmax=resps.map { |el| el[1] }.minmax
  32.  
  33. page_and_num = PageUtil.calc_last_page(resps_minmax[1]+1,20)
  34. lpage = page_and_num[0]
  35. lcount = page_and_num[1]
  36.  
  37. url_templ = "https://bitcointalk.org/index.php?topic=%s.%s"
  38. url = url_templ % [tid,(lpage-1)*40]
  39.  
  40. downl_pages=BCTalkParser.calc_arr_downl_pages(tid,lpage,lcount, BCTalkParser.from_date).take(10)
  41.  
  42. downloaded_pages=[]
  43.  
  44. ranks_stat_all=Hash.new(0)
  45. downl_pages.each do |pp|
  46. downloaded_pages<<pp[0]
  47. begin
  48. data = BCTalkParser.set_opt({rank:1}).parse_thread_page(tid, pp[0])
  49.  
  50. ranks_stat= data[:stat].group_by{|x| x}.map{|k,vv| [k,vv.size]}.to_h
  51. [1,2,3,4,5,11].each{|x| ranks_stat_all[x]+= (ranks_stat[x]||0)}
  52.  
  53. fpdate = data[:first_post_date]
  54. break if fpdate<from
  55.  
  56. rescue =>ex
  57. #puts ex.backtrace
  58. p "-------------------"
  59. p "--err tid #{tid} pg #{pp} --#{ex}"
  60. end
  61. end
  62.  
  63. if ranks_stat_all
  64. rr={fid:fid, tid:tid, description:"downloaded_pages #{downloaded_pages}" ,
  65. start_date:from, end_date:to, added: date_now,
  66. r1_count:ranks_stat_all[1],
  67. r2_count:ranks_stat_all[2],
  68. r3_count:ranks_stat_all[3],
  69. r4_count:ranks_stat_all[4],
  70. r5_count:ranks_stat_all[5],
  71. r11_count:ranks_stat_all[11],
  72. }
  73. DB[:threads_stat].insert(rr)
  74. end
  75.  
  76. planned_str=downl_pages.map { |pp| "#{pp[0]}" }.join(' ')
  77.  
  78. p "[[#{idx}] load_thr #{tid} last pg,count: #{page_and_num}".ljust(50)+
  79. "planned:#{planned_str.ljust(40)} down:#{downloaded_pages} ranks_stat_all: #{ranks_stat_all}" if downl_pages.size>0
  80.  
  81. end
  82.  
  83. end
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement