Advertisement
Guest User

Untitled

a guest
Sep 19th, 2014
222
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
Ruby 5.48 KB | None | 0 0
  1. require "bundler"
  2. Bundler.require
  3. require "open-uri"
  4.  
  5. class Array; def to_proc; proc { |a| a[*self] } end end
  6.  
  7. def open_uri uri, &block
  8.   open uri, {ssl_verify_mode: OpenSSL::SSL::VERIFY_NONE, "Cookie" => "accept=1"}, &block
  9. end
  10.  
  11. def stat_of uri
  12.   # 4chan api
  13.   open_uri uri+"/0.json" do |i|
  14.     a = JSON.parse i.read
  15.    
  16.     posts = a["threads"]
  17.       .select { |i| i["posts"][0]["sticky"] == 0 }[0..10]
  18.       .map(&["posts"])
  19.       .flatten
  20.      
  21.     return 0.0 if posts.empty?
  22.        
  23.     oldest = posts.min { |a,b| a["no"] <=> b["no"] }
  24.     newest = posts.max { |a,b| a["no"] <=> b["no"] }
  25.    
  26.     posts = newest["no"] - oldest["no"] + 1
  27.     time = Time.now - Time.at(oldest["time"])
  28.    
  29.     posts.to_f / time.to_f
  30.   end
  31. rescue SocketError
  32.   retry
  33. rescue OpenURI::HTTPError
  34.   begin
  35.     # mitsuba? kusaba?
  36.     doc = Nokogiri::HTML open_uri uri+"/"
  37.    
  38.     mitsuba = !doc.css('.absBotDisclaimer').empty?
  39.     kusaba = !doc.css('.footer a[href$="cultnet.net/"],
  40.                      .footer a[href$="kusabax.org/"]').empty?
  41.     tinyboard = !doc.css('footer a[href$="tinyboard.org/"]').empty?
  42.     northboard = !doc.css('#software a[href$="NorthBoard/"]').empty?
  43.    
  44.     if not mitsuba and not tinyboard and not kusaba and not northboard
  45.       raise "Not supported: #{uri}"
  46.     end
  47.    
  48.     thread_selector = (mitsuba|northboard) ? ".thread" :
  49.                       (kusaba|tinyboard) ? 'div[id^="thread"]' :
  50.                         false
  51.                        
  52.     sticky_selector = mitsuba ? "img.stickyIcon" :
  53.                       kusaba ? 'img[src="pin.png"]' :
  54.                       tinyboard ? "i.fa-thumb-tack" :
  55.                       northboard ? 'img[src$="attach.png"]' :
  56.                         false
  57.                        
  58.     postinfo_selector = mitsuba ? ".postInfo" :
  59.                         kusaba ? ".reply" :
  60.                         tinyboard ? ".intro" :
  61.                         northboard ? ".postinfo" :
  62.                           false
  63.                          
  64.     postid_selector = mitsuba ? '.quotePost, a[title="Reply to this post"]' :
  65.                       kusaba ? ".reflink>a:last" :
  66.                       tinyboard ? ">.post_no" :
  67.                       northboard ? ".post_number > a[onclick]" :
  68.                         false
  69.                        
  70.     date_selector = mitsuba ? ".dateTime" :
  71.                     kusaba ? "label:first" :
  72.                     tinyboard ? "time" :
  73.                     northboard ? ".post_time" :
  74.                       false
  75.                      
  76.     threads = doc.css(thread_selector)
  77.     threads = threads.select { |t| t.css(sticky_selector).empty? }[0..10]
  78.    
  79.     posts = threads.map { |t| t.css(postinfo_selector) }
  80.     posts += threads if kusaba
  81.     posts = posts.flatten
  82.    
  83.     posts = posts.map do |p|
  84.       pid = p.css(postid_selector)
  85.      
  86.       if tinyboard
  87.         pid = pid.last
  88.       else
  89.         pid = pid.first
  90.       end
  91.      
  92.       pid = pid.text.strip.to_i
  93.      
  94.       date = p.css(date_selector).first
  95.       if date["datetime"]
  96.         date = date["datetime"]
  97.       elsif date["data-utc"]
  98.         date = Time.at(date["data-utc"].to_i)
  99.       else
  100.         date = date.children.last.text.strip
  101.       end
  102.          
  103.       date = Time.parse(date) if date.class != Time
  104.      
  105.       [pid, date]
  106.     end
  107.    
  108.     return 0.0 if posts.empty?
  109.        
  110.     oldest = posts.min { |a,b| a[0] <=> b[0] }
  111.     newest = posts.max { |a,b| a[1] <=> b[1] }
  112.    
  113.     posts = newest[0] - oldest[0] + 1
  114.     time = Time.now - oldest[1]
  115.    
  116.     posts.to_f / time.to_f
  117.   rescue SocketError
  118.     retry
  119.   rescue Exception => e
  120.     $error = e
  121.     puts "Error: #{e}: #{uri}"
  122.     0.0
  123.   end
  124. end
  125.  
  126. chans = nil
  127. case ARGV[0]
  128. when "pl"
  129.   vi = %w[b cp r+oc id waifu wiz veto int slav
  130.           sci psl h c c++ vg lsd ku fso btc trv
  131.           a az ac mu tv lit vp x hk fr
  132.           sr swag sex pro med soc trap pr psy
  133.           meta chan mit 3 fem synch]
  134.   kara = %w[4 b fz z r id $ c co a edu f fa
  135.             h kib ku l med mil mu oc p po pony
  136.             sci sp tech thc trv v8 vg wall x og
  137.             int kara g hen s dew]
  138.   kiwi = %w[b a co hob kul tec v wc kiwi]
  139.   wilchan = %w[b vg admin]
  140.   _8chan = %w[rzabczan heretyk flutter ebolachan
  141.               sierpchan kib g]
  142.   misc = %w[http://heretyk.tk/* http://heretichan.tk/b]
  143.            
  144.   vi = vi.map { |i| "https://pl.vichan.net/"+i }
  145.   kara = kara.map { |i| "http://karachan.org/"+i }
  146.   kiwi = kiwi.map { |i| "https://kiwiszon.org/boards/"+i }
  147.   wilchan = wilchan.map { |i| "http://wilchan.tk/"+i }
  148.   _8chan = _8chan.map { |i| "https://8chan.co/"+i }
  149.  
  150.   chans = misc + vi + kara + kiwi + wilchan + _8chan
  151. when "v"
  152.   _4chan = %w[b vg v int pol a co tg sp fit]
  153.   _8chan = %w[b v int burgers pol anime co tg sp]
  154.  
  155.   _4chan = _4chan.map { |i| "https://boards.4chan.org/"+i }
  156.   _8chan = _8chan.map { |i| "https://8chan.co/"+i }
  157.  
  158.   chans = _4chan + _8chan
  159. end
  160.  
  161.  
  162. require "thread"
  163. require "pp"
  164. results = []
  165. ths = []
  166. mut = Mutex.new
  167.  
  168. chans.each do |i|
  169.   ths << Thread.new do
  170.     k = stat_of i
  171.     mut.synchronize do
  172.       puts "Got #{i}"
  173.       results << [i, k]
  174.     end
  175.   end
  176. end
  177.  
  178. ths.each do |i|
  179.   i.join
  180. end
  181.  
  182. File.open("out.html", "w") do |f|
  183.   f << "<!DOCTYPE html><html><body><table><tr><th>Board<th>Posts per hour</tr>"
  184.   results.sort { |a,b| a[1] <=> b[1] }.reverse.each do |b,c|
  185.     f << "<tr><td><a href='#{b}'>#{b}</a><td>#{c*3600}</tr>"
  186.   end
  187.   f << "</table></body></html>"
  188. end
  189.  
  190. binding.pry if $error
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement