daily pastebin goal
68%
SHARE
TWEET

Untitled

a guest Apr 9th, 2011 199 Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
  1. #!/usr/bin/env ruby
  2. # encoding: utf-8
  3. =begin
  4. Permission is hereby granted, free of charge, to any person obtaining a copy
  5. of this software and associated documentation files (the "Software"), to deal
  6. in the Software without restriction, including without limitation the rights
  7. to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
  8. copies of the Software, and to permit persons to whom the Software is
  9. furnished to do so, subject to the following conditions:
  10.  
  11. The above copyright notice and this permission notice shall be included in
  12. all copies or substantial portions of the Software.
  13.  
  14. The Software shall be used for Good, not Evil.
  15.  
  16. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17. IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18. FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  19. AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  20. LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  21. OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  22. SOFTWARE.
  23. =end
  24.  
  25. require 'rubygems'
  26. require 'nokogiri'
  27. require 'open-uri'
  28. require 'net/http'
  29.  
  30.  
  31. BASE_URL = 'http://www.yle.fi'
  32. File.open("yle_vaalikone.txt", 'w'){ |f| f.write("") }
  33.  
  34.  
  35.  
  36. def scrape_constituency(url)
  37.   list_of_cands = Nokogiri::HTML(open(url))
  38.   district = list_of_cands.xpath("//div[@id='em-headingbar']/h2[1]").inner_html
  39.   district = district.chomp(": Kaikki ehdokkaat")
  40.  
  41.   links = list_of_cands.xpath("//td[@class='em-cell-name']/a").collect[0..-1]
  42.   cand_links = links.grep /\d\s/
  43.  
  44.   cand_links.each do |el|
  45.     full_url = BASE_URL + el["href"]
  46.     scrape_candidate(full_url, district)
  47.   end
  48. end
  49.  
  50.  
  51. def scrape_candidate(url,district)
  52.   cand = {:district => district}
  53.   open_page = Nokogiri::HTML(open(url), nil, 'iso8859-1')
  54.  
  55.   # SCRAPE: GET INFO
  56.   cand_info = open_page.xpath("//div[@class='em-details']")
  57.   cand_name = cand_info.xpath("h3[1]")
  58.   cand[:name] = cand_name.inner_html
  59.  
  60.   # get party
  61.   find_cell = cand_info.xpath("dl[1]/dt[2]").to_s
  62.   cand_party_raw =
  63.     find_cell.include?('Puolue') ?
  64.       cand_info.xpath("dl[1]/dd[2]") : cand_info.xpath("dl[1]/dd[1]")
  65.   cand_party = cand_party_raw.inner_html
  66.   cand[:party] = cand_party[/(.*)\s<br>/,1].strip
  67.  
  68.   # get age
  69.   find_cell = cand_info.xpath("dl[1]/dt[4]").to_s
  70.   cand_age_raw =
  71.     find_cell.include?('Ik') ?
  72.       cand_info.xpath("dl[1]/dd[4]") : cand_info.xpath("dl[1]/dd[3]")
  73.   cand[:age] = cand_age_raw.inner_html
  74.  
  75.   # SCRAPE: GET ANSWERS
  76.   cand[:opts] = [] # numeeriset vastaukset
  77.   cand[:xopts] = [] # monivalinta
  78.   q = open_page.xpath("//div[@class='em-compare-container']").collect[0..32]
  79.   q.each do |a|
  80.     question = a.xpath("h3").inner_html.strip
  81.     qa = {:q => question}
  82.     opt = a.xpath("table[1]/tr").collect[0..-1]
  83.     opt.each_with_index do |b,opt_count|
  84.       if b.xpath("td[1]").to_html.include? "acronym" # "&#9679;" html-merkki ongelmallinen
  85.         # ehdokas on tätä mieltä..
  86.         cleartxt = b.xpath("td[4]").inner_html.strip
  87.         # talletetaan myös numeroarvo (1-5)
  88.         qa.update({:a => cleartxt, :n => opt_count+1})
  89.       end
  90.     end
  91.     # rekisteröidään myös tyhjät vastaukset
  92.     cand[:opts] << qa
  93.   end
  94.  
  95.   ### MONIVALINNAT
  96.  
  97.   # q34 - Mielestäni seuraavassa hallituksessa on oltava mukana:
  98.   # q35 - Suosikkini tulevan hallituksen pääministeriksi on (vain yksi):
  99.   (34..35).each do |q|
  100.     _q = open_page.xpath("//div[@class='em-compare-container'][%i]" % [q])
  101.     question = _q.xpath("h3").inner_html.strip
  102.     #puts question
  103.     opt = _q.xpath("table[1]/tr").collect[0..-1]
  104.     a = [] # monivalinta
  105.     opt.each do |b|
  106.       if b.xpath("td[1]").to_html.include? "acronym"
  107.         a << b.xpath("td[4]").inner_html.strip
  108.       end
  109.     end
  110.     cand[:xopts] << {:q => question, :a => a.join(', ')}
  111.   end
  112.  
  113.   ### TULOSTA JA TALLENNA
  114.   print_and_save cand
  115.   sleep 2 # älä kuormita YLEn palvelimia liikaa..
  116.  
  117. end
  118.  
  119.  
  120. def print_and_save(cand)
  121.   f = File.open("yle_vaalikone.txt", 'a')
  122.   f.write "***\n"
  123.   f.write [
  124.     cand[:name],
  125.     cand[:district],
  126.     cand[:party],
  127.     cand[:age]
  128.   ].join("\t")
  129.   f.write "\n"
  130.  
  131.   # numeeriset vastaukset pilkulla eroteltuina
  132.   f.write cand[:opts].map{|h| h[:n]}.join(',')
  133.   f.write "\n"
  134.  
  135.   f.write cand[:xopts].collect{|h| "%s %s" % [h[:q], h[:a]]}.join("\n")
  136.   f.write "\n"
  137.   f.close
  138.  
  139.   ### TULOSTA RUUDULLE
  140.  
  141.   puts [
  142.     cand[:name],
  143.     cand[:district],
  144.     cand[:party],
  145.     cand[:age],
  146.     '============================================='
  147.   ].join("\n")
  148.   (cand[:opts] + cand[:xopts]).each do |h|
  149.     # selväkielinen printti:
  150.     puts "%s %s\n\n" % [h[:q], h[:a]]
  151.   end
  152.  
  153. end
  154.  
  155.  
  156. (1..16).each do |district|
  157.   url = "%s/vaalikone11/index.php?emp=l-1.d-%i.s-3.q-1.ps-1" % [BASE_URL, district]
  158.   scrape_constituency(url)
  159. end
RAW Paste Data
We use cookies for various purposes including analytics. By continuing to use Pastebin, you agree to our use of cookies as described in the Cookies Policy. OK, I Understand
 
Top