Advertisement
Guest User

Untitled

a guest
Mar 30th, 2014
45
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
Scala 2.97 KB | None | 0 0
  1. package parser.theiphonewiki
  2.  
  3. import akka.actor.Actor
  4. import akka.event.Logging
  5.  
  6. import org.rovak.scraper.ScrapeManager._
  7. import org.jsoup.nodes.Element
  8.  
  9. import play.api.libs.json._
  10.  
  11. import scala.collection.Seq
  12. import scala.collection.mutable.LinkedList
  13.  
  14. object WikiiPhone {
  15.  
  16.   val results = ".wikitable"
  17.  
  18.   def url = "http://theiphonewiki.com/wiki/IPhone"
  19.  
  20. }
  21.  
  22. class WikiiPhoneScraperActor extends Actor {
  23.  
  24.   val log = Logging(context.system, this)
  25.  
  26.   var result: Option[String] = None
  27.  
  28.   def receive = {
  29.     case "run" => if (!run) sender ! ("outdated", WikiiPhone.url)
  30.     case "collect" => sender ! ("result", WikiiPhone.url, result)
  31.     case _ => log.info("Bad message to theiphonewiki.WikiiPhoneScraperActor!")
  32.   }
  33.  
  34.   def run :Boolean = {
  35.     var good = true
  36.     scrape from WikiiPhone.url open { implicit page =>
  37.       WikiiPhone.results each { table: Element =>
  38.         if (table.children.size == 1) {
  39.           val tbody = table.children.get(0)
  40.           if (tbody.children.size > 1) {
  41.             val tr1Children = tbody.children.get(0).children
  42.             val tr2Children = tbody.children.get(1).children
  43.  
  44.             var commonNamesList:LinkedList[(String, LinkedList[String])] = LinkedList()
  45.             var j = 0
  46.  
  47.             for (i <- 1 until tr1Children.size) {
  48.               val rowUp = tr1Children.get(i)
  49.               var upperName:String = null
  50.               if (rowUp.children.size == 0) {
  51.                 good = false
  52.               } else {
  53.                 upperName = rowUp.children.get(0).ownText
  54.               }
  55.  
  56.               val colspan = rowUp.attr("colspan")
  57.               var subList:LinkedList[String] = LinkedList()
  58.               if (colspan != null && colspan != "") {
  59.                 val target = j + colspan.toInt
  60.                 while (j != target) {
  61.                   val rowDown = tr2Children.get(j)
  62.                   if (rowDown.children.size == 0) {
  63.                     good = false
  64.                   } else {
  65.                     subList = subList :+ rowDown.children.get(0).ownText
  66.                   }
  67.  
  68.                   j += 1
  69.                 }
  70.               }
  71.  
  72.               if (good) {
  73.                 commonNamesList = commonNamesList :+ (upperName, subList)
  74.               }
  75.             }
  76.  
  77.             var jsonArray = new JsArray
  78.             for (tuple <- commonNamesList) {
  79.               val subSeq = Seq(tuple._2)
  80.  
  81.               val obj = Json.toJson(Map(
  82.                 "model" -> Json.toJson(tuple._1),
  83.                 "factory_names" -> Json.toJson(subSeq)
  84.               ))
  85.  
  86.               jsonArray = jsonArray :+ obj
  87.             }
  88.  
  89.             val json = Json.toJson(Map(
  90.               "url" -> Json.toJson(WikiiPhone.url),
  91.               "results" -> jsonArray
  92.             ))
  93.  
  94.             result = Some(json.toString)
  95.             good = false
  96.           } else {
  97.             good = false
  98.           }
  99.         } else {
  100.           good = false
  101.         }
  102.       }
  103.     }
  104.  
  105.     good
  106.   }
  107.  
  108. }
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement