Advertisement
Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- package parser.theiphonewiki
- import akka.actor.Actor
- import akka.event.Logging
- import org.rovak.scraper.ScrapeManager._
- import org.jsoup.nodes.Element
- import play.api.libs.json._
- import scala.collection.Seq
- import scala.collection.mutable.LinkedList
- object WikiiPhone {
- val results = ".wikitable"
- def url = "http://theiphonewiki.com/wiki/IPhone"
- }
- class WikiiPhoneScraperActor extends Actor {
- val log = Logging(context.system, this)
- var result: Option[String] = None
- def receive = {
- case "run" => if (!run) sender ! ("outdated", WikiiPhone.url)
- case "collect" => sender ! ("result", WikiiPhone.url, result)
- case _ => log.info("Bad message to theiphonewiki.WikiiPhoneScraperActor!")
- }
- def run :Boolean = {
- var good = true
- scrape from WikiiPhone.url open { implicit page =>
- WikiiPhone.results each { table: Element =>
- if (table.children.size == 1) {
- val tbody = table.children.get(0)
- if (tbody.children.size > 1) {
- val tr1Children = tbody.children.get(0).children
- val tr2Children = tbody.children.get(1).children
- var commonNamesList:LinkedList[(String, LinkedList[String])] = LinkedList()
- var j = 0
- for (i <- 1 until tr1Children.size) {
- val rowUp = tr1Children.get(i)
- var upperName:String = null
- if (rowUp.children.size == 0) {
- good = false
- } else {
- upperName = rowUp.children.get(0).ownText
- }
- val colspan = rowUp.attr("colspan")
- var subList:LinkedList[String] = LinkedList()
- if (colspan != null && colspan != "") {
- val target = j + colspan.toInt
- while (j != target) {
- val rowDown = tr2Children.get(j)
- if (rowDown.children.size == 0) {
- good = false
- } else {
- subList = subList :+ rowDown.children.get(0).ownText
- }
- j += 1
- }
- }
- if (good) {
- commonNamesList = commonNamesList :+ (upperName, subList)
- }
- }
- var jsonArray = new JsArray
- for (tuple <- commonNamesList) {
- val subSeq = Seq(tuple._2)
- val obj = Json.toJson(Map(
- "model" -> Json.toJson(tuple._1),
- "factory_names" -> Json.toJson(subSeq)
- ))
- jsonArray = jsonArray :+ obj
- }
- val json = Json.toJson(Map(
- "url" -> Json.toJson(WikiiPhone.url),
- "results" -> jsonArray
- ))
- result = Some(json.toString)
- good = false
- } else {
- good = false
- }
- } else {
- good = false
- }
- }
- }
- good
- }
- }
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement