grab the articles at ptt.cc

library(xml2)
library(RCurl)
library(magrittr)
library(stringi)

i <- 2793
data <- NULL
repeat{
  url.list <- try(paste(i, '.html', sep='') %>%
                         paste('https://www.ptt.cc/bbs/Boy-Girl/index', ., sep='') %>%
                         xml2::read_html() %>% xml2::xml_find_all("//div[@class='title']/a[@href]") %>%
                         xml2::xml_attr("href"), silent = TRUE)
  if (any(grepl("Error", url.list)))
    break
  data %<>% c(url.list)
  i <- i + 1
}
data %<>% paste('https://www.ptt.cc', ., sep='')

getDoc <- function(line){
  xml2::read_html(line, "UTF-8") %>% xml2::xml_find_one("//div[@id='main-content']") %>%
    xml_text %>% stri_conv("UTF-8", "Big5")
  # stri_conv for windows
}

st <- proc.time()
articles <- sapply(data, getDoc)
proc.time() - st
# user  system elapsed
# 3.35    3.70    7.21