daily pastebin goal
85%
SHARE
TWEET

Untitled

a guest Feb 19th, 2019 57 Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
  1. package main
  2.  
  3. import (
  4.     "fmt"
  5.     "log"
  6.     "strconv"
  7.     "net/http"
  8.     "github.com/PuerkitoBio/goquery"
  9. )
  10.  
  11. // Custom user agent.
  12. const (
  13.     userAgent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) " +
  14.         "AppleWebKit/537.36 (KHTML, like Gecko) " +
  15.         "Chrome/53.0.2785.143 " +
  16.         "Safari/537.36"
  17. )
  18.  
  19. func getTrns() []string
  20.  
  21. // fetchUrl opens a url with GET method and sets a custom user agent.
  22. // If url cannot be opened, then log it to a dedicated channel.
  23. func fetchUrl(url string, chFailedUrls chan string, chIsFinished chan bool) {
  24.  
  25.     client := &http.Client{}
  26.     req, _ := http.NewRequest("GET", url, nil)
  27.     req.Header.Set("User-Agent", userAgent)
  28.     resp, err := client.Do(req)
  29.     if err != nil || resp.StatusCode != 200 {
  30.         fmt.Println("Url failed:", url)
  31.         chFailedUrls <- url
  32.     }
  33.     defer resp.Body.Close()
  34.  
  35.     doc, err := goquery.NewDocumentFromReader(resp.Body)
  36.     if err != nil {
  37.         log.Fatal(err)
  38.     }
  39.     doc.Find(".tournament-name").Each(
  40.         func(index int, item *goquery.Selection) {
  41.             trnLinkTag := item.Find("a")
  42.             link, _ := trnLinkTag.Attr("href")
  43.             fmt.Println(link)
  44.     })
  45.  
  46.     defer func() {
  47.         chIsFinished <- true
  48.     }()
  49. }
  50.  
  51. func main() {
  52.     var first_year = 1980
  53.     var last_year = 2018
  54.     var base_url = "https://www.pgatour.com/tournaments/schedule.history."
  55.  
  56.     // Create 2 channels, 1 to track urls we could not open
  57.     // and 1 to inform url fetching is done:
  58.     chFailedUrls := make(chan string)
  59.     chIsFinished := make(chan bool)
  60.     urlsList := make([]string,0)
  61.  
  62.     // Open all urls concurrently using the 'go' keyword:
  63.     for yr := first_year; yr <= last_year; yr++ {
  64.         var url = base_url + strconv.Itoa(yr) + ".html"
  65.         urlsList = append(urlsList, url)
  66.         go fetchUrl(url, chFailedUrls, chIsFinished)
  67.     }
  68.  
  69.     failedUrls := make([]string, 0)
  70.     for i := 0; i < len(urlsList); {
  71.         select {
  72.         case url := <-chFailedUrls:
  73.             failedUrls = append(failedUrls, url)
  74.         case <-chIsFinished:
  75.             i++
  76.         }
  77.     }
  78. }
RAW Paste Data
We use cookies for various purposes including analytics. By continuing to use Pastebin, you agree to our use of cookies as described in the Cookies Policy. OK, I Understand
 
Top