Advertisement
Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- package main
- import (
- "fmt"
- "log"
- "strconv"
- "net/http"
- "github.com/PuerkitoBio/goquery"
- )
- // Custom user agent.
- const (
- userAgent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) " +
- "AppleWebKit/537.36 (KHTML, like Gecko) " +
- "Chrome/53.0.2785.143 " +
- "Safari/537.36"
- )
- func getTrns() []string
- // fetchUrl opens a url with GET method and sets a custom user agent.
- // If url cannot be opened, then log it to a dedicated channel.
- func fetchUrl(url string, chFailedUrls chan string, chIsFinished chan bool) {
- client := &http.Client{}
- req, _ := http.NewRequest("GET", url, nil)
- req.Header.Set("User-Agent", userAgent)
- resp, err := client.Do(req)
- if err != nil || resp.StatusCode != 200 {
- fmt.Println("Url failed:", url)
- chFailedUrls <- url
- }
- defer resp.Body.Close()
- doc, err := goquery.NewDocumentFromReader(resp.Body)
- if err != nil {
- log.Fatal(err)
- }
- doc.Find(".tournament-name").Each(
- func(index int, item *goquery.Selection) {
- trnLinkTag := item.Find("a")
- link, _ := trnLinkTag.Attr("href")
- fmt.Println(link)
- })
- defer func() {
- chIsFinished <- true
- }()
- }
- func main() {
- var first_year = 1980
- var last_year = 2018
- var base_url = "https://www.pgatour.com/tournaments/schedule.history."
- // Create 2 channels, 1 to track urls we could not open
- // and 1 to inform url fetching is done:
- chFailedUrls := make(chan string)
- chIsFinished := make(chan bool)
- urlsList := make([]string,0)
- // Open all urls concurrently using the 'go' keyword:
- for yr := first_year; yr <= last_year; yr++ {
- var url = base_url + strconv.Itoa(yr) + ".html"
- urlsList = append(urlsList, url)
- go fetchUrl(url, chFailedUrls, chIsFinished)
- }
- failedUrls := make([]string, 0)
- for i := 0; i < len(urlsList); {
- select {
- case url := <-chFailedUrls:
- failedUrls = append(failedUrls, url)
- case <-chIsFinished:
- i++
- }
- }
- }
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement