SHARE
TWEET

Untitled

a guest Jan 18th, 2020 77 Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
  1. package main
  2.  
  3. import (
  4.     "bufio"
  5.     "encoding/csv"
  6.     "fmt"
  7.     "net/http"
  8.     "os"
  9.     "strconv"
  10.     "strings"
  11.     "sync"
  12.     "time"
  13. )
  14.  
  15. type output struct {
  16.     url        string
  17.     statusCode int
  18.     success    bool
  19. }
  20.  
  21. func main() {
  22.     workers, err := strconv.Atoi(os.Getenv("WORKERS"))
  23.     if err != nil || workers == 0 {
  24.         workers = 10
  25.     }
  26.  
  27.     t := time.Now()
  28.  
  29.     file, err := os.Open("urls20.txt")
  30.     if err != nil {
  31.         println(err)
  32.     }
  33.     defer file.Close()
  34.     scanner := bufio.NewScanner(file)
  35.     maxBufferSize := 1024 * 20
  36.     buf := make([]byte, 0, maxBufferSize)
  37.     scanner.Buffer(buf, 10*1024)
  38.  
  39.     resultsFile, err := os.OpenFile("results.csv", os.O_RDONLY|os.O_CREATE, 0666)
  40.     if err != nil {
  41.         panic(fmt.Sprintf("Cannot open/create file. Error: %s", err))
  42.     }
  43.     defer resultsFile.Close()
  44.     writer := csv.NewWriter(file)
  45.     defer writer.Flush()
  46.  
  47.     urls := make(chan string)
  48.     results := make(chan output)
  49.  
  50.     client := http.Client{
  51.         Timeout: time.Second * 1,
  52.     }
  53.  
  54.     wg := sync.WaitGroup{}
  55.     go func() {
  56.         for scanner.Scan() {
  57.             urls <- strings.TrimSpace(scanner.Text())
  58.         }
  59.         close(urls)
  60.     }()
  61.  
  62.     for i := 0; i < workers; i++ {
  63.         wg.Add(1)
  64.         go func() {
  65.             defer doneWaitGroup("Reader worker finished", wg)
  66.             count := 0
  67.             for url := range urls {
  68.                 count++
  69.                 fmt.Println("Reading url", count)
  70.                 results <- doSomething(url, client)
  71.             }
  72.         }()
  73.     }
  74.     defer close(results)
  75.     for i := 0; i < 1; i++ {
  76.         wg.Add(1)
  77.         go func() {
  78.             defer doneWaitGroup("Writer worker finished", wg)
  79.             count := 0
  80.             for result := range results {
  81.                 count++
  82.                 _ = result
  83.                 fmt.Println("Wrote url", count)
  84.             }
  85.         }()
  86.     }
  87.  
  88.     wg.Wait()
  89.     elapsed := time.Since(t)
  90.     fmt.Println("Finished in", elapsed)
  91.     if err != nil {
  92.         panic("Error closing file")
  93.     }
  94. }
  95.  
  96. func doneWaitGroup(message string, wg sync.WaitGroup) {
  97.     fmt.Println(message)
  98.     wg.Done()
  99. }
  100.  
  101. func doSomething(url string, client http.Client) output {
  102.     var result output
  103.     req, err := http.NewRequest("HEAD", url, nil)
  104.     if err != nil {
  105.         panic(fmt.Sprintf("Preparing request failed. Error: %s", err))
  106.     }
  107.     resp, err := client.Do(req)
  108.     if err != nil {
  109.         result.success = false
  110.         result.statusCode = -1
  111.         result.url = url
  112.         return result
  113.     }
  114.  
  115.     if resp.StatusCode < 300 && resp.StatusCode > 199 {
  116.         result.success = true
  117.     } else {
  118.         result.success = false
  119.     }
  120.     result.statusCode = resp.StatusCode
  121.     result.url = url
  122.  
  123.     return result
  124. }
RAW Paste Data
We use cookies for various purposes including analytics. By continuing to use Pastebin, you agree to our use of cookies as described in the Cookies Policy. OK, I Understand
Top