package main

import (
    "bufio"
    "encoding/csv"
    "fmt"
    "net/http"
    "os"
    "strconv"
    "strings"
    "sync"
    "time"
)

type output struct {
    url        string
    statusCode int
    success    bool
}

func main() {
    workers, err := strconv.Atoi(os.Getenv("WORKERS"))
    if err != nil || workers == 0 {
        workers = 10
    }

    t := time.Now()

    file, err := os.Open("urls20.txt")
    if err != nil {
        println(err)
    }
    defer file.Close()
    scanner := bufio.NewScanner(file)
    maxBufferSize := 1024 * 20
    buf := make([]byte, 0, maxBufferSize)
    scanner.Buffer(buf, 10*1024)

    resultsFile, err := os.OpenFile("results.csv", os.O_RDONLY|os.O_CREATE, 0666)
    if err != nil {
        panic(fmt.Sprintf("Cannot open/create file. Error: %s", err))
    }
    defer resultsFile.Close()
    writer := csv.NewWriter(file)
    defer writer.Flush()

    urls := make(chan string)
    results := make(chan output)

    client := http.Client{
        Timeout: time.Second * 1,
    }

    wg := sync.WaitGroup{}
    go func() {
        for scanner.Scan() {
            urls <- strings.TrimSpace(scanner.Text())
        }
        close(urls)
    }()

    for i := 0; i < workers; i++ {
        wg.Add(1)
        go func() {
            defer doneWaitGroup("Reader worker finished", wg)
            count := 0
            for url := range urls {
                count++
                fmt.Println("Reading url", count)
                results <- doSomething(url, client)
            }
        }()
    }
    defer close(results)
    for i := 0; i < 1; i++ {
        wg.Add(1)
        go func() {
            defer doneWaitGroup("Writer worker finished", wg)
            count := 0
            for result := range results {
                count++
                _ = result
                fmt.Println("Wrote url", count)
            }
        }()
    }

    wg.Wait()
    elapsed := time.Since(t)
    fmt.Println("Finished in", elapsed)
    if err != nil {
        panic("Error closing file")
    }
}

func doneWaitGroup(message string, wg sync.WaitGroup) {
    fmt.Println(message)
    wg.Done()
}

func doSomething(url string, client http.Client) output {
    var result output
    req, err := http.NewRequest("HEAD", url, nil)
    if err != nil {
        panic(fmt.Sprintf("Preparing request failed. Error: %s", err))
    }
    resp, err := client.Do(req)
    if err != nil {
        result.success = false
        result.statusCode = -1
        result.url = url
        return result
    }

    if resp.StatusCode < 300 && resp.StatusCode > 199 {
        result.success = true
    } else {
        result.success = false
    }
    result.statusCode = resp.StatusCode
    result.url = url

    return result
}