package main

import (
    "fmt"
    "net/http"
    "os"
    "time"
)

const (
    bufferSize int = 128
)

type Requester struct {
    client    *http.Client
    poolSize  int
    buffer    []string
    bufferIdx int
    attempts  int
}

type RequesterConfig struct {
    PoolSize int
    Timeout  time.Duration
    Attempts int
}

func New(config RequesterConfig) *Requester {
    requester := &Requester{
        client: &http.Client{
            Timeout: config.Timeout,
        },
        poolSize:  config.PoolSize,
        buffer:    make([]string, bufferSize),
        bufferIdx: 0,
        attempts:  config.Attempts,
    }

    return requester
}

func (r *Requester) Request(urls []string, filename string) {
    outChan := make(chan string, len(urls))
    defer close(outChan)

    r.asyncGet(urls, outChan)

    os.Remove(filename)
    for i := 0; i < len(urls); i++ {
        if r.bufferIdx == bufferSize {
            r.dumpToFile(filename)
        }

        resp := <-outChan
        r.buffer[r.bufferIdx] = resp
        r.bufferIdx++
    }

    r.dumpToFile(filename)
}

func (r *Requester) asyncGet(urls []string, outChan chan<- string) {
    inChan := make(chan string, len(urls))
    for _, url := range urls {
        inChan <- url
    }
    close(inChan)

    for i := 0; i < r.poolSize; i++ {
        go r.worker(inChan, outChan)
    }
}

func (r *Requester) worker(inChan <-chan string, outChan chan<- string) {
    for url := range inChan {
        for i := 1; i <= r.attempts; i++ {
            response, err := r.client.Get(url)
            if err == nil {
                defer response.Body.Close()
                outChan <- fmt.Sprintf("%s %d\n", url, response.StatusCode)

                if response.StatusCode < 500 {
                    break
                }
            }
        }
    }
}

func (r *Requester) dumpToFile(filename string) {
    outfile, err := os.OpenFile(filename, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0600)
    if err != nil {
        panic(err)
    }

    defer outfile.Close()

    for i := 0; i < r.bufferIdx; i++ {
        outfile.WriteString(r.buffer[i])
    }

    r.bufferIdx = 0
}

func main() {
    sites := []string{
        "https://www.avito.ru/",
        "https://www.ozon.ru/",
        "https://vk.com/",
        "https://yandex.ru/",
        "https://www.google.com/",
        "https://mail.ru/",
        "https://ok.ru/",
    }

    requester := New(RequesterConfig{
        PoolSize: 32,
        Timeout:  2 * time.Second,
        Attempts: 3,
    })
    requester.Request(sites, "./out.txt")
}