Advertisement
Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- package yalago
- import (
- "archive/zip"
- "bytes"
- "encoding/json"
- "errors"
- "fmt"
- "io"
- "io/ioutil"
- "log"
- "net/http"
- "os"
- "path/filepath"
- "strings"
- "sync"
- )
- type client struct {
- cli *http.Client
- baseURL string
- apikey string
- }
- func (c *client) fetch(url string, reqBody []byte) (body []byte, err error) {
- req, err := http.NewRequest(http.MethodPost, url, bytes.NewReader(reqBody))
- if err != nil {
- return nil, err
- }
- req.Header.Set("X-Api-Key", c.apikey)
- req.Header.Set("Content-Type", "application/json")
- resp, err := c.cli.Do(req)
- if err != nil {
- return nil, err
- }
- defer resp.Body.Close()
- if resp.StatusCode != http.StatusOK {
- return nil, errors.New(resp.Status)
- }
- body, err = ioutil.ReadAll(resp.Body)
- if err != nil {
- return nil, err
- }
- return
- }
- func (c *client) fetchLocations() ([]Location, error) {
- var provinces []Province
- var provLock sync.Mutex
- var locations []Location
- var locLock sync.Mutex
- countriesBody, err := c.fetch(c.baseURL+"/GetCountries", nil)
- if err != nil {
- return nil, err
- }
- var countries CountryResp
- err = json.Unmarshal(countriesBody, &countries)
- if len(countries.Countries) == 0 {
- return nil, errors.New("No countries!")
- }
- // NOW WE HAVE COUNTRIES
- var cChan = make(chan Country)
- for i := 0; i < 10; i++ {
- go func(in chan Country) {
- for {
- x, ok := <-in
- if !ok {
- break
- }
- provReq, _ := json.Marshal(ProvinceReq{int(x.ID)})
- provincesBody, err := c.fetch(c.baseURL+"/GetProvinces", provReq)
- if err != nil {
- log.Println("cannot fetch some provinces")
- }
- var prov ProvinceResp
- err = json.Unmarshal(provincesBody, &provinces)
- if err != nil {
- log.Println("cannot unmarshal some provinces")
- }
- for _, p := range prov.Provinces {
- provLock.Lock()
- provinces = append(provinces, Province{
- ID: p.ID,
- Title: p.Title,
- Locations: p.Locations,
- CountryID: x.ID,
- CountryTitle: x.Title,
- })
- provLock.Unlock()
- }
- }
- }(cChan)
- }
- for _, c := range countries.Countries {
- cChan <- c
- }
- close(cChan)
- // NOW WE HAVE PROVINCES
- var pChan = make(chan Province)
- for i := 0; i < 20; i++ {
- go func(in chan Province) {
- for {
- x, ok := <-in
- if !ok {
- break
- }
- locReq, _ := json.Marshal(
- LocationReq{
- CountryID: int(x.ID),
- ProvinceID: int(x.CountryID),
- },
- )
- locationBody, err := c.fetch(c.baseURL+"/GetLocations", locReq)
- if err != nil {
- return nil, err
- }
- var locs LocationResp
- err = json.Unmarshal(locationBody, &locs)
- for _, vl := range locs.Locations {
- locLock.Lock()
- locations = append(locations, Location{
- ID: ID(vl.LocationID),
- CountryID: vc.ID,
- CountryTitle: vc.Title,
- ProvinceID: vp.ID,
- ProvinceTitle: vp.Title,
- LocationTitle: vl.Title,
- },
- )
- locLock.Unlock()
- }
- }
- }(cChan)
- }
- for _, c := range countries.Countries {
- cChan <- c
- }
- close(cChan)
- }
- //func (c *client) fetchLocations() ([]Location, error) {
- //countriesBody, err := c.fetch(c.baseURL+"/GetCountries", nil)
- //if err != nil {
- //return nil, err
- //}
- //var countries CountryResp
- //err = json.Unmarshal(countriesBody, &countries)
- //if len(countries.Countries) == 0 {
- //return nil, errors.New("No countries!")
- //}
- //type locationMap struct {
- //sync.Mutex
- //LocSlice []Location
- //}
- //lc := new(locationMap)
- //for _, vc := range countries.Countries {
- //provReq, _ := json.Marshal(ProvinceReq{int(vc.ID)})
- //provincesBody, err := c.fetch(c.baseURL+"/GetProvinces", provReq)
- //if err != nil {
- //return nil, err
- //}
- //var provinces ProvinceResp
- //err = json.Unmarshal(provincesBody, &provinces)
- //for _, vp := range provinces.Provinces {
- //locReq, _ := json.Marshal(
- //LocationReq{
- //CountryID: int(vc.ID),
- //ProvinceID: int(vp.ID),
- //},
- //)
- //locationBody, err := c.fetch(c.baseURL+"/GetLocations", locReq)
- //if err != nil {
- //return nil, err
- //}
- //var locs LocationResp
- //err = json.Unmarshal(locationBody, &locs)
- //for _, vl := range locs.Locations {
- //lc.Lock()
- //lc.LocSlice = append(lc.LocSlice, Location{
- //ID: ID(vl.LocationID),
- //CountryID: vc.ID,
- //CountryTitle: vc.Title,
- //ProvinceID: vp.ID,
- //ProvinceTitle: vp.Title,
- //LocationTitle: vl.Title,
- //},
- //)
- //lc.Unlock()
- //}
- //}
- //}
- //return lc.LocSlice, err
- //}
- func (c *client) fetchEstablishmentsList(locSlice []Location) ([]ShortEstablishment, error) {
- var err error
- type establishmentMap struct {
- sync.Mutex
- EstMap map[int]ShortEstablishment
- EstSlice []ShortEstablishment
- }
- et := new(establishmentMap)
- for _, v := range locSlice {
- estReq, _ := json.Marshal(EstablishmentsReq{
- CountryID: v.CountryID,
- ProvinceID: v.ProvinceID,
- LocationID: int(v.ID),
- Languages: []string{"en"},
- })
- estBody, err := c.fetch(c.baseURL+"/GetEstablishments", estReq)
- if err != nil {
- log.Println("Can't get establishments by request: ", string(estReq))
- }
- var establishments EstablishmentResp
- err = json.Unmarshal(estBody, &establishments)
- for _, e := range establishments.Establishments {
- et.Lock()
- //et.EstMap[int(e.ID)] = e
- et.EstSlice = append(et.EstSlice, e)
- log.Println("find estID: ", int(e.ID))
- et.Unlock()
- }
- }
- return et.EstSlice, err
- }
- func (c *client) fetchAndWriteEstablishmentDetails(establishments []ShortEstablishment, fileLoc string) {
- for _, v := range establishments {
- estDetReq, _ := json.Marshal(
- EstDetailReq{
- EstablishmentID: int(v.ID),
- Languages: extraLangMap,
- },
- )
- estBody, err := c.fetch(c.baseURL+"/GetEstablishment", estDetReq)
- if err != nil {
- log.Println("cannot fetch Establishment")
- }
- var est DetailEstablishmentResp
- err = json.Unmarshal(estBody, &est)
- tmpFile, _ := json.Marshal(est.Establishment)
- log.Println("write file Establishment ", int(est.Establishment.ID))
- err = ioutil.WriteFile(fmt.Sprintf("%s/est%d.json", fileLoc, int(est.Establishment.ID)), tmpFile, 0644)
- }
- return
- }
- func (c *client) fetchAndWriteFacilities(fileLoc string) {
- langs, _ := json.Marshal(extraLangMap)
- body, err := c.fetch(c.baseURL+"/GetFacilities", langs)
- if err != nil {
- log.Println("cannot fetch Facilities")
- }
- var fclt FacilityResp
- err = json.Unmarshal(body, &fclt)
- tmpFile, _ := json.Marshal(fclt.Facilities)
- log.Println("write file Facilities")
- err = ioutil.WriteFile(fmt.Sprintf("%s/facilities.json", fileLoc), tmpFile, 0644)
- if err != nil {
- log.Println(err.Error())
- }
- return
- }
- func downloadZip(fileName, url string) error {
- resp, err := http.Get(url)
- if err != nil {
- return err
- }
- defer resp.Body.Close()
- if resp.StatusCode != http.StatusOK {
- return errors.New(resp.Status)
- }
- f, err := os.Create(fileName)
- if err != nil {
- return err
- }
- if _, err = io.Copy(f, resp.Body); err != nil {
- f.Close()
- return err
- }
- f.Close()
- return err
- }
- // Solution by golangcode.com/unzip-files-in-go/
- func unzipAndDelete(location, src string) ([]string, error) {
- var filenames []string
- zipReader, err := zip.OpenReader(src)
- defer zipReader.Close()
- defer os.Remove(src)
- if err != nil {
- return filenames, err
- }
- if len(zipReader.File) == 0 {
- return filenames, errors.New("no files in archive")
- }
- for _, f := range zipReader.File {
- rc, err := f.Open()
- if err != nil {
- return filenames, err
- }
- defer rc.Close()
- // Store filename/path for returning and using later on
- fpath := filepath.Join(location, f.Name)
- filenames = append(filenames, fpath)
- if f.FileInfo().IsDir() {
- // Make Folder
- os.MkdirAll(fpath, os.ModePerm)
- } else {
- // Make File
- var fdir string
- if lastIndex := strings.LastIndex(fpath, string(os.PathSeparator)); lastIndex > -1 {
- fdir = fpath[:lastIndex]
- }
- err = os.MkdirAll(fdir, os.ModePerm)
- if err != nil {
- log.Fatal(err)
- return filenames, err
- }
- f, err := os.OpenFile(
- fpath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, f.Mode())
- if err != nil {
- return filenames, err
- }
- defer f.Close()
- _, err = io.Copy(f, rc)
- if err != nil {
- return filenames, err
- }
- }
- }
- return filenames, nil
- }
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement