Advertisement
Guest User

Untitled

a guest
Feb 20th, 2018
86
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
Go 8.30 KB | None | 0 0
  1. package yalago
  2.  
  3. import (
  4.     "archive/zip"
  5.     "bytes"
  6.     "encoding/json"
  7.     "errors"
  8.     "fmt"
  9.     "io"
  10.     "io/ioutil"
  11.     "log"
  12.     "net/http"
  13.     "os"
  14.     "path/filepath"
  15.     "strings"
  16.     "sync"
  17. )
  18.  
  19. type client struct {
  20.     cli     *http.Client
  21.     baseURL string
  22.     apikey  string
  23. }
  24.  
  25. func (c *client) fetch(url string, reqBody []byte) (body []byte, err error) {
  26.     req, err := http.NewRequest(http.MethodPost, url, bytes.NewReader(reqBody))
  27.     if err != nil {
  28.         return nil, err
  29.     }
  30.     req.Header.Set("X-Api-Key", c.apikey)
  31.     req.Header.Set("Content-Type", "application/json")
  32.  
  33.     resp, err := c.cli.Do(req)
  34.     if err != nil {
  35.         return nil, err
  36.     }
  37.  
  38.     defer resp.Body.Close()
  39.  
  40.     if resp.StatusCode != http.StatusOK {
  41.         return nil, errors.New(resp.Status)
  42.     }
  43.     body, err = ioutil.ReadAll(resp.Body)
  44.     if err != nil {
  45.         return nil, err
  46.     }
  47.     return
  48. }
  49.  
  50. func (c *client) fetchLocations() ([]Location, error) {
  51.     var provinces []Province
  52.     var provLock sync.Mutex
  53.  
  54.     var locations []Location
  55.     var locLock sync.Mutex
  56.  
  57.     countriesBody, err := c.fetch(c.baseURL+"/GetCountries", nil)
  58.     if err != nil {
  59.         return nil, err
  60.     }
  61.     var countries CountryResp
  62.     err = json.Unmarshal(countriesBody, &countries)
  63.     if len(countries.Countries) == 0 {
  64.         return nil, errors.New("No countries!")
  65.     }
  66.     // NOW WE HAVE COUNTRIES
  67.     var cChan = make(chan Country)
  68.     for i := 0; i < 10; i++ {
  69.         go func(in chan Country) {
  70.             for {
  71.                 x, ok := <-in
  72.                 if !ok {
  73.                     break
  74.                 }
  75.                 provReq, _ := json.Marshal(ProvinceReq{int(x.ID)})
  76.                 provincesBody, err := c.fetch(c.baseURL+"/GetProvinces", provReq)
  77.                 if err != nil {
  78.                     log.Println("cannot fetch some provinces")
  79.                 }
  80.                 var prov ProvinceResp
  81.                 err = json.Unmarshal(provincesBody, &provinces)
  82.                 if err != nil {
  83.                     log.Println("cannot unmarshal some provinces")
  84.                 }
  85.                 for _, p := range prov.Provinces {
  86.                     provLock.Lock()
  87.                     provinces = append(provinces, Province{
  88.                         ID:           p.ID,
  89.                         Title:        p.Title,
  90.                         Locations:    p.Locations,
  91.                         CountryID:    x.ID,
  92.                         CountryTitle: x.Title,
  93.                     })
  94.                     provLock.Unlock()
  95.                 }
  96.  
  97.             }
  98.         }(cChan)
  99.     }
  100.     for _, c := range countries.Countries {
  101.         cChan <- c
  102.     }
  103.     close(cChan)
  104.  
  105.     // NOW WE HAVE PROVINCES
  106.     var pChan = make(chan Province)
  107.     for i := 0; i < 20; i++ {
  108.         go func(in chan Province) {
  109.             for {
  110.                 x, ok := <-in
  111.                 if !ok {
  112.                     break
  113.                 }
  114.                 locReq, _ := json.Marshal(
  115.                     LocationReq{
  116.                         CountryID:  int(x.ID),
  117.                         ProvinceID: int(x.CountryID),
  118.                     },
  119.                 )
  120.                 locationBody, err := c.fetch(c.baseURL+"/GetLocations", locReq)
  121.                 if err != nil {
  122.                     return nil, err
  123.                 }
  124.                 var locs LocationResp
  125.                 err = json.Unmarshal(locationBody, &locs)
  126.                 for _, vl := range locs.Locations {
  127.                     locLock.Lock()
  128.                     locations = append(locations, Location{
  129.                         ID:            ID(vl.LocationID),
  130.                         CountryID:     vc.ID,
  131.                         CountryTitle:  vc.Title,
  132.                         ProvinceID:    vp.ID,
  133.                         ProvinceTitle: vp.Title,
  134.                         LocationTitle: vl.Title,
  135.                     },
  136.                     )
  137.                     locLock.Unlock()
  138.                 }
  139.  
  140.             }
  141.         }(cChan)
  142.     }
  143.     for _, c := range countries.Countries {
  144.         cChan <- c
  145.     }
  146.     close(cChan)
  147.  
  148. }
  149.  
  150. //func (c *client) fetchLocations() ([]Location, error) {
  151. //countriesBody, err := c.fetch(c.baseURL+"/GetCountries", nil)
  152. //if err != nil {
  153. //return nil, err
  154. //}
  155. //var countries CountryResp
  156. //err = json.Unmarshal(countriesBody, &countries)
  157. //if len(countries.Countries) == 0 {
  158. //return nil, errors.New("No countries!")
  159. //}
  160. //type locationMap struct {
  161. //sync.Mutex
  162. //LocSlice []Location
  163. //}
  164. //lc := new(locationMap)
  165. //for _, vc := range countries.Countries {
  166. //provReq, _ := json.Marshal(ProvinceReq{int(vc.ID)})
  167. //provincesBody, err := c.fetch(c.baseURL+"/GetProvinces", provReq)
  168. //if err != nil {
  169. //return nil, err
  170. //}
  171. //var provinces ProvinceResp
  172. //err = json.Unmarshal(provincesBody, &provinces)
  173. //for _, vp := range provinces.Provinces {
  174. //locReq, _ := json.Marshal(
  175. //LocationReq{
  176. //CountryID:  int(vc.ID),
  177. //ProvinceID: int(vp.ID),
  178. //},
  179. //)
  180. //locationBody, err := c.fetch(c.baseURL+"/GetLocations", locReq)
  181. //if err != nil {
  182. //return nil, err
  183. //}
  184. //var locs LocationResp
  185. //err = json.Unmarshal(locationBody, &locs)
  186. //for _, vl := range locs.Locations {
  187. //lc.Lock()
  188. //lc.LocSlice = append(lc.LocSlice, Location{
  189. //ID:            ID(vl.LocationID),
  190. //CountryID:     vc.ID,
  191. //CountryTitle:  vc.Title,
  192. //ProvinceID:    vp.ID,
  193. //ProvinceTitle: vp.Title,
  194. //LocationTitle: vl.Title,
  195. //},
  196. //)
  197. //lc.Unlock()
  198. //}
  199.  
  200. //}
  201.  
  202. //}
  203. //return lc.LocSlice, err
  204.  
  205. //}
  206.  
  207. func (c *client) fetchEstablishmentsList(locSlice []Location) ([]ShortEstablishment, error) {
  208.     var err error
  209.     type establishmentMap struct {
  210.         sync.Mutex
  211.         EstMap   map[int]ShortEstablishment
  212.         EstSlice []ShortEstablishment
  213.     }
  214.     et := new(establishmentMap)
  215.     for _, v := range locSlice {
  216.         estReq, _ := json.Marshal(EstablishmentsReq{
  217.             CountryID:  v.CountryID,
  218.             ProvinceID: v.ProvinceID,
  219.             LocationID: int(v.ID),
  220.             Languages:  []string{"en"},
  221.         })
  222.         estBody, err := c.fetch(c.baseURL+"/GetEstablishments", estReq)
  223.         if err != nil {
  224.             log.Println("Can't get establishments by request: ", string(estReq))
  225.         }
  226.         var establishments EstablishmentResp
  227.         err = json.Unmarshal(estBody, &establishments)
  228.         for _, e := range establishments.Establishments {
  229.             et.Lock()
  230.             //et.EstMap[int(e.ID)] = e
  231.             et.EstSlice = append(et.EstSlice, e)
  232.             log.Println("find estID: ", int(e.ID))
  233.             et.Unlock()
  234.         }
  235.     }
  236.     return et.EstSlice, err
  237. }
  238. func (c *client) fetchAndWriteEstablishmentDetails(establishments []ShortEstablishment, fileLoc string) {
  239.     for _, v := range establishments {
  240.         estDetReq, _ := json.Marshal(
  241.             EstDetailReq{
  242.                 EstablishmentID: int(v.ID),
  243.                 Languages:       extraLangMap,
  244.             },
  245.         )
  246.         estBody, err := c.fetch(c.baseURL+"/GetEstablishment", estDetReq)
  247.         if err != nil {
  248.             log.Println("cannot fetch Establishment")
  249.         }
  250.         var est DetailEstablishmentResp
  251.         err = json.Unmarshal(estBody, &est)
  252.         tmpFile, _ := json.Marshal(est.Establishment)
  253.         log.Println("write file Establishment ", int(est.Establishment.ID))
  254.         err = ioutil.WriteFile(fmt.Sprintf("%s/est%d.json", fileLoc, int(est.Establishment.ID)), tmpFile, 0644)
  255.     }
  256.     return
  257. }
  258.  
  259. func (c *client) fetchAndWriteFacilities(fileLoc string) {
  260.     langs, _ := json.Marshal(extraLangMap)
  261.     body, err := c.fetch(c.baseURL+"/GetFacilities", langs)
  262.     if err != nil {
  263.         log.Println("cannot fetch Facilities")
  264.     }
  265.     var fclt FacilityResp
  266.     err = json.Unmarshal(body, &fclt)
  267.     tmpFile, _ := json.Marshal(fclt.Facilities)
  268.     log.Println("write file Facilities")
  269.     err = ioutil.WriteFile(fmt.Sprintf("%s/facilities.json", fileLoc), tmpFile, 0644)
  270.     if err != nil {
  271.         log.Println(err.Error())
  272.     }
  273.     return
  274. }
  275.  
  276. func downloadZip(fileName, url string) error {
  277.     resp, err := http.Get(url)
  278.     if err != nil {
  279.         return err
  280.     }
  281.     defer resp.Body.Close()
  282.     if resp.StatusCode != http.StatusOK {
  283.         return errors.New(resp.Status)
  284.     }
  285.  
  286.     f, err := os.Create(fileName)
  287.     if err != nil {
  288.         return err
  289.     }
  290.     if _, err = io.Copy(f, resp.Body); err != nil {
  291.         f.Close()
  292.         return err
  293.     }
  294.     f.Close()
  295.     return err
  296. }
  297.  
  298. // Solution by golangcode.com/unzip-files-in-go/
  299. func unzipAndDelete(location, src string) ([]string, error) {
  300.     var filenames []string
  301.  
  302.     zipReader, err := zip.OpenReader(src)
  303.     defer zipReader.Close()
  304.     defer os.Remove(src)
  305.     if err != nil {
  306.         return filenames, err
  307.     }
  308.     if len(zipReader.File) == 0 {
  309.         return filenames, errors.New("no files in archive")
  310.     }
  311.  
  312.     for _, f := range zipReader.File {
  313.  
  314.         rc, err := f.Open()
  315.         if err != nil {
  316.             return filenames, err
  317.         }
  318.         defer rc.Close()
  319.  
  320.         // Store filename/path for returning and using later on
  321.         fpath := filepath.Join(location, f.Name)
  322.         filenames = append(filenames, fpath)
  323.  
  324.         if f.FileInfo().IsDir() {
  325.  
  326.             // Make Folder
  327.             os.MkdirAll(fpath, os.ModePerm)
  328.  
  329.         } else {
  330.  
  331.             // Make File
  332.             var fdir string
  333.             if lastIndex := strings.LastIndex(fpath, string(os.PathSeparator)); lastIndex > -1 {
  334.                 fdir = fpath[:lastIndex]
  335.             }
  336.  
  337.             err = os.MkdirAll(fdir, os.ModePerm)
  338.             if err != nil {
  339.                 log.Fatal(err)
  340.                 return filenames, err
  341.             }
  342.             f, err := os.OpenFile(
  343.                 fpath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, f.Mode())
  344.             if err != nil {
  345.                 return filenames, err
  346.             }
  347.             defer f.Close()
  348.  
  349.             _, err = io.Copy(f, rc)
  350.             if err != nil {
  351.                 return filenames, err
  352.             }
  353.  
  354.         }
  355.     }
  356.     return filenames, nil
  357. }
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement