Advertisement
Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- package main
- import (
- "fmt"
- "io"
- "log"
- "net/http"
- "os"
- "plugin"
- "strings"
- "time"
- _ "github.com/denisenkom/go-mssqldb"
- "github.com/gorilla/mux"
- "github.com/jmoiron/sqlx"
- )
- var (
- configProvidersMap map[string]*Provider
- serverConnString = fmt.Sprintf("server=%s;user id=%s;password=%s;port=%d;database=%s", "europedev\\dev14", "WILDBERRIES\\makaryuschenko.d", "Ss12341234", 1433, "WBR_Content")
- )
- type imageValid = func(ProductInfo, string) bool
- type imageURL = func(ProductInfo) []string
- type downloaders = map[string]interface{} // isImageValid, imageURL
- func init() {
- configProvidersMap = parseXML()
- }
- func connectToDB(connString string) *sqlx.DB {
- db, err := sqlx.Open("mssql", connString)
- checkerr(err)
- return db
- }
- func getRv() []ProductInfo {
- sliceOfProducts := []ProductInfo{}
- db := connectToDB(serverConnString)
- defer db.Close()
- err := db.Ping()
- checkerr(err)
- err = db.Select(&sliceOfProducts, "EXEC PhotoStudio.Photojob_GetProductsForDownload")
- checkerr(err)
- return sliceOfProducts
- }
- func getAllProducts() (products map[string][]ProductInfo) {
- products = map[string][]ProductInfo{}
- sliceOfProducts := getRv()
- for _, productFromDB := range sliceOfProducts {
- for _, v := range configProvidersMap {
- for _, brand := range v.BrandList {
- if productFromDB.BrandName == brand {
- products[v.Name] = append(products[v.Name], productFromDB)
- }
- }
- }
- }
- return
- }
- func downloadImage(product ProductInfo, c NecessarySettings, downloader interface{}) error {
- if c.ConnType == "web" {
- return totalParserWeb(product, downloader.(imageURL))
- }
- return downloadImageFTP(product, c, downloader.(imageValid))
- }
- func totalParserWeb(product ProductInfo, getImageURL imageURL) error {
- sliceOfURL := getImageURL(product)
- return downloadImageWeb(sliceOfURL)
- }
- func downloadImageFTP(product ProductInfo, c NecessarySettings, isImageValid imageValid) error {
- if c.Host == "ftp.vesta-baby.ru" {
- return downloadImageFTPWindows(product, c, isImageValid)
- }
- // if c.FileSystem == "win" {
- // return downloadImageFTPWindows(product, c, isImageValid)
- // }
- // return downloadImageFTPLinux(product, c, isImageValid)
- return nil
- }
- func downloadImageFTPWindows(product ProductInfo, c NecessarySettings, isImageValid imageValid) error {
- return FTPwindows(isImageValid, c, product)
- }
- func downloadImageFTPLinux(product ProductInfo, c NecessarySettings, isImageValid imageValid) error {
- return nil
- }
- func downloadAllProviderImages(provider string, providerProducts []ProductInfo) {
- // for _, product := range products {
- // var fileExists bool
- // for method, settings := range mapOfSettings {
- // fileExists = false
- // if settings.Enable {
- // plug, err := plugin.Open("./plugins/" + method)
- // checkerr(err)
- // symHelper, err := plug.Lookup(method)
- // checkerr(err)
- // err = downloadImage(product, *settings, symHelper)
- // if err == nil {
- // fileExists = true
- // break
- // }
- // } else {
- // fmt.Println("disabled", settings.Reason)
- // }
- // }
- // if !fileExists {
- // log.Println("failed to retrieve", product)
- // }
- // }
- configProvidersMap[provider].mx.Lock()
- for _, product := range providerProducts {
- var fileExists bool
- for method, settings := range configProvidersMap[provider].MapOfSettings {
- fileExists = false
- if settings.Enable == true {
- plug, err := plugin.Open("./plugins/" + method)
- // checkerr(err)
- if err != nil {
- continue
- }
- symHelper, err := plug.Lookup(method)
- checkerr(err)
- err = downloadImage(product, *settings, symHelper) // try to retrieve the product with all methods
- if err == nil {
- fileExists = true
- break
- }
- } else {
- fmt.Println("disabled", settings.Reason)
- }
- }
- if !fileExists {
- log.Println("failed to retrieve", product)
- }
- }
- configProvidersMap[provider].mx.Unlock()
- }
- func selectiveDownload() {
- router := mux.NewRouter()
- router.HandleFunc("/download/{provider}", handlerForProvider)
- http.ListenAndServe(":3000", router)
- }
- func handlerForProvider(w http.ResponseWriter, r *http.Request) {
- vars := mux.Vars(r)
- provider := vars["provider"]
- products := getAllProducts()
- downloadAllProviderImages(provider, products[provider])
- }
- func main() {
- go selectiveDownload()
- products := getAllProducts()
- // fmt.Println(products)
- // var products = map[string][]ProductInfo{
- // "nife": {{Article: "horev"}},
- // "ginorossi": {{Article: "denis"}},
- // }
- for {
- for provider, providerProducts := range products {
- go downloadAllProviderImages(provider, providerProducts)
- }
- // providerInterfaces.Range(func(key, value interface{}) bool {
- // fmt.Println(key)
- // downloaderInterface := value.(downloaders)
- // provider := key.(string)
- // go downloadAllProviderImages(products[provider], provider, downloaderInterface)
- // providerInterfaces.Delete(key)
- // fmt.Println(products[provider])
- // return true
- // })
- time.Sleep(time.Hour * 6)
- }
- }
- func downloadImageWeb(url []string) error {
- var out *os.File
- var err error
- var resp *http.Response
- for _, img := range url {
- if img != "" {
- nameOfFile := "default"
- sliceOfImgURL := strings.Split(img, "/")
- if len(sliceOfImgURL) > 2 {
- nameOfFile = sliceOfImgURL[2]
- }
- if out, err = os.Create(nameOfFile + ".jpg"); err != nil {
- return err
- }
- if resp, err = http.Get(img); err != nil {
- return err
- }
- if _, err := io.Copy(out, resp.Body); err != nil {
- return err
- }
- resp.Body.Close()
- out.Close()
- }
- }
- return nil
- }
- func checkerr(err error) {
- if err != nil {
- panic(err)
- }
- }
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement