Advertisement
Guest User

Untitled

a guest
Jul 23rd, 2018
94
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 5.70 KB | None | 0 0
  1. package main
  2.  
  3. import (
  4. "fmt"
  5. "io"
  6. "log"
  7. "net/http"
  8. "os"
  9. "plugin"
  10. "strings"
  11. "time"
  12.  
  13. _ "github.com/denisenkom/go-mssqldb"
  14. "github.com/gorilla/mux"
  15. "github.com/jmoiron/sqlx"
  16. )
  17.  
  18. var (
  19. configProvidersMap map[string]*Provider
  20. serverConnString = fmt.Sprintf("server=%s;user id=%s;password=%s;port=%d;database=%s", "europedev\\dev14", "WILDBERRIES\\makaryuschenko.d", "Ss12341234", 1433, "WBR_Content")
  21. )
  22.  
  23. type imageValid = func(ProductInfo, string) bool
  24. type imageURL = func(ProductInfo) []string
  25. type downloaders = map[string]interface{} // isImageValid, imageURL
  26.  
  27. func init() {
  28. configProvidersMap = parseXML()
  29. }
  30.  
  31. func connectToDB(connString string) *sqlx.DB {
  32. db, err := sqlx.Open("mssql", connString)
  33. checkerr(err)
  34. return db
  35. }
  36.  
  37. func getRv() []ProductInfo {
  38. sliceOfProducts := []ProductInfo{}
  39. db := connectToDB(serverConnString)
  40. defer db.Close()
  41. err := db.Ping()
  42. checkerr(err)
  43. err = db.Select(&sliceOfProducts, "EXEC PhotoStudio.Photojob_GetProductsForDownload")
  44. checkerr(err)
  45. return sliceOfProducts
  46. }
  47.  
  48. func getAllProducts() (products map[string][]ProductInfo) {
  49. products = map[string][]ProductInfo{}
  50. sliceOfProducts := getRv()
  51. for _, productFromDB := range sliceOfProducts {
  52. for _, v := range configProvidersMap {
  53. for _, brand := range v.BrandList {
  54. if productFromDB.BrandName == brand {
  55. products[v.Name] = append(products[v.Name], productFromDB)
  56. }
  57. }
  58. }
  59. }
  60.  
  61. return
  62. }
  63.  
  64. func downloadImage(product ProductInfo, c NecessarySettings, downloader interface{}) error {
  65. if c.ConnType == "web" {
  66. return totalParserWeb(product, downloader.(imageURL))
  67. }
  68. return downloadImageFTP(product, c, downloader.(imageValid))
  69. }
  70.  
  71. func totalParserWeb(product ProductInfo, getImageURL imageURL) error {
  72. sliceOfURL := getImageURL(product)
  73. return downloadImageWeb(sliceOfURL)
  74. }
  75.  
  76. func downloadImageFTP(product ProductInfo, c NecessarySettings, isImageValid imageValid) error {
  77. if c.Host == "ftp.vesta-baby.ru" {
  78. return downloadImageFTPWindows(product, c, isImageValid)
  79. }
  80. // if c.FileSystem == "win" {
  81. // return downloadImageFTPWindows(product, c, isImageValid)
  82. // }
  83. // return downloadImageFTPLinux(product, c, isImageValid)
  84. return nil
  85. }
  86.  
  87. func downloadImageFTPWindows(product ProductInfo, c NecessarySettings, isImageValid imageValid) error {
  88. return FTPwindows(isImageValid, c, product)
  89. }
  90.  
  91. func downloadImageFTPLinux(product ProductInfo, c NecessarySettings, isImageValid imageValid) error {
  92. return nil
  93. }
  94.  
  95. func downloadAllProviderImages(provider string, providerProducts []ProductInfo) {
  96. // for _, product := range products {
  97. // var fileExists bool
  98. // for method, settings := range mapOfSettings {
  99. // fileExists = false
  100. // if settings.Enable {
  101. // plug, err := plugin.Open("./plugins/" + method)
  102. // checkerr(err)
  103. // symHelper, err := plug.Lookup(method)
  104. // checkerr(err)
  105. // err = downloadImage(product, *settings, symHelper)
  106. // if err == nil {
  107. // fileExists = true
  108. // break
  109. // }
  110. // } else {
  111. // fmt.Println("disabled", settings.Reason)
  112. // }
  113. // }
  114. // if !fileExists {
  115. // log.Println("failed to retrieve", product)
  116. // }
  117. // }
  118. configProvidersMap[provider].mx.Lock()
  119. for _, product := range providerProducts {
  120. var fileExists bool
  121. for method, settings := range configProvidersMap[provider].MapOfSettings {
  122. fileExists = false
  123. if settings.Enable == true {
  124. plug, err := plugin.Open("./plugins/" + method)
  125. // checkerr(err)
  126. if err != nil {
  127. continue
  128. }
  129. symHelper, err := plug.Lookup(method)
  130. checkerr(err)
  131. err = downloadImage(product, *settings, symHelper) // try to retrieve the product with all methods
  132. if err == nil {
  133. fileExists = true
  134. break
  135. }
  136. } else {
  137. fmt.Println("disabled", settings.Reason)
  138. }
  139. }
  140. if !fileExists {
  141. log.Println("failed to retrieve", product)
  142. }
  143. }
  144. configProvidersMap[provider].mx.Unlock()
  145. }
  146.  
  147. func selectiveDownload() {
  148. router := mux.NewRouter()
  149. router.HandleFunc("/download/{provider}", handlerForProvider)
  150. http.ListenAndServe(":3000", router)
  151. }
  152.  
  153. func handlerForProvider(w http.ResponseWriter, r *http.Request) {
  154. vars := mux.Vars(r)
  155. provider := vars["provider"]
  156. products := getAllProducts()
  157. downloadAllProviderImages(provider, products[provider])
  158. }
  159.  
  160. func main() {
  161. go selectiveDownload()
  162.  
  163. products := getAllProducts()
  164. // fmt.Println(products)
  165. // var products = map[string][]ProductInfo{
  166. // "nife": {{Article: "horev"}},
  167. // "ginorossi": {{Article: "denis"}},
  168. // }
  169.  
  170. for {
  171. for provider, providerProducts := range products {
  172. go downloadAllProviderImages(provider, providerProducts)
  173. }
  174.  
  175. // providerInterfaces.Range(func(key, value interface{}) bool {
  176. // fmt.Println(key)
  177. // downloaderInterface := value.(downloaders)
  178. // provider := key.(string)
  179. // go downloadAllProviderImages(products[provider], provider, downloaderInterface)
  180. // providerInterfaces.Delete(key)
  181. // fmt.Println(products[provider])
  182. // return true
  183. // })
  184. time.Sleep(time.Hour * 6)
  185. }
  186. }
  187.  
  188. func downloadImageWeb(url []string) error {
  189. var out *os.File
  190. var err error
  191. var resp *http.Response
  192. for _, img := range url {
  193. if img != "" {
  194. nameOfFile := "default"
  195. sliceOfImgURL := strings.Split(img, "/")
  196. if len(sliceOfImgURL) > 2 {
  197. nameOfFile = sliceOfImgURL[2]
  198. }
  199. if out, err = os.Create(nameOfFile + ".jpg"); err != nil {
  200. return err
  201. }
  202. if resp, err = http.Get(img); err != nil {
  203. return err
  204. }
  205. if _, err := io.Copy(out, resp.Body); err != nil {
  206. return err
  207. }
  208. resp.Body.Close()
  209. out.Close()
  210. }
  211. }
  212. return nil
  213. }
  214.  
  215. func checkerr(err error) {
  216. if err != nil {
  217. panic(err)
  218. }
  219. }
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement