Advertisement
Guest User

Untitled

a guest
Jan 12th, 2018
274
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 7.34 KB | None | 0 0
  1. using System;
  2. using Newtonsoft.Json;
  3. using ScrapperLib.HtmlBasedScrappers;
  4. using System.Collections.Generic;
  5. using System.Globalization;
  6. using System.IO;
  7. using System.Linq;
  8. using System.Net;
  9. using ScrapingTools;
  10. using ScrapperLib.Shops;
  11. using Shoppler.ScrapingLib;
  12. using Shoppler.SharedLib.Scraping;
  13.  
  14. namespace ConsoleAppForScrapingTesting
  15. {
  16. public class Program
  17. {
  18. public static void Main(string[] args)
  19. {
  20. SetCulture();
  21.  
  22. // Uncomment line for your stuff here.
  23. // Put all your logic in "your" method.
  24. // Don't put any logic in Main method. I'll delete it. - Rafal
  25. // All I want to see here are commented out calls of each person function.
  26. // This way if I want to push some of my testing code it won't affect others. You can do anything in your function what you want.
  27.  
  28.  
  29. //RafalTesting();
  30. //MounTesting();
  31. BillTesting();
  32. //AsmitTesting();
  33. //TomekTesting();
  34.  
  35. }
  36.  
  37. private static void UseProxy()
  38. {
  39. LuminatiProxyManager.username = "lum-customer-shoppler-zone-champs";
  40. LuminatiProxyManager.password = "xby9x47g5kwr";
  41.  
  42. ScrapingConnectionManager.ProxyEnabled = true;
  43. }
  44.  
  45. public static void RafalTesting()
  46. {
  47. UseProxy();
  48. var url =
  49. "https://www.bloomingdales.com/shop/womens-designer-shoes/flats?id=16963&cm_sp=NAVIGATION_INTL-_-TOP_NAV-_-16961-SHOES-BY-STYLE-Flats";
  50. TestGettingLinksFromCatPage(new Bloomingdales(), url);
  51.  
  52. }
  53.  
  54. public static void MounTesting()
  55. {
  56.  
  57. UseProxy();
  58. TestGettingLinksFromCatPage(new Nordstrom(),
  59. "https://shop.nordstrom.com/c/mens-pants?origin=topnav&cm_sp=Top%20Navigation-_-Men-_-Pants");
  60. //var url =
  61. // "http://oldnavy.gap.com/browse/product.do?pid=822818002";
  62. //TestScraperProductGet(new OldNavy(), url);
  63. //TestGettingLinksFromCatPage(new Champs(), url);
  64. //TestScraperProductGet(new Footlocker(), url);
  65.  
  66. }
  67.  
  68. public static void AsmitTesting()
  69. {
  70. UseProxy();
  71.  
  72. //var url =
  73. // "https://www.macys.com/shop/mens-clothing/mens-pants?id=89&edge=hybrid";
  74. //TestGettingLinksFromCatPage(new Macys(), url);
  75.  
  76.  
  77.  
  78. var url =
  79. "https://www.macys.com/shop/product/32-degrees-mens-performance-jogger-pants?ID=4752689&CategoryID=89";
  80. TestScraperProductGet(new Macys(), url);
  81.  
  82. }
  83.  
  84. public static void TomekTesting()
  85. {
  86.  
  87. }
  88.  
  89. public static void BillTesting()
  90. {
  91. UseProxy();
  92.  
  93. TestGettingLinksFromCatPage(new Bloomingdales(),
  94. "https://www.bloomingdales.com/shop/mens/t-shirts-henleys?id=11536&cm_sp=NAVIGATION-_-TOP_NAV-_-3864-CLOTHING-T-Shirts-%26-Henleys");
  95.  
  96.  
  97. // Console.Write("request started");
  98. // var request = (HttpWebRequest)WebRequest.Create("https://www.bloomingdales.com/shop/womens-apparel/tops-tees?id=5619&viewall=true");
  99. //
  100. // request.AllowAutoRedirect = true;
  101. //
  102. // request.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8";
  103. // request.Headers["Accept-Language"] = "en-GB,en-US;q=0.9,en;q=0.8";
  104. // request.UserAgent = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.62 Safari/537.36";
  105. //
  106. // request.Headers["Cookie"] = "shippingCountry=US;";
  107. //
  108. // request.CookieContainer = new CookieContainer();
  109. // var response = request.GetResponse();
  110. // using (var reader = new StreamReader(response.GetResponseStream()))
  111. // {
  112. // using (StreamWriter writer =
  113. // new StreamWriter("/log_b.html", true))
  114. // {
  115. // var json = reader.ReadToEnd();
  116. // writer.WriteLine(json);
  117. // }
  118. // }
  119.  
  120. // TestGettingLinksFromCatPage(new Footlocker(),
  121. // "https://www.footlocker.com/Mens/Running/Shoes/_-_/N-24Zf9Zrj");
  122. // var tt = new Footlocker().variations_curl("http://www.footlocker.com/storepickup/locations?action=getLocations&latlng=34.01,-118.49&sku=43880090&size=04.0&qty=1");
  123. // Console.WriteLine(tt);
  124.  
  125. }
  126.  
  127. public void TestScrapingCategory(BaseShopScrapper scraper, String caturl, bool printResult = true)
  128. {
  129. Console.Write("Getting Category List\n");
  130. var urlList = TestGettingLinksFromCatPage(scraper, caturl, false);
  131. Console.Write("Got List: (" + urlList.Count + ") URLs\n\n");
  132. Console.Write("++Starting Scrape\n");
  133. foreach (string url in urlList) {
  134. TestScraperProductGet(scraper, url, printResult);
  135. } Console.Write("--Finished Scrape\n");
  136. }
  137.  
  138. private static List<Tuple<string, string>> TestGettingColorsInfo(BaseShopScrapper scraper, String url, bool printResult = true)
  139. {
  140. var result = scraper.GetAllColorFilters(url);
  141. if (printResult) PrintResult(result);
  142.  
  143. return result;
  144. }
  145.  
  146.  
  147. private static List<String> TestGettingAdditionalFilters(BaseShopScrapper scraper, String url, bool printResult = true)
  148. {
  149. var result = scraper.GetProductUrlsForAdditionalFilter("", "", url);
  150. if (printResult) PrintResult(result);
  151.  
  152. return result;
  153. }
  154.  
  155.  
  156. private static void TestIP()
  157. {
  158. var url = "http://lumtest.com/myip.json";
  159. var request = new ScrapingConnectionManager(ShopName.Nordstrom, "lum-customer-shoppler-zone-dc_shared_cheapest", "099vkrc0bt4s", "dc_shared_cheapest").Create(
  160. new ScrapingConnectionManager.ScrapingConnectionCreateOptions(
  161. ScrapingConnectionManager.ScrapingRequestSize.Small, url));
  162. var result = request.Get().AsString();
  163. Console.WriteLine(result);
  164. }
  165.  
  166. static List<String> TestGettingLinksFromCatPage(BaseShopScrapper scraper, String url, bool printResult = true)
  167. {
  168. var result = scraper.GetProductsLinksFromCat(url);
  169. if (printResult) PrintResult(result);
  170.  
  171. return result;
  172. }
  173.  
  174. static void PrintResult(object result)
  175. {
  176. Console.WriteLine();
  177. Console.WriteLine();
  178.  
  179. var outputString = JsonConvert.SerializeObject(result, Formatting.Indented);
  180. Console.WriteLine(outputString);
  181.  
  182. Console.WriteLine();
  183. Console.WriteLine();
  184. }
  185.  
  186. public static void SetCulture()
  187. {
  188. var cultureInfo = new CultureInfo("en-US");
  189. CultureInfo.DefaultThreadCurrentCulture = cultureInfo;
  190. }
  191.  
  192. static GetProductFromUrlResult TestScraperProductGet(BaseShopScrapper scraper, String url, bool printResult = true)
  193. {
  194.  
  195. var prod = scraper.GetProductFromUrl(url);
  196. if (printResult) PrintResult(prod);
  197. return prod;
  198. }
  199. }
  200. }
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement