Advertisement
crown111

Untitled

Apr 13th, 2017
170
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 4.77 KB | None | 0 0
  1. using System;
  2. using Newtonsoft.Json;
  3. using ScrapperLib.HtmlBasedScrappers;
  4. using System.Collections.Generic;
  5. using System.Globalization;
  6. using System.Linq;
  7. using ScrapingTools;
  8. using ScrapperLib.Shops;
  9. using Shoppler.ScrapingLib;
  10.  
  11. namespace ConsoleAppForScrapingTesting
  12. {
  13. class Program
  14. {
  15. static void Main(string[] args)
  16. {
  17. TestIP();
  18. SetCulture();
  19. if (args?.Any() ?? false)
  20. {
  21. if (args.Contains("enableProxy")) ScrapingConnectionManager.ProxyEnabled = true;
  22. }
  23.  
  24. LuminatiProxyManager.username = "lum-customer-shoppler-zone-dc_shared_cheapest";
  25. LuminatiProxyManager.password = "099vkrc0bt4s";
  26. ScrapingConnectionManager.ProxyEnabled = true;
  27.  
  28. //TestScraperProductGet(new Bloomingdales(),
  29. // "https://www.bloomingdales.com/shop/product/honey-punch-lace-up-side-tee-100-exclusive?ID=2459925&CategoryID=5619#fn=spp%3D138");
  30. //Console.WriteLine("Click enter to finish");
  31. //Console.ReadLine();
  32.  
  33. //TestScraperProductGet(new Vans(),
  34. // "http://www.vans.com/shop/senior-year-muscle-tee-phantom");
  35. //Console.WriteLine("Click enter to finish");
  36. //Console.ReadLine();
  37.  
  38. TestGettingLinksFromCatPage(new Bloomingdales(), "http://www1.bloomingdales.com/shop/mens/polos?id=1000663&cm_sp=NAVIGATION_INTL-_-TOP_NAV-_-3864-CLOTHING-Polos");
  39.  
  40. //TestGettingColorsInfo(new Nordstrom(), "http://shop.nordstrom.com/c/womens-activewear-shop?origin=leftnav");
  41.  
  42. //TestScraperProductGet(new OldNavy(), "http://shop.nordstrom.com/s/loveappella-v-neck-jersey-maxi-dress-regular-petite/3965546");
  43.  
  44. Console.WriteLine("Click enter to finish");
  45. Console.ReadLine();
  46.  
  47. //TestGettingLinksFromCatPage(new Guess(),
  48. // "https://shop.guess.com/en/catalog/browse/women/dresses/view-all/?f_color_group=Black");
  49. //Console.WriteLine("Click enter to finish");
  50. //Console.ReadLine();
  51.  
  52. }
  53.  
  54. public void TestScrapingCategory(BaseShopScrapper scraper, String caturl, bool printResult = true)
  55. {
  56. Console.Write("Getting Category List\n");
  57. var urlList = TestGettingLinksFromCatPage(scraper, caturl, false);
  58. Console.Write("Got List: (" + urlList.Count + ") URLs\n\n");
  59. Console.Write("++Starting Scrape\n");
  60. foreach (string url in urlList) {
  61. TestScraperProductGet(scraper, url, printResult);
  62. } Console.Write("--Finished Scrape\n");
  63. }
  64.  
  65. private static List<Tuple<string, string>> TestGettingColorsInfo(BaseShopScrapper scraper, String url, bool printResult = true)
  66. {
  67. var result = scraper.GetAllColorFilters(url);
  68. if (printResult) PrintResult(result);
  69.  
  70. return result;
  71. }
  72.  
  73.  
  74. private static List<String> TestGettingAdditionalFilters(BaseShopScrapper scraper, String url, bool printResult = true)
  75. {
  76. var result = scraper.GetProductUrlsForAdditionalFilter("", "", url);
  77. if (printResult) PrintResult(result);
  78.  
  79. return result;
  80. }
  81.  
  82.  
  83. private static void TestIP()
  84. {
  85. var url = "http://lumtest.com/myip.json";
  86. var request = new ScrapingConnectionManager(ShopName.Nordstrom).Create(
  87. new ScrapingConnectionManager.ScrapingConnectionCreateOptions(
  88. ScrapingConnectionManager.ScrapingRequestSize.Small, url));
  89. var result = request.Get().AsString();
  90. Console.WriteLine(result);
  91. }
  92.  
  93. static List<String> TestGettingLinksFromCatPage(BaseShopScrapper scraper, String url, bool printResult = true)
  94. {
  95. var result = scraper.GetProductsLinksFromCat(url);
  96. if (printResult) PrintResult(result);
  97.  
  98. return result;
  99. }
  100.  
  101. static void PrintResult(object result)
  102. {
  103. Console.WriteLine();
  104. Console.WriteLine();
  105.  
  106. var outputString = JsonConvert.SerializeObject(result, Formatting.Indented);
  107. Console.WriteLine(outputString);
  108.  
  109. Console.WriteLine();
  110. Console.WriteLine();
  111. }
  112.  
  113. public static void SetCulture()
  114. {
  115. var cultureInfo = new CultureInfo("en-US");
  116. CultureInfo.DefaultThreadCurrentCulture = cultureInfo;
  117. }
  118.  
  119. static object TestScraperProductGet(BaseShopScrapper scraper, String url, bool printResult = true)
  120. {
  121.  
  122. var prod = scraper.GetProductFromUrl(url);
  123. if (printResult) PrintResult(prod);
  124. return prod;
  125. }
  126. }
  127. }
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement