Advertisement
Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- using System;
- using Newtonsoft.Json;
- using ScrapperLib.HtmlBasedScrappers;
- using System.Collections.Generic;
- using System.Globalization;
- using System.Linq;
- using ScrapingTools;
- using ScrapperLib.Shops;
- using Shoppler.ScrapingLib;
- namespace ConsoleAppForScrapingTesting
- {
- class Program
- {
- static void Main(string[] args)
- {
- TestIP();
- SetCulture();
- if (args?.Any() ?? false)
- {
- if (args.Contains("enableProxy")) ScrapingConnectionManager.ProxyEnabled = true;
- }
- LuminatiProxyManager.username = "lum-customer-shoppler-zone-dc_shared_cheapest";
- LuminatiProxyManager.password = "099vkrc0bt4s";
- ScrapingConnectionManager.ProxyEnabled = true;
- //TestScraperProductGet(new Bloomingdales(),
- // "https://www.bloomingdales.com/shop/product/honey-punch-lace-up-side-tee-100-exclusive?ID=2459925&CategoryID=5619#fn=spp%3D138");
- //Console.WriteLine("Click enter to finish");
- //Console.ReadLine();
- //TestScraperProductGet(new Vans(),
- // "http://www.vans.com/shop/senior-year-muscle-tee-phantom");
- //Console.WriteLine("Click enter to finish");
- //Console.ReadLine();
- TestGettingLinksFromCatPage(new Bloomingdales(), "http://www1.bloomingdales.com/shop/mens/polos?id=1000663&cm_sp=NAVIGATION_INTL-_-TOP_NAV-_-3864-CLOTHING-Polos");
- //TestGettingColorsInfo(new Nordstrom(), "http://shop.nordstrom.com/c/womens-activewear-shop?origin=leftnav");
- //TestScraperProductGet(new OldNavy(), "http://shop.nordstrom.com/s/loveappella-v-neck-jersey-maxi-dress-regular-petite/3965546");
- Console.WriteLine("Click enter to finish");
- Console.ReadLine();
- //TestGettingLinksFromCatPage(new Guess(),
- // "https://shop.guess.com/en/catalog/browse/women/dresses/view-all/?f_color_group=Black");
- //Console.WriteLine("Click enter to finish");
- //Console.ReadLine();
- }
- public void TestScrapingCategory(BaseShopScrapper scraper, String caturl, bool printResult = true)
- {
- Console.Write("Getting Category List\n");
- var urlList = TestGettingLinksFromCatPage(scraper, caturl, false);
- Console.Write("Got List: (" + urlList.Count + ") URLs\n\n");
- Console.Write("++Starting Scrape\n");
- foreach (string url in urlList) {
- TestScraperProductGet(scraper, url, printResult);
- } Console.Write("--Finished Scrape\n");
- }
- private static List<Tuple<string, string>> TestGettingColorsInfo(BaseShopScrapper scraper, String url, bool printResult = true)
- {
- var result = scraper.GetAllColorFilters(url);
- if (printResult) PrintResult(result);
- return result;
- }
- private static List<String> TestGettingAdditionalFilters(BaseShopScrapper scraper, String url, bool printResult = true)
- {
- var result = scraper.GetProductUrlsForAdditionalFilter("", "", url);
- if (printResult) PrintResult(result);
- return result;
- }
- private static void TestIP()
- {
- var url = "http://lumtest.com/myip.json";
- var request = new ScrapingConnectionManager(ShopName.Nordstrom).Create(
- new ScrapingConnectionManager.ScrapingConnectionCreateOptions(
- ScrapingConnectionManager.ScrapingRequestSize.Small, url));
- var result = request.Get().AsString();
- Console.WriteLine(result);
- }
- static List<String> TestGettingLinksFromCatPage(BaseShopScrapper scraper, String url, bool printResult = true)
- {
- var result = scraper.GetProductsLinksFromCat(url);
- if (printResult) PrintResult(result);
- return result;
- }
- static void PrintResult(object result)
- {
- Console.WriteLine();
- Console.WriteLine();
- var outputString = JsonConvert.SerializeObject(result, Formatting.Indented);
- Console.WriteLine(outputString);
- Console.WriteLine();
- Console.WriteLine();
- }
- public static void SetCulture()
- {
- var cultureInfo = new CultureInfo("en-US");
- CultureInfo.DefaultThreadCurrentCulture = cultureInfo;
- }
- static object TestScraperProductGet(BaseShopScrapper scraper, String url, bool printResult = true)
- {
- var prod = scraper.GetProductFromUrl(url);
- if (printResult) PrintResult(prod);
- return prod;
- }
- }
- }
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement