Advertisement
Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- using System;
- using Newtonsoft.Json;
- using ScrapperLib.HtmlBasedScrappers;
- using System.Collections.Generic;
- using System.Globalization;
- using System.IO;
- using System.Linq;
- using System.Net;
- using ScrapingTools;
- using ScrapperLib.Shops;
- using Shoppler.ScrapingLib;
- using Shoppler.SharedLib.Scraping;
- namespace ConsoleAppForScrapingTesting
- {
- public class Program
- {
- public static void Main(string[] args)
- {
- SetCulture();
- // Uncomment line for your stuff here.
- // Put all your logic in "your" method.
- // Don't put any logic in Main method. I'll delete it. - Rafal
- // All I want to see here are commented out calls of each person function.
- // This way if I want to push some of my testing code it won't affect others. You can do anything in your function what you want.
- //RafalTesting();
- //MounTesting();
- BillTesting();
- //AsmitTesting();
- //TomekTesting();
- }
- private static void UseProxy()
- {
- LuminatiProxyManager.username = "lum-customer-shoppler-zone-champs";
- LuminatiProxyManager.password = "xby9x47g5kwr";
- ScrapingConnectionManager.ProxyEnabled = true;
- }
- public static void RafalTesting()
- {
- UseProxy();
- var url =
- "https://www.bloomingdales.com/shop/womens-designer-shoes/flats?id=16963&cm_sp=NAVIGATION_INTL-_-TOP_NAV-_-16961-SHOES-BY-STYLE-Flats";
- TestGettingLinksFromCatPage(new Bloomingdales(), url);
- }
- public static void MounTesting()
- {
- UseProxy();
- TestGettingLinksFromCatPage(new Nordstrom(),
- "https://shop.nordstrom.com/c/mens-pants?origin=topnav&cm_sp=Top%20Navigation-_-Men-_-Pants");
- //var url =
- // "http://oldnavy.gap.com/browse/product.do?pid=822818002";
- //TestScraperProductGet(new OldNavy(), url);
- //TestGettingLinksFromCatPage(new Champs(), url);
- //TestScraperProductGet(new Footlocker(), url);
- }
- public static void AsmitTesting()
- {
- UseProxy();
- //var url =
- // "https://www.macys.com/shop/mens-clothing/mens-pants?id=89&edge=hybrid";
- //TestGettingLinksFromCatPage(new Macys(), url);
- var url =
- "https://www.macys.com/shop/product/32-degrees-mens-performance-jogger-pants?ID=4752689&CategoryID=89";
- TestScraperProductGet(new Macys(), url);
- }
- public static void TomekTesting()
- {
- }
- public static void BillTesting()
- {
- UseProxy();
- TestGettingLinksFromCatPage(new Bloomingdales(),
- "https://www.bloomingdales.com/shop/mens/t-shirts-henleys?id=11536&cm_sp=NAVIGATION-_-TOP_NAV-_-3864-CLOTHING-T-Shirts-%26-Henleys");
- // Console.Write("request started");
- // var request = (HttpWebRequest)WebRequest.Create("https://www.bloomingdales.com/shop/womens-apparel/tops-tees?id=5619&viewall=true");
- //
- // request.AllowAutoRedirect = true;
- //
- // request.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8";
- // request.Headers["Accept-Language"] = "en-GB,en-US;q=0.9,en;q=0.8";
- // request.UserAgent = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.62 Safari/537.36";
- //
- // request.Headers["Cookie"] = "shippingCountry=US;";
- //
- // request.CookieContainer = new CookieContainer();
- // var response = request.GetResponse();
- // using (var reader = new StreamReader(response.GetResponseStream()))
- // {
- // using (StreamWriter writer =
- // new StreamWriter("/log_b.html", true))
- // {
- // var json = reader.ReadToEnd();
- // writer.WriteLine(json);
- // }
- // }
- // TestGettingLinksFromCatPage(new Footlocker(),
- // "https://www.footlocker.com/Mens/Running/Shoes/_-_/N-24Zf9Zrj");
- // var tt = new Footlocker().variations_curl("http://www.footlocker.com/storepickup/locations?action=getLocations&latlng=34.01,-118.49&sku=43880090&size=04.0&qty=1");
- // Console.WriteLine(tt);
- }
- public void TestScrapingCategory(BaseShopScrapper scraper, String caturl, bool printResult = true)
- {
- Console.Write("Getting Category List\n");
- var urlList = TestGettingLinksFromCatPage(scraper, caturl, false);
- Console.Write("Got List: (" + urlList.Count + ") URLs\n\n");
- Console.Write("++Starting Scrape\n");
- foreach (string url in urlList) {
- TestScraperProductGet(scraper, url, printResult);
- } Console.Write("--Finished Scrape\n");
- }
- private static List<Tuple<string, string>> TestGettingColorsInfo(BaseShopScrapper scraper, String url, bool printResult = true)
- {
- var result = scraper.GetAllColorFilters(url);
- if (printResult) PrintResult(result);
- return result;
- }
- private static List<String> TestGettingAdditionalFilters(BaseShopScrapper scraper, String url, bool printResult = true)
- {
- var result = scraper.GetProductUrlsForAdditionalFilter("", "", url);
- if (printResult) PrintResult(result);
- return result;
- }
- private static void TestIP()
- {
- var url = "http://lumtest.com/myip.json";
- var request = new ScrapingConnectionManager(ShopName.Nordstrom, "lum-customer-shoppler-zone-dc_shared_cheapest", "099vkrc0bt4s", "dc_shared_cheapest").Create(
- new ScrapingConnectionManager.ScrapingConnectionCreateOptions(
- ScrapingConnectionManager.ScrapingRequestSize.Small, url));
- var result = request.Get().AsString();
- Console.WriteLine(result);
- }
- static List<String> TestGettingLinksFromCatPage(BaseShopScrapper scraper, String url, bool printResult = true)
- {
- var result = scraper.GetProductsLinksFromCat(url);
- if (printResult) PrintResult(result);
- return result;
- }
- static void PrintResult(object result)
- {
- Console.WriteLine();
- Console.WriteLine();
- var outputString = JsonConvert.SerializeObject(result, Formatting.Indented);
- Console.WriteLine(outputString);
- Console.WriteLine();
- Console.WriteLine();
- }
- public static void SetCulture()
- {
- var cultureInfo = new CultureInfo("en-US");
- CultureInfo.DefaultThreadCurrentCulture = cultureInfo;
- }
- static GetProductFromUrlResult TestScraperProductGet(BaseShopScrapper scraper, String url, bool printResult = true)
- {
- var prod = scraper.GetProductFromUrl(url);
- if (printResult) PrintResult(prod);
- return prod;
- }
- }
- }
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement