Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- using System;
- using System.Collections.Generic;
- using System.IO;
- using System.Linq;
- using System.Net;
- using System.Text.RegularExpressions;
- using System.Threading.Tasks;
- namespace orcatech_proxyscraper
- {
- class Program
- {
- /*
- * Simple Proxy Scraper Coded by Vinyl
- * Release 7/8/16
- */
- public static List<string> Links { get; set; } = new List<string>();
- public static List<string> proxies { get; set; } = new List<string>();
- public static string path;
- static void Main()
- {
- Console.Title = ("Simple Proxy Scraper | Vinyl ");
- restart:
- var start = Task.Factory.StartNew(() => { check(); }).ContinueWith((prevTask => request()));
- Task.WaitAll(start);
- var nodupes = proxies.Distinct().ToList();
- Console.WriteLine("Scraped {0} Unique Proxies", nodupes.Count);
- File.WriteAllLines("ScrapedProxies.txt", nodupes.ConvertAll(Convert.ToString));
- Console.WriteLine("Saved proxies to ScrapedProxies.txt" + Environment.NewLine + ("Restarting in an hour from now"));
- Task.Delay(1 * 3600000).Wait();
- goto restart;
- }
- static void check()
- {
- path = @"sites.txt";
- if (!File.Exists(path))
- {
- File.Create(path).Dispose();
- using (TextWriter tw = new StreamWriter(path))
- {
- tw.WriteLine("http://sslproxies24.blogspot.com/feeds/posts/default");
- tw.Close();
- }
- }
- }
- static void request()
- {
- var lines = File.ReadLines(@"sites.txt");
- Console.WriteLine("Scraping Proxies");
- Parallel.ForEach(lines, x =>
- {
- try
- {
- var i = 0;
- var rq = WebRequest.Create(x) as HttpWebRequest;
- var resp = (HttpWebResponse)rq.GetResponse();
- using (var streamReader = new StreamReader(resp.GetResponseStream()))
- {
- var src = streamReader.ReadToEnd();
- var regex = @"\b(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\:\d{1,6})\b";
- var matches = Regex.Matches(src, regex);
- Parallel.ForEach(matches.OfType<Match>(), m =>
- {
- i++;
- var proxy = m.Groups[0].Value;
- proxies.Add(proxy);
- });
- }
- Console.WriteLine("Scraped {0} Proxies from {1}", i, x);
- }
- catch (Exception ex)
- { Console.WriteLine("" + ex); }
- });
- }
- }
- }
Advertisement
Add Comment
Please, Sign In to add comment