SHOW:
|
|
- or go back to the newest paste.
| 1 | using System; | |
| 2 | using System.Collections.Generic; | |
| 3 | using System.IO; | |
| 4 | using System.Linq; | |
| 5 | using System.Net; | |
| 6 | using System.Text.RegularExpressions; | |
| 7 | using System.Threading.Tasks; | |
| 8 | ||
| 9 | namespace orcatech_proxyscraper | |
| 10 | {
| |
| 11 | class Program | |
| 12 | {
| |
| 13 | /* | |
| 14 | * Simple Proxy Scraper Coded by Vinyl | |
| 15 | * Release 7/8/16 | |
| 16 | */ | |
| 17 | public static List<string> Links { get; set; } = new List<string>();
| |
| 18 | public static List<string> proxies { get; set; } = new List<string>();
| |
| 19 | public static string path; | |
| 20 | ||
| 21 | static void Main() | |
| 22 | {
| |
| 23 | Console.Title = ("Simple Proxy Scraper | Vinyl ");
| |
| 24 | restart: | |
| 25 | var start = Task.Factory.StartNew(() => { check(); }).ContinueWith((prevTask => request()));
| |
| 26 | Task.WaitAll(start); | |
| 27 | ||
| 28 | var nodupes = proxies.Distinct().ToList(); | |
| 29 | Console.WriteLine("Scraped {0} Unique Proxies", nodupes.Count);
| |
| 30 | File.WriteAllLines("ScrapedProxies.txt", nodupes.ConvertAll(Convert.ToString));
| |
| 31 | Console.WriteLine("Saved proxies to ScrapedProxies.txt" + Environment.NewLine + ("Restarting in an hour from now"));
| |
| 32 | Task.Delay(1 * 3600000).Wait(); | |
| 33 | goto restart; | |
| 34 | } | |
| 35 | static void check() | |
| 36 | {
| |
| 37 | path = @"sites.txt"; | |
| 38 | if (!File.Exists(path)) | |
| 39 | {
| |
| 40 | File.Create(path).Dispose(); | |
| 41 | using (TextWriter tw = new StreamWriter(path)) | |
| 42 | {
| |
| 43 | tw.WriteLine("http://sslproxies24.blogspot.com/feeds/posts/default");
| |
| 44 | tw.Close(); | |
| 45 | } | |
| 46 | } | |
| 47 | ||
| 48 | } | |
| 49 | static void request() | |
| 50 | {
| |
| 51 | var lines = File.ReadLines(@"sites.txt"); | |
| 52 | Console.WriteLine("Scraping Proxies");
| |
| 53 | Parallel.ForEach(lines, x => | |
| 54 | {
| |
| 55 | try | |
| 56 | {
| |
| 57 | var i = 0; | |
| 58 | var rq = WebRequest.Create(x) as HttpWebRequest; | |
| 59 | var resp = (HttpWebResponse)rq.GetResponse(); | |
| 60 | using (var streamReader = new StreamReader(resp.GetResponseStream())) | |
| 61 | {
| |
| 62 | var src = streamReader.ReadToEnd(); | |
| 63 | var regex = @"\b(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\:\d{1,6})\b";
| |
| 64 | var matches = Regex.Matches(src, regex); | |
| 65 | Parallel.ForEach(matches.OfType<Match>(), m => | |
| 66 | {
| |
| 67 | i++; | |
| 68 | var proxy = m.Groups[0].Value; | |
| 69 | proxies.Add(proxy); | |
| 70 | }); | |
| 71 | } | |
| 72 | Console.WriteLine("Scraped {0} Proxies from {1}", i, x);
| |
| 73 | } | |
| 74 | catch (Exception ex) | |
| 75 | { Console.WriteLine("" + ex); }
| |
| 76 | }); | |
| 77 | } | |
| 78 | } | |
| 79 | } |