View difference between Paste ID: vZHY9uTa and eYpjUJRR
SHOW: | | - or go back to the newest paste.
1
using System;
2
using System.Collections.Generic;
3
using System.IO;
4
using System.Linq;
5
using System.Net;
6
using System.Text.RegularExpressions;
7
using System.Threading.Tasks;
8
9
namespace orcatech_proxyscraper
10
{
11
    class Program
12
    {
13
        /*
14
         * Simple Proxy Scraper Coded by Vinyl
15
         * Release 7/8/16
16
         */
17
        public static List<string> Links { get; set; } = new List<string>();
18
        public static List<string> proxies { get; set; } = new List<string>();
19
        public static string path;
20
21
        static void Main()
22
        {
23
            Console.Title = ("Simple Proxy Scraper | Vinyl ");
24
            restart:
25
            var start = Task.Factory.StartNew(() => { check(); }).ContinueWith((prevTask => request()));
26
            Task.WaitAll(start);
27
28
            var nodupes = proxies.Distinct().ToList();
29
            Console.WriteLine("Scraped {0} Unique Proxies", nodupes.Count);
30
            File.WriteAllLines("ScrapedProxies.txt", nodupes.ConvertAll(Convert.ToString));
31
            Console.WriteLine("Saved proxies to ScrapedProxies.txt" + Environment.NewLine + ("Restarting in an hour from now"));
32
            Task.Delay(1 * 3600000).Wait();
33
            goto restart;
34
        }
35
        static void check()
36
        {
37
            path = @"sites.txt";
38
            if (!File.Exists(path))
39
            {
40
                File.Create(path).Dispose();
41
                using (TextWriter tw = new StreamWriter(path))
42
                {
43
                    tw.WriteLine("http://sslproxies24.blogspot.com/feeds/posts/default");
44
                    tw.Close();
45
                }
46
            }
47
48
        }
49
        static void request()
50
        {
51
            var lines = File.ReadLines(@"sites.txt");
52
            Console.WriteLine("Scraping Proxies");
53
            Parallel.ForEach(lines, x =>
54
            {
55
                try
56
                {
57
                    var i = 0;
58
                    var rq = WebRequest.Create(x) as HttpWebRequest;
59
                    var resp = (HttpWebResponse)rq.GetResponse();
60
                    using (var streamReader = new StreamReader(resp.GetResponseStream()))
61
                    {
62
                        var src = streamReader.ReadToEnd();
63
                        var regex = @"\b(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\:\d{1,6})\b";
64
                        var matches = Regex.Matches(src, regex);
65
                        Parallel.ForEach(matches.OfType<Match>(), m =>
66
                        {
67
                            i++;
68
                            var proxy = m.Groups[0].Value;
69
                            proxies.Add(proxy);
70
                        });
71
                    }
72
                    Console.WriteLine("Scraped {0} Proxies from {1}", i, x);
73
                }
74
                catch (Exception ex)
75
                { Console.WriteLine("" + ex); }
76
            });
77
        }
78
    }
79
}