HTTPCrawler

using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading.Tasks;

namespace HTTPCrawler
{
    class Program
    {
        static HashSet<string> pages = new HashSet<string>();
        static StringBuilder sb = new StringBuilder();
        static int maxDepth = 1;

        static void Main(string[] args)
        {
            //test();
            //return;

            //crawlPage("http://detox.wi.ps.pl");
            crawlPage("http://www.bisnode.pl/informacje/baza-maili/");
            File.WriteAllText(@"E:\out.xml", sb.ToString());
            Console.WriteLine("done");
            Console.ReadLine();
        }

        static void appendAux(string s, int depth, bool EOL = true)
        {
            for (int i = 0; i < depth; i++)
                sb.Append("\t");
            sb.Append(s);
            if (EOL)
                sb.Append(Environment.NewLine);
        }

        static void crawlPage(string url, int depth = 0)
        {
            if (depth >= maxDepth)
                return;

            if (pages.Contains(url)) return;
            pages.Add(url);

            if (depth == 0)
            {
                appendAux("<?xml version=\"1.0\" encoding=\"utf‐8\" ?>", depth);
                appendAux($"<SITE url=\"{ url}\" + maxDepth={maxDepth}>", depth);
            }

            string htmlCode = "";
            using (WebClient client = new WebClient())
                try
                {
                    htmlCode = client.DownloadString(url);
                }
                catch
                {
                    return;
                }

            string[] lines = htmlCode.Split('\n');

            foreach (string line in lines) // w srodku tej petli wolamy appendAux z depth +1, żeby zaczac od wciecia juz na poczatku
            {
                Match result;

                result = Regex.Match(line, "<img (?<nie_obrazek>.*?)src=\"(?<obrazek>.+?)\"");
                if (result.Groups["obrazek"].ToString() != "")
                    appendAux("<IMAGE>" + result.Groups["obrazek"] + "</IMAGE>", depth + 1);

                result = Regex.Match(line, "href='mailto:(?<mail>.+?)'");
                if (result.Groups["mail"].ToString() != "")
                    appendAux("<EMAIL>"+ result.Groups["mail"]+ "<EMAIL>", depth + 1);

                result = Regex.Match(line, "<a (?<nie_link>.*?)href=\"(?<link>.+?)\"");
                if (result.Groups["link"].ToString() != "")
                {
                    System.Threading.Thread.Sleep(100);

                    string link = result.Groups["link"].ToString();
                    appendAux("<FILE " + link + " currentDepth=" + depth + " >", depth + 1, false);
                    if (link.StartsWith("http://") || link.StartsWith("https://"))
                        crawlPage(link, depth + 1);
                    else
                        crawlPage(url + link, depth + 1);
                    appendAux("</FILE>", 0); //0 zeby nie dodal tabulatora
                }

            }

            if (depth == 0)
                appendAux($"</SITE>", depth);
        }

        static void test()
        {
            string pattern = "<img (?<nie_obrazek>.+?) src=\"(?<obrazek>.+?)\"";
            string input = "<big><big><big><big><span style=\"font - weight: bold; color: red; text - decoration: blink; \"><blink></blink></span></big></big></big></big><span style=\"font - weight: bold; color: red; \"></span><img style=\"border: 0px solid; width: 640px; height: 194px; \" alt=\"\" src=\"https://leftyconcarne.files.wordpress.com/2011/12/dilbert-cartoon.gif\" usemap=\"#map\"><br>\"";

            Match result = Regex.Match(input, pattern);
            Console.WriteLine("Image name: " + result.Groups["obrazek"]);
        }
    }
}