WebCrawl

using System.Threading.Tasks;

class Solution {
    public IList<string> Crawl(string startUrl, HtmlParser htmlParser) {
        var uri = new Uri(startUrl);
        var visited = new HashSet<string>();
        var queue = new List<string>();
        queue.Add(startUrl);

        while (queue.Count > 0)
        {
            var nextWave = new List<string>();
            Parallel.ForEach(queue, (next) =>
            {
                if (visited.Contains(next) || !next.StartsWith($"http://{uri.Host}")) return;

                visited.Add(next);
                nextWave.AddRange(htmlParser.GetUrls(next));
            });
            queue = nextWave;
        }

        return visited.ToArray();
    }
}