using System;
using System.IO;
using System.Net;
using System.Text;
using System.Linq;
using System.Drawing;
using System.Threading;
using System.Diagnostics;
using System.Drawing.Imaging;
using System.Threading.Tasks;
using System.Collections.Generic;
using HtmlAgilityPack;
using OpenQA.Selenium.Chrome;
using OpenQA.Selenium;
using Newtonsoft.Json.Linq;
using System.Text.RegularExpressions;
using AngleSharp;
using AngleSharp.Parser.Html;

namespace AMRequests
{
    public enum TestTypes
    {
        Favicon, Tagmanager, Lifeline, Opengraph, DesktopCheck, Gzip, Sitemap, Robots, Custom404, Ssl,
        PageTitles, LoremIpsum, ContentSize, LinkChecker, AltTags, NetworkRequests, ConsoleCheck, TakeScreenshot,
        Login
    }

    public enum RequestFlags
    {
        HEAD, GZIP, STATUSCAKE, NONE
    }

    public class ResponseData
    {
        public string StatusCode { get; set; }
        public string Content { get; set; }
        public string ContentEncoding { get; set; }

        public ResponseData(HttpWebResponse response)
        {
            StatusCode = response.GetStatusCode();
            Content = response.GetContent();
            ContentEncoding = response.ContentEncoding;
            response.Close();
        }
    }

    public class Request
    {
        string _dictKey;
        public string Key { get { return _dictKey; }}
        public string Url { get; }
        public string Content { get; }
        public string StatusCode { get; }
        public string AgentID { get; }
        public string Encoding { get; }
        public RequestFlags[] Flags { get; }

        public Request(string url, HttpWebResponse response, string agentid = "win10chrome", RequestFlags[] flags = null)
        {
            Url = url;
            Content = response.GetContent();
            StatusCode = response.GetStatusCode();
            AgentID = agentid;
            Encoding = response.ContentEncoding;
            Flags = flags ?? new RequestFlags[]{RequestFlags.NONE};
            _dictKey = GenerateKey(Url, AgentID, Flags);
            response.Close();
        }

        public Request(string url, ResponseData responseData, string agentid = "win10chrome", RequestFlags[] flags = null)
        {
            Url = url;
            Content = responseData.Content;
            StatusCode = responseData.StatusCode;
            Encoding = responseData.ContentEncoding;
            AgentID = agentid;
            Flags = flags ?? new RequestFlags[] { RequestFlags.NONE };
            _dictKey = GenerateKey(Url, AgentID, Flags);
        }

        public static string GenerateKey(string url, string agentid = "win10chrome", RequestFlags[] flags = null)
        {
            flags = flags ?? new RequestFlags[] { RequestFlags.NONE };
            string key = "";

            key = "";
            key += url.GetType() + "||";
            key += url + "||";
            key += agentid.GetType() + "||";
            key += agentid + "||";

            foreach (RequestFlags flag in flags)
            {
                key += flag.GetType() + "||";
                key += flag + "||";
            }

            key = key.TrimEnd('|');
            return key;
        }

        public Object[] DecodeKey(string key)
        {
            string[] keyStrings = key.Split("||".ToCharArray());
            object[] keyData = new object[keyStrings.Length / 2];
            int index = 0;

            for (int i = 0; i < keyStrings.Length; i += 2)
            {
                if (keyStrings[i] == "System.String")
                {
                    keyData[index] = keyStrings[i + 1];
                }
                else if (keyStrings[i] == "Requests_Lib.RequestFlags")
                {
                    keyData[index] = (RequestFlags)Enum.Parse(typeof(RequestFlags), keyStrings[i + 1]);
                }

                index++;
            }

            return keyData;
        }
    }

    public class RequestFactory
    {
        string _domainUrl;
        string[] _domainAuth;
        bool _authRequired;
        bool _cookieRequired;
        Queue<string> _linksToCheck;
        HashSet<string> _uniqueLinksSet;

        public Dictionary<string, UserAgent> Agents { get; set; }
        public Dictionary<string, Request> RequestsList { get; set; }

        public RequestFactory(string domainUrl)
        {
            Agents = new Dictionary<string, UserAgent>();
            RequestsList = new Dictionary<string, Request>();

            _domainUrl = domainUrl;
            _domainAuth = GetAuth(_domainUrl);
            _authRequired = _domainAuth[0] != "";
            _cookieRequired = _domainAuth[2] != "";
            _linksToCheck = new Queue<string>();
            _uniqueLinksSet = new HashSet<string>();
        }

        public Request GetRequest(string url, string agentID = "win10chrome", RequestFlags[] flags = null)
        {
            string key = Request.GenerateKey(url, agentID, flags);

            if (RequestsList.Keys.Contains(key))
            {
                return RequestsList[key];
            }

            int remainingAttempts = 3;
            ResponseData responseData = null;

            while (remainingAttempts > 0)
            {
                remainingAttempts--;
                responseData = GetUrlResponse(url, agentID, flags);

                if (responseData.StatusCode != "200")
                    continue;
            }

            Request request = new Request(url, responseData, agentID, flags);
            RequestsList.Add(key, request);
            return request;
        }

        HttpWebRequest GetUrlRequest(string url, string agentID = "win10chrome")
        {
            try
            {
                HttpWebRequest request = (HttpWebRequest)WebRequest.Create(url);
                request.UserAgent = AgentHelpers.AgentIDToAgentString(agentID);

                if (_cookieRequired && url.Contains(_domainUrl))
                {
                    Debug.WriteLine("********** Domain Cookie Required **********");
                    request.Headers[HttpRequestHeader.Cookie] = _domainAuth[2] + "=" + _domainAuth[3];
                }
                else
                {
                    if (_authRequired && url.Contains(_domainUrl))
                    {
                        Debug.WriteLine("********** Domain Auth Required **********");
                        string creds = "Basic " + Convert.ToBase64String(Encoding.UTF8.GetBytes(_domainAuth[0] + ":" + _domainAuth[1]));
                        request.Headers[HttpRequestHeader.Authorization] = creds;
                    }
                }
                return request;
            }
            catch(UriFormatException)
            {
                Debug.WriteLine($"XXXXXXXXXX {url} XXXXXXXXXX");
                return null;
            }
        }

        public ResponseData GetUrlResponse(string url, string agentID = "win10chrome", RequestFlags[] flags = null)
        {
            flags = flags ?? new RequestFlags[] { RequestFlags.NONE };
            HttpWebRequest req = null;

            try
            {
                req = GetUrlRequest(url, agentID);

                if (req == null)
                {
                    Debug.WriteLine($"XXXXXXXXXX {url} XXXXXXXXXX");
                    return null;
                }

                req.Proxy = null;

                foreach (RequestFlags flag in flags)
                {
                    switch (flag)
                    {
                        case RequestFlags.HEAD:
                            req.Method = "HEAD";
                            Debug.WriteLine("********** HEAD Request Only **********");
                            break;
                        case RequestFlags.GZIP:
                            req.Headers.Add(HttpRequestHeader.AcceptEncoding, "gzip, deflate");
                            Debug.WriteLine("********** Decompression Method GZip **********");
                            break;
                        case RequestFlags.STATUSCAKE:
                            req.Headers.Add("API", Config.STATUSCAKE_API_KEY);
                            req.Headers.Add("Username", Config.STATUSCAKE_USERNAME);
                            break;
                    }
                }

                HttpWebResponse response = (HttpWebResponse)req.GetResponse();
                return new ResponseData(response);
            }
            catch (WebException ex)
            {
                HttpWebResponse response = (HttpWebResponse)ex.Response;
                return new ResponseData(response);
            }
        }

        List<string> GetImagesFromUrl(string url)
        {
            //var timer = Stopwatch.StartNew();
            List<string> imageList = new List<string>();

            if (string.IsNullOrEmpty(url))
            {
                //Debug.WriteLine($"GetImagesFromUrl() execution time: {timer.Elapsed.Milliseconds.ToString()}");
                return imageList;
            }

            Request request = GetRequest(url);

            if (request == null)
            {
                //Debug.WriteLine($"GetImagesFromUrl() execution time: {timer.Elapsed.Milliseconds.ToString()}");
                return imageList;
            }

            string content = request.Content;
            var doc = new HtmlDocument();
            doc.LoadHtml(content);

            var images = doc.DocumentNode.Descendants("img")
                            .Select(e => e.GetAttributeValue("src", null))
                            .Where(s => !String.IsNullOrEmpty(s));

            foreach (var img in images)
            {
                string src = img.FormatAsImageUrl(url);

                if (String.IsNullOrEmpty(src))
                    continue;

                imageList.Add(src);
            }

            //Debug.WriteLine($"GetImagesFromUrl() execution time: {timer.Elapsed.Milliseconds.ToString()}");
            return imageList;
        }

        List<string> GetLinksFromUrl(string url)
        {
            var timer = Stopwatch.StartNew();
            List<string> linkList = new List<string>();

            if (string.IsNullOrEmpty(url))
            {
                Debug.WriteLine($"GetLinksFromUrl() execution time: {timer.Elapsed.Milliseconds.ToString()}, for 0 links.");
                return linkList;
            }

            if (Path.HasExtension(url))
            {
                if (!Path.GetExtension(url).Contains("htm") && !Path.GetExtension(url).Contains("com"))
                {
                    return linkList;
                }
            }

            Request request = GetRequest(url);

            string content = request.Content;
            var doc = new HtmlDocument();
            doc.LoadHtml(content);

            foreach (HtmlNode link in doc.DocumentNode.SelectNodes("//a[@href]"))
            {
                string src = link.GetAttributeValue("href", null);

                if (String.IsNullOrEmpty(src))
                    continue;

                if (Path.HasExtension(src))
                {
                    if (!Path.GetExtension(src).Contains(".htm"))
                        linkList.Add(src.FormatAsImageUrl(url));
                    else
                        linkList.Add(src.FormatAsUrl(_domainUrl));
                }
                else
                    linkList.Add(src.FormatAsUrl(_domainUrl));
            }

            Debug.WriteLine($"GetLinksFromUrl() execution time: {timer.Elapsed.Milliseconds.ToString()}, from {url}");
            return linkList;
        }

        public List<string[]> GetAltTagsFromUrl(string url)
        {
            List<string[]> results = new List<string[]>();
            string content = GetRequest(url).Content;
            var doc = new HtmlDocument();
            doc.LoadHtml(content);
            var images = doc.DocumentNode.Descendants("img");

            foreach (var img in images)
            {
                string src = img.GetAttributeValue("src", null);
                src = src.FormatAsImageUrl(url);

                if (string.IsNullOrEmpty(src))
                    continue;

                string alt = "";

                if (src.Contains("tracking.gif"))
                    alt = "(ignore) TRACKING GIF";
                else if (src.Contains("shim.png"))
                    alt = "(ignore) SHIM PNG";
                else
                    alt = System.Web.HttpUtility.HtmlDecode(img.GetAttributeValue("alt", null));

                if (alt == null)
                    alt = "MISSING";
                else if (alt == "")
                    alt = "BLANK";

                results.Add(new string[] { src, alt });
            }

            return results;
        }

        List<string[]> GetElementsWithText(string url, string text)
        {
            List<string[]> elements = new List<string[]>();

            if (string.IsNullOrEmpty(url))
                return elements;

            Request request = GetRequest(url);

            if (request == null)
                return elements;

            string content = request.Content;
            var doc = new HtmlDocument();
            doc.LoadHtml(content);

            var links = doc.DocumentNode.Descendants()
                            .Where(n => n.Attributes.Any(a => a.Value.Contains(text)));

            foreach (HtmlNode link in links)
            {
                elements.Add(new string[] { url, link.InnerHtml });
            }

            return elements;
        }

        List<string> GetLinksFromRequest(Request request)
        {
            List<string> linkList = new List<string>();
            string content = GetRequest(request.Url).Content;
            var doc = new HtmlDocument();
            doc.LoadHtml(content);

            var links = doc.DocumentNode.Descendants("a")
                            .Select(e => e.GetAttributeValue("href", null))
                            .Where(s => !String.IsNullOrEmpty(s));

            foreach (var lnk in links)
            {
                string src = lnk.FormatAsUrl(_domainUrl);

                if (String.IsNullOrEmpty(src))
                    continue;

                linkList.Add(src);
            }

            return linkList;
        }

        public void SetDefaultConnectionLimit(int minLimit)
        {
            if (ServicePointManager.DefaultConnectionLimit < minLimit)
                ServicePointManager.DefaultConnectionLimit = minLimit;

            ServicePointManager.Expect100Continue = false;
        }

        public Dictionary<string, List<string>> CrawlSite(string domainURL)
        {
            Stopwatch timer = Stopwatch.StartNew();
            SetDefaultConnectionLimit(100);

            Dictionary<string, List<string>> _toReturn = new Dictionary<string, List<string>>();
            _linksToCheck.Enqueue(domainURL);
            _uniqueLinksSet.Add(domainURL);
            int requestsMade = 0;
            var options = new ParallelOptions { MaxDegreeOfParallelism = 10 };

            while (_linksToCheck.Any())
            {
                Debug.WriteLine($"{_linksToCheck.Count} links to check, {requestsMade} requests made, {_uniqueLinksSet.Count} unique links found.");

                var p = _linksToCheck.Dequeue();

                List<string> urls = GetLinksFromUrl(p);
                requestsMade++;

                string code = GetRequest(p).StatusCode;

                if (!_toReturn.ContainsKey(code))
                {
                    _toReturn.Add(code, new List<string>());
                }

                _toReturn[code].Add(p);

                Debug.WriteLine($"{_linksToCheck.Count} links to check, {requestsMade} requests made, {_uniqueLinksSet.Count} unique links found.");

                if (urls != null && urls.Count != 0)
                    foreach (string url in urls)
                        if (_uniqueLinksSet.Add(url))
                            if (url.Contains(domainURL))
                                _linksToCheck.Enqueue(url);

                var images = GetImagesFromUrl(p);

                if (images != null && images.Count > 0)
                    foreach (string image in images)
                        _uniqueLinksSet.Add(image);
            }
            //while (_linksToCheck.Any())
            //{
            //    //Thread.Sleep(50);

            //    Debug.WriteLine($"{_linksToCheck.Count} links to check, {requestsMade} requests made, {_uniqueLinksSet.Count} unique links found.");

            //    var pages = _linksToCheck.ToList();
            //    _linksToCheck.Clear();

            //    Parallel.ForEach(pages, options, p =>
            //    {
            //        List<string> urls = new List<string>();

            //        urls = GetLinksFromUrl(p);
            //        requestsMade++;

            //        string code = GetRequest(p).StatusCode;

            //        if (!_toReturn.ContainsKey(code))
            //        {
            //            _toReturn.Add(code, new List<string>());
            //        }

            //        _toReturn[code].Add(p);

            //        Debug.WriteLine($"{_linksToCheck.Count} links to check, {requestsMade} requests made, {_uniqueLinksSet.Count} unique links found.");

            //        if (urls != null && urls.Count != 0)
            //            foreach (string url in urls)
            //                if (_uniqueLinksSet.Add(url))
            //                    if (url.Contains(domainURL))
            //                        _linksToCheck.Add(url);

            //        var images = GetImagesFromUrl(p);

            //        if (images != null && images.Count > 0)
            //            foreach (string image in images)
            //                _uniqueLinksSet.Add(image);
            //    });
            //}

            Debug.WriteLine($"{_linksToCheck.Count} links to check, {requestsMade} requests made, {_uniqueLinksSet.Count} unique links found.");
            Debug.WriteLine($"CrawlSite() execution time: {timer.Elapsed.Milliseconds.ToString()}");
            return _toReturn;
        }

        public Tuple<string, string> TakeScreenshot(string url = "", bool clearScreenshotDir = true, bool quitAfterSS = true, bool appendUrl=false)
        {
            BrowserDriver driver = BrowserDriver.NewDriver();
            return driver.TakeScreenshot(url, clearScreenshotDir, quitAfterSS, appendUrl);
        }

        public string GetPageTitle(string url, bool appendUrl=false)
        {
            string content = GetRequest(url).Content;
            int index = content.GetLastCharIndex("<title>");
            int length = content.GetIndexOf("</title>") - index;
            string substring = content.Substring(index, length);

            if (appendUrl)
                return $"{substring} ({url})";

            return substring;
        }

        public static string[] GetAuth(string url)
        {
            string username = "";
            string password = "";
            string cookieName = "";
            string cookieValue = "";

            foreach (var key in Config.AUTH.Keys)
            {
                if (url.Contains(key))
                {
                    username = Config.AUTH[key][0];
                    password = Config.AUTH[key][1];

                    foreach (var ckey in Config.COOKIES.Keys)
                    {
                        if (url.Contains(ckey))
                        {
                            cookieName = Config.COOKIES[ckey][0];
                            cookieValue = Config.COOKIES[ckey][1];
                            break;
                        }
                    }

                    break;
                }
            }

            return new string[] { username, password, cookieName, cookieValue };
        }
    }

    public class BrowserDriver : ChromeDriver
    {

        public string ScreenshotPath { get; private set; }

        public static BrowserDriver NewDriver(bool headless = true, bool startMaximized = true, bool logPerformanceAll = false, bool logBrowserAll = false)
        {
            ChromeOptions _options;
            ChromePerformanceLoggingPreferences _logPrefs;
            BrowserDriver driver;

            string chromeDriverPath = Environment.CurrentDirectory;
            chromeDriverPath.TrimEnd('/');

            if (!chromeDriverPath.EndsWith("Requests"))
            {
                int lastIndex = chromeDriverPath.LastIndexOf('/');
                chromeDriverPath = chromeDriverPath.Substring(0, lastIndex);
                chromeDriverPath += "/Requests";
            }

            chromeDriverPath += "/bin/Debug/netstandard2.0/";


            if (headless || startMaximized || logPerformanceAll || logBrowserAll)
            {
                _options = new ChromeOptions();
                _options.AddArgument("--user-agent=" + AgentHelpers.AgentIDToAgentString("win10chrome"));
                _options.AddArgument("--window-size=1920,1080");

                if (headless)
                    _options.AddArgument("--headless");
                if (startMaximized)
                    _options.AddArgument("--start-maximized");
                if (logPerformanceAll)
                {
                    _logPrefs = new ChromePerformanceLoggingPreferences();
                    _logPrefs.AddTracingCategories(new string[] { "devtools.timeline" });
                    _options.PerformanceLoggingPreferences = _logPrefs;
                    _options.SetLoggingPreference("performance", LogLevel.All);
                }
                if (logBrowserAll)
                {
                    _options.SetLoggingPreference(LogType.Browser, LogLevel.All);
                }
                driver = new BrowserDriver(chromeDriverPath, _options);
            }
            else
            {
                driver = new BrowserDriver();
            }

            return driver;
        }

        public BrowserDriver() { }
        public BrowserDriver(string chromeDriverDirectory, ChromeOptions options) : base(chromeDriverDirectory, options) { }

        ~BrowserDriver()
        {
            Quit();
        }

        public void GoToUrl(string url)
        {
            var auth = RequestFactory.GetAuth(url);

            if (auth[0] == "" && auth[2] == "")
            {
                Navigate().GoToUrl(url);
                return;
            }

            if (auth[2] != "")
            {
                // cookie auth
                DateTime time = DateTime.Now;
                time.AddDays(10);
                OpenQA.Selenium.Cookie ck = new OpenQA.Selenium.Cookie(auth[2], auth[3], url.GetDomain(), "/", time);
                Navigate().GoToUrl(url);
                Manage().Cookies.AddCookie(ck);
                Navigate().GoToUrl(url);
                return;
            }
            if (auth[0] != "")
            {
                // username/password auth
                Navigate().GoToUrl(url.InsertCredentials(auth[0], auth[1]));
                return;
            }
        }

        public void ClearScreenshots()
        {
            DirectoryInfo di = new DirectoryInfo(Directory.GetCurrentDirectory() + "/wwwroot/screenshots");

            foreach (FileInfo file in di.GetFiles())
                file.Delete();
        }

        public Tuple<string, string> TakeScreenshot(string url = "", bool clearScreenshotDir = true, bool quitAfterSS = true, bool appendUrl = false)
        {
            bool overrideMarqueeHeight = url.Contains("www.am.com") || url.Contains("am-stage") || url.Contains("ogemec-dev") || url.Contains("ogetogether");

            if (url != "")
            {
                GoToUrl(url);
                Thread.Sleep(2000);
            }

            if (clearScreenshotDir)
                ClearScreenshots();

            string FileName = "screenshot_" + Guid.NewGuid().ToString() + ".png";
            ScreenshotPath = Directory.GetCurrentDirectory() + "/wwwroot/screenshots/" + FileName;
            int oldHeight = Manage().Window.Size.Height;

            // HACK: If we're on am.com, we need to stop the header from dynamically resizing
            if (overrideMarqueeHeight)
            {
                try
                {
                    IWebElement element = FindElementByClassName("MarqueePartial");
                    string pixelHeight = element.GetCssValue("height");
                    IJavaScriptExecutor js = this as IJavaScriptExecutor;
                    js.ExecuteScript("$('head').append('<style> .MarqueePartial { height: " + pixelHeight + " !important; }</style>');");
                }
                catch (NoSuchElementException) { }

                try
                {
                    IWebElement bgImage = FindElementByClassName("fixedBackgroundImage");
                    string pixelHeight = bgImage.GetCssValue("height");
                    IJavaScriptExecutor js = this as IJavaScriptExecutor;
                    js.ExecuteScript("$('head').append('<style> .fixedBackgroundImage { height: " + pixelHeight + " !important; }</style>');");
                }
                catch (NoSuchElementException) { }
            }

            // Get the full page height of the website.
            int scrollHeight = (int)(long)ExecuteScript("return document.body.scrollHeight");
            int siteWidth = Manage().Window.Size.Width;

            // Set the new browser dimensions and maximizethe window.
            Manage().Window.Size = new Size(siteWidth, scrollHeight);
            Manage().Window.Maximize();
            Console.WriteLine(Manage().Window.Size.Height.ToString());

            byte[] bitmap = ((ITakesScreenshot)this).GetScreenshot().AsByteArray;

            using (Image image = Image.FromStream(new MemoryStream(bitmap)))
            {
                image.Save(ScreenshotPath, ImageFormat.Png);
            }

            string pageTitle = Title;

            if (quitAfterSS)
                Quit();
            else
            {
                // Return it back to normal after the screenshot.
                Manage().Window.Size = new Size(siteWidth, oldHeight);
                Manage().Window.Maximize();
            }

            if (appendUrl)
                return Tuple.Create($"{pageTitle} ({url})", "/screenshots/" + FileName);

            return Tuple.Create(pageTitle, "/screenshots/" + FileName);
        }

        public HashSet<LogEntry> GetPerformanceLogs() => new HashSet<LogEntry>(Manage().Logs.GetLog("performance"));
    }

    public class UserAgent
    {
        public string AgentID     { get; set; }
        public string AgentString { get; set; }
        public int ScreenWidth    { get; set; }
        public int ScreenHeight   { get; set; }

        public UserAgent(string agentID = "win10chrome", int width = 1920, int height = 1080)
        {
            AgentID = agentID;
            AgentString = AgentHelpers.AgentIDToAgentString(agentID);
            ScreenWidth = width;
            ScreenHeight = height;
        }
    }

    public static class Extensions
    {
        public static string StripTrailingSlashes(this string url) => url.TrimEnd('/');

        public static string StripExtension(this string url)
        {
            if (Path.HasExtension(url))
            {
                return url.Substring(0, url.LastIndexOf('/') + 1);
            }

            return url;
        }

        public static string GetDomain(this string url)
        {
            url = url.StripTrailingSlashes();
            url = url.TrimStart("https://www.".ToCharArray());
            url = url.TrimStart("http://www.".ToCharArray());
            url = url.TrimStart("https://".ToCharArray());
            url = url.TrimStart("http://".ToCharArray());

            if (url.IndexOf('.') != url.LastIndexOf('.'))
            {
                url = url.Substring(url.IndexOf('.') + 1);
            }

            return url;
        }

        public static int GetIndexOf(this string content, string substring)
        {
            return content.IndexOf(substring, StringComparison.CurrentCulture);
        }

        public static int GetIndexOf(this string content, string substring, int index)
        {
            return content.IndexOf(substring, index, StringComparison.CurrentCulture);
        }

        public static int GetLastCharIndex(this string content, string substring)
        {
            return content.IndexOf(substring, StringComparison.CurrentCulture) + substring.Length;
        }

        public static string InsertCredentials(this string data, string username, string password)
        {
            string temp = data;

            if (temp.StartsWith("https://"))
            {
                return temp.Insert("https://".Length, username + ":" + password + "@");
            }
            if (temp.StartsWith("http://"))
            {
                return temp.Insert("http://".Length, username + ":" + password + "@");
            }

            return data;
        }

        public static string FormatAsImageUrl(this string src, string url)
        {
            if (src.StartsWith("http"))
            {
                return src;
            }

            string imgSrc = src;
            string addy = url.StripExtension();

            if (!addy.EndsWith("/") && !imgSrc.StartsWith("/"))
                imgSrc = "/" + imgSrc;

            string imgUrl = addy + imgSrc;

            return imgUrl;
        }

        public static string FormatAsUrl(this string url, string domainUrl)
        {
            if (url.StartsWith("javascript"))
                return "";
            if (url.StartsWith("tel:"))
                return "";
            if (url.StartsWith("mailto:"))
                return "";
            if (url.StartsWith("#"))
                return "";
            if (url.Contains("sitecore"))
                return "";
            if (url.StartsWith("http"))
                return url.StripTrailingSlashes();

            url = url.TrimEnd(new char[] { '/' });

            if (url.StartsWith("//"))
            {
                return "https:" + url;
            }
            if (url.StartsWith("/"))
            {
                return domainUrl + url;
            }

            return url;
        }

        //public static (string key, string value) FormatUrl(this string link, string url)
        //{
        //    link = link.StripTrailingSlashes();

        //    if (Path.HasExtension(link))
        //    {

        //    }
        //    else
        //    {
        //        if (url.StartsWith("javascript"))
        //            return ("", "");
        //        if (url.StartsWith("tel:"))
        //            return ("", "");
        //        if (url.StartsWith("mailto:"))
        //            return ("", "");
        //        if (url.StartsWith("#"))
        //            return ("", "");
        //        if (url.Contains("sitecore"))
        //            return ("", "");
        //        if (url.StartsWith("//"))
        //        {
        //            return ("link", "https:" + url);
        //        }
        //        if (url.StartsWith("/"))
        //        {
        //            return ("link", url + link);
        //        }

        //        return ("link", link);
        //    }
        //}

        public static string GetContent(this HttpWebResponse response)
        {
            Stream data = response.GetResponseStream();
            string content = String.Empty;

            using (StreamReader sr = new StreamReader(data))
            {
                content = sr.ReadToEnd();
            }

            return content;
        }

        public static string GetStatusCode(this HttpWebResponse response)
        {
            int statusCode = 0;

            if (response != null)
            {
                statusCode = (int)response.StatusCode;
            }
            else
            {
                statusCode = (int)HttpStatusCode.NotFound;
            }

            return statusCode.ToString();
        }

        public static string ToStringList(this List<string> list, string separator)
        {
            var str = string.Join(separator, list.Select(s => AgentHelpers.FormatAgentID(s)));
            return "(" + str + ")";
        }

        public static List<string> GetTwitterCardInfo(this string content)
        {
            List<string> results = new List<string>
            {
                "Card: ", "Title: ", "Description: ", "Image: ", "Site: ", "Creator: "
            };

            var doc = new HtmlDocument();
            doc.LoadHtml(content);

            var elements = doc.DocumentNode.Descendants("meta")
                              .Where(s => !String.IsNullOrEmpty(s.GetAttributeValue("name", null))
                                     && s.GetAttributeValue("name", null).ToLower().StartsWith(""));

            foreach (var item in elements)
            {
                string text = "";
                string temp = "";

                if (item.GetAttributeValue("name", null).Contains("card"))
                {
                    temp = item.GetAttributeValue("content", null);
                    text = temp == "" ? text = "missing" : text = temp;
                    results[0] += System.Web.HttpUtility.HtmlDecode(text);
                    continue;
                }
                if (item.GetAttributeValue("name", null).Contains("title"))
                {
                    temp = item.GetAttributeValue("content", null);
                    text = temp == "" ? text = "missing" : text = temp;
                    results[1] += System.Web.HttpUtility.HtmlDecode(text);
                    continue;
                }
                if (item.GetAttributeValue("name", null).Contains("description"))
                {
                    temp = item.GetAttributeValue("content", null);
                    text = temp == "" ? text = "missing" : text = temp;
                    results[2] += System.Web.HttpUtility.HtmlDecode(text);
                    continue;
                }
                if (item.GetAttributeValue("name", null).Contains("image"))
                {
                    temp = item.GetAttributeValue("content", null);
                    text = temp == "" ? text = "missing" : text = temp;
                    results[3] += System.Web.HttpUtility.HtmlDecode(text);
                    continue;
                }
                if (item.GetAttributeValue("name", null).Contains("site"))
                {
                    temp = item.GetAttributeValue("content", null);
                    text = temp == "" ? text = "missing" : text = temp;
                    results[4] += System.Web.HttpUtility.HtmlDecode(text);
                    continue;
                }
                if (item.GetAttributeValue("name", null).Contains("creator"))
                {
                    temp = item.GetAttributeValue("content", null);
                    text = temp == "" ? text = "missing" : text = temp;
                    results[5] += System.Web.HttpUtility.HtmlDecode(text);
                    continue;
                }
            }

            return results;
        }
    }

    public static class AgentHelpers
    {
        public static string FormatAgentID(string agent)
        {
            if (agent == "osxChrome")
                return "OSX Chrome";
            if (agent == "osxFirefox")
                return "OSX Firefox";
            if (agent == "osxSafari")
                return "OSX Safari";
            if (agent == "win7ie9")
                return "Win7 IE9";
            if (agent == "win7ie10")
                return "Win7 IE10";
            if (agent == "win10ie11")
                return "Win10 IE10";
            if (agent == "win10edge")
                return "Win10 Edge";
            if (agent == "win10chrome")
                return "Win10 Chrome";
            if (agent == "win10ff")
                return "Win10 Firefox";
            if (agent == "iphone")
                return "iPhone 6";
            if (agent == "ipad")
                return "iPad Mini 2";
            if (agent == "android")
                return "Galaxy Tab 4";

            return "Undefined Agent";
        }

        public static string AgentIDToAgentString(string agentID)
        {
            switch (agentID)
            {
                case "osxChrome":
                    return "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.113 Safari/537.36";
                case "osxFirefox":
                    return "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.11; rv:49.0) Gecko/20100101 Firefox/49.0";
                case "osxSafari":
                    return "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/602.2.14 (KHTML, like Gecko) Version/10.0.1 Safari/602.2.14";
                case "win10chrome":
                    return "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.115 Safari/537.36";
                case "win10ff":
                    return "Mozilla/5.0 (Windows NT 10.0; WOW64; rv:50.0) Gecko/20100101 Firefox/50.0";
                case "win10edge":
                    return "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/46.0.2486.0 Safari/537.36 Edge/13.10586";
                case "win10ie11":
                    return "Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; rv:11.0) like Gecko";
                case "win7ie10":
                    return "Mozilla / 5.0(compatible; MSIE 10.0; Windows NT 6.1; Trident / 6.0)";
                case "win7ie9":
                    return "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0)";
                case "iphone":
                    return "Mozilla/5.0 (iPhone; CPU iPhone OS 10_1_1 like Mac OS X) AppleWebKit/602.2.14 (KHTML, like Gecko) Version/10.0 Mobile/14B100 Safari/602.1";
                case "ipad":
                    return "Mozilla/5.0 (iPad; CPU OS 10_1_1 like Mac OS X) AppleWebKit/602.2.14 (KHTML, like Gecko) Version/10.0 Mobile/14B100 Safari/602.1";
                case "android":
                    return "Mozilla/5.0 (Linux; Android 5.1.1; SM-T330NU Build/LMY47X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.85 Safari/537.36";
                default:
                    return "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.115 Safari/537.36";
            }
        }

        public class UserAgent
        {
            public string AgentString { get; set; }
            public int ScreenWidth { get; set; }
            public int ScreenHeight { get; set; }

            public UserAgent(string agentString, int width, int height)
            {
                AgentString = agentString;
                ScreenWidth = width;
                ScreenHeight = height;
            }
        }

        public static string GetElementFromJson(this string data, string element)
        {
            var item = JToken.Parse(data)[element];

            if (item == null)
                return null;

            return JToken.Parse(data)[element].Value<string>();
        }
    }

    public class MissingCaseException : Exception
    {
        public MissingCaseException(){}
        public MissingCaseException(string message) : base(message){}
        public MissingCaseException(string message, Exception inner) : base(message, inner){}
    }
}