UNICAMP Crawler

Dictionary<String, int>   courseIndex;
            Dictionary<String,String> htmlPages;
            List<Subject>      parsedSubjects;
            WebRequests        webReq           = new WebRequests();
            int                semesterCounter  = 0;
            string             auxHtml;
            string             htmlPiece =  String.Empty;

            // Creating Html Map
            courseIndex    = new Dictionary<String,int>();
            htmlPages      = new Dictionary<String,String> ();
            auxHtml        = htmlPage;

            // Creating Regular Expression to find name of courses
            Regex regex = new Regex ("<p><br><b><font face=\"Arial,Helvetica\"><font color=\"#000099\"><font size=-1>(.+?)</font></font></font></b>", RegexOptions.Singleline);

            // Splitting HTML Page in Fragments of courses
            int startIndex = 0;
            while (regex.IsMatch (auxHtml, startIndex))
            {
                try
                {
                    // Checking name of the course and saving it's offset
                    int index         = regex.Match (auxHtml, startIndex).Groups[1].Index;
                    string courseName = regex.Match (auxHtml, startIndex).Groups[1].Value;

                    // Adding name of the course and offset to dictionary
                    courseIndex.Add (courseName, index);
                    startIndex = index;

                    // Splitting HTML Page
                    if (regex.IsMatch (auxHtml, startIndex))
                    {
                        int endIndex = regex.Match (auxHtml, startIndex).Groups[1].Index;
                        endIndex     = endIndex - startIndex;
                        htmlPiece    = auxHtml.Substring (startIndex, endIndex);
                    }
                    else
                    {
                        htmlPiece = auxHtml.Substring (startIndex);
                    }

                    htmlPages.Add (courseName, htmlPiece);
                }
                catch (Exception ex)
                {
                    Logs.LogWriter.LogError (ex);
                }
            }

            // Reading Html Fragments and Parsing Subjects
            foreach (KeyValuePair<String, String> kv in htmlPages)
            {
                // Loading html map based on current html fragment
                parsedSubjects       = new List<Subject> ();
                semesterCounter      = 0;
                HtmlDocument htmlMap = new HtmlDocument ();
                htmlMap.LoadHtml (kv.Value);

                // Reaching tables with subjects
                HtmlNodeCollection semesterNodes = htmlMap.DocumentNode.SelectNodes ("//table[@bgcolor='#D7EBFF']");

                // Iterating over tables with subjects
                foreach (HtmlNode smNode in semesterNodes)
                {
                    try
                    {
                        // Incrementing counter of semesters
                        semesterCounter++;

                        // Reaching subject nodes
                        HtmlNodeCollection subjectNodes = smNode.SelectNodes (".//font[@size='-1']//a");

                        // Checking for subject with only "ELET." subject. This ignores these cases
                        if (subjectNodes == null)
                        {
                            continue;
                        }

                        // Iterating over subject nodes
                        foreach (HtmlNode sbNode in subjectNodes)
                        {
                            // Parsing code of the subject
                            string code = sbNode.InnerText;
                            string name = "SEM NOME";

                            code = String.IsNullOrEmpty(code) ? "SEMCODIGO" : code;

                            // Web Request for the page with subject names and descriptions
                            string pageLink  = Consts.EMENTASSUFIX + sbNode.Attributes["href"].Value;
                            pageLink         = pageLink.Replace ("..", String.Empty);
                            string subjsPage = webReq.Get (pageLink);

                            // Loading HtmlMap for the page of subject names and descriptions
                            HtmlDocument subjectsMap = new HtmlDocument();
                            subjectsMap.LoadHtml (subjsPage);
                            HtmlNodeCollection subjects = subjectsMap.DocumentNode.SelectNodes (".//font[@size='-1']//a");

                            // Iterating over subject nodes until both codes match
                            // Once codes match, it parses the name of the course that matched
                            foreach (HtmlNode node in subjects)
                            {
                                if (node.InnerText.Contains (code))
                                {
                                    name = node.InnerText.Replace (code, String.Empty).Trim();
                                    break;
                                }
                            }

                            // Adding Subject to list of subjects of this course
                            parsedSubjects.Add (new Subject (name, code, 0, "", semesterCounter));
                        }
                    }
                    catch (Exception ex)
                    {
                        Logs.LogWriter.LogError (ex);
                    }
                }

                // Creating Course Instance
                Course tmpCourse = new Course (kv.Key, semesterCounter, parsedSubjects);
                courses.Add (tmpCourse);

                // Updating UI
                LoadCoursesUIGrid (uiComps, tmpCourse);

                // Adding Unique subjects to list of subjects
                foreach (Subject sub in parsedSubjects)
                {
                    if (!allSubjects.Exists( m => m.Name.Equals(sub.Name) && m.Code.Equals(sub.Code)))
                    {
                        allSubjects.Add(sub);
                    }
                }
            }
        }