allareacodes.com Scraper

Imports System.IO
Imports System.Threading
Imports Strings
Imports HttpSync

Module Module1
    Private codes As New List(Of String)
    Private threadCount As Integer
    Private closedThreads As Integer
    Private ReadOnly threadLocker As New Object
    Private keepRunning As Boolean
    Private totalScraped As Integer
    Private codeIndex As Integer
    Private triggers As New List(Of String)

    Sub Main()
        'intro
        Console.WriteLine("Welcome to allareacodes.com scraper.")
        Console.WriteLine("Make sure areacodes.txt exists with desired area codes, one per line.")
        Console.WriteLine("Example:")
        Console.WriteLine("123")
        Console.WriteLine("456")
        Console.WriteLine("789")
        Console.WriteLine()

        'area codes
        Console.WriteLine("Press any key to load the file.")
        Console.WriteLine()

        Console.ReadKey()

        While Not File.Exists("areacodes.txt")
            Console.WriteLine("Could not find file areacodes.txt")
            Console.WriteLine("Create it and press any key to load it.")
            Console.WriteLine()
            Console.ReadKey()
        End While

        While New FileInfo("areacodes.txt").Length = 0
            Console.WriteLine("File areacodes.txt exists but it is empty, add something to it.")
            Console.WriteLine("Press any key to load it.")
            Console.WriteLine()
            Console.ReadKey()
        End While

        codes.AddRange(File.ReadLines("areacodes.txt"))
        Console.WriteLine($"Loaded {codes.Count} area code/s.")
        Console.WriteLine()

        'triggers
        Console.WriteLine("Make sure triggers.txt exists with desired trigger text, one per line.")
        Console.WriteLine("Example:")
        Console.WriteLine("Verizon")
        Console.WriteLine("Sprint")
        Console.WriteLine("etc..")
        Console.WriteLine()

        Console.WriteLine("Press any key to load the file.")
        Console.WriteLine()

        Console.ReadKey()

        While Not File.Exists("triggers.txt")
            Console.WriteLine("Could not find file triggers.txt")
            Console.WriteLine("Create it and press any key to load it.")
            Console.WriteLine()
            Console.ReadKey()
        End While

        While New FileInfo("triggers.txt").Length = 0
            Console.WriteLine("File triggers.txt exists but it is empty, add something to it.")
            Console.WriteLine("Press any key to load it.")
            Console.WriteLine()
            Console.ReadKey()
        End While

        triggers.AddRange(File.ReadLines("triggers.txt"))
        Console.WriteLine($"Loaded {triggers.Count} trigger/s.")
        Console.WriteLine()

        'Threads
        Console.WriteLine("Enter the amount of threads to use.")
        Console.WriteLine("Thread count is 0 based. (Entering 0 will use one thread.)")
        Console.WriteLine()

        While Not Integer.TryParse(Console.ReadLine(), threadCount)
            Console.WriteLine("That is not a valid number.")
            Console.WriteLine("Enter the amount of threads to use.")
            Console.WriteLine()
        End While

        Console.WriteLine($"Threads set to {threadCount}.")
        Console.WriteLine()

        'start
        Console.WriteLine("Press any key to start.")
        Console.ReadKey()
        Console.WriteLine("Key press accepted. Started..")
        Console.WriteLine()

        keepRunning = True
        For i As Integer = 0 To threadCount
            Dim t As New Thread(AddressOf StartThread) With {.IsBackground = True}
            t.Start()
        Next

        While closedThreads <= threadCount
            Thread.Sleep(10000)
            Console.WriteLine($"{Now.ToLongTimeString} - Scraped {totalScraped.ToThousands}")
            Console.WriteLine()
        End While

        Console.WriteLine("All threads have been closed.")
        Console.WriteLine($"Scraped: {totalScraped.ToThousands}")
        Console.WriteLine()
        Console.WriteLine("Press any key to exit.")
        Console.ReadKey()
    End Sub

    Private Sub StartThread()
        Dim strCode As String
        Dim strNumber As String
        Dim strProvider As String

        Dim h As New HttpProperties
        With h
            .Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8"
            .AutoDecompress = True
            .AcceptLanguage = "en-US,en;q=0.9"
            .UpgradeInsecureRequestsToDefault()
            .UserAgent = "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36"
            .CookiesEnabled = False
        End With

        While keepRunning
            SyncLock threadLocker
                If codeIndex >= codes.Count Then
                    Exit While
                End If

                strCode = codes(codeIndex)
                codeIndex += 1
            End SyncLock

            With h
                .Url = New Uri($"https://www.allareacodes.com/{strCode}")
            End With

            Using r As HttpResponse = HttpRequest.Get(h)
                If Not r.Success OrElse r.IsEmptyBody Then
                    SyncLock threadLocker
                        Using sw As New StreamWriter("Redo.txt", True)
                            sw.WriteLine(strCode)
                        End Using
                    End SyncLock

                    Continue While
                End If

                Dim b As String = r.ReadBody

                For Each s As String In b.ParseToArray("<div class=""list-group-item"">", "</span>")
                    strNumber = s.Parse("<div class=""col-xs-12 prefix-col1"">", "</div>").Trim
                    strProvider = s.Parse("<div class=""col-xs-12 prefix-col4"">", "</div>").Trim

                    If Not strNumber.IsEmpty AndAlso Not strProvider.IsEmpty Then
                        For Each strTrigger In triggers
                            If strProvider.Contains(strTrigger) Then
                                SyncLock threadLocker
                                    Using sw As New StreamWriter($"{strTrigger}.txt", True)
                                        sw.WriteLine(strNumber)
                                    End Using
                                End SyncLock

                                Interlocked.Increment(totalScraped)
                                Exit For
                            End If
                        Next
                    End If

                    'Debug.Print(strProvider & " - " & strNumber)
                Next
            End Using
        End While

        SyncLock threadLocker
            closedThreads += 1
            Console.WriteLine($"Closed thread #{closedThreads}")

            If closedThreads > threadCount Then
                Console.WriteLine("Program stopped.")
                Console.WriteLine()
            End If
        End SyncLock
    End Sub
End Module