Imports System.IO
Imports System.Text

Module Module1

    ' Removes duplicated lines from any text file that is found inside the working directory.
    ' It generates a backup file.

    Sub Main()

        Dim dirPath As String = My.Application.Info.DirectoryPath
        Dim curLines As Integer
        Dim totalLines As Integer
        Dim totalFiles As Integer
        Dim lineSet As New HashSet(Of String)(StringComparer.Ordinal)
        Dim enc As Encoding = Encoding.Default
        Dim sw As New Stopwatch()

        For Each fi As FileInfo In New DirectoryInfo(dirPath).GetFiles("*.txt", SearchOption.TopDirectoryOnly)

            totalFiles += 1

            Console.WriteLine(String.Format("Parsing file....: {0}", fi.FullName))
            sw.Restart()
            For Each line As String In File.ReadLines(fi.FullName, enc)
                curLines += 1
                lineSet.Add(line)
            Next line
            sw.Stop()

            Console.WriteLine(String.Format("Total lines.....: {0}", curLines.ToString("n0")))
            Console.WriteLine(String.Format("Acchieved lines.: {0}", lineSet.Count.ToString("n0")))
            Console.WriteLine(String.Format("Duplicated lines: {0}", (curLines - lineSet.Count).ToString("n0")))
            Console.WriteLine(String.Format("Time elapsed....: {0}", sw.Elapsed.ToString("mm\:ss\:ffff")))
            Console.WriteLine(String.Format("Saving file.....: {0}", fi.FullName))

            Dim bakFilePath As String = Path.Combine(fi.DirectoryName, fi.Name & ".bak")
            fi.CopyTo(bakFilePath, overwrite:=True)
            File.WriteAllLines(fi.FullName, lineSet, enc)

            lineSet.Clear()
            totalLines += curLines
            curLines = 0

            Console.WriteLine(Environment.NewLine)
        Next

        Console.WriteLine(String.Format("Total amount of text files parsed..: {0}", totalFiles))
        Console.WriteLine(String.Format("Total amount of lines in text files: {0}", totalLines))

        Console.WriteLine("Press any key to exit...")
        Console.ReadKey(intercept:=True)
        Environment.Exit(0)

    End Sub

End Module