open FSharp.Data
open System
open System.IO
open System.Net
open System.Text
open System.Collections.Specialized

type Word =
    Id of int
    | String of string

type Text = (Word seq) * (string array)

let task1 (s:string) =
    let rec process (tmp : char * int) (result : (char * int) list) (pos:int) (s:string) : ((char * int) list) =
        if (s.Length <= pos) then
            tmp::result
        elif (s.[pos] = (fst tmp)) then
            process (fst tmp, (snd tmp) + 1) result (pos + 1) s
        else
            process (s.[pos], 1) (tmp::result) (pos + 1) s
    if (s.Length = 0) then
        []
    else
        process (s.[0], 1) [] 1 s
        |> List.rev

let rec task2 (compressed: (char * int) list) =
    let rec decompress (part: char * int) =
        if (snd part <= 0) then
            ""
        else
            string(fst part) + (decompress (fst part, (snd part) - 1))
    if (compressed.Length <= 0) then
        ""
    else
        let a::b = compressed
        (decompress a) + (task2 b)

let task3 (path:string) =
    let lines = File.ReadAllLines(path)
    lines
    |> Array.map (fun x -> x.Split([|' '; '.'; ','; ';'|]) |> Array.filter (fun x -> x.Length > 0))
    |> Array.concat
    |> Array.toSeq

let task4 (words: string seq) : Text =
    let wordsCount = words |> Seq.countBy id
    let dict =  wordsCount
                |> Seq.filter (fun x -> (snd x) > 5)
                |> Seq.map (fst)
                |> Seq.toArray
    let text =  words
                |> Seq.map (fun x -> try Id(Array.findIndex (fun y -> y = x) dict) with | :? System.Collections.Generic.KeyNotFoundException -> String(x))
    (text, dict)

let task5 (text:Text) =
    let dict = snd text
    let words = fst text
    Seq.map (fun x -> match x with | Id(t) -> dict.[t] | String(s) -> s) words

let task6 (paths: string seq) =
    paths
    |> Seq.map (fun x -> async { return task4 (task3 x)})
    |> Async.Parallel
    |> Async.RunSynchronously


[<EntryPoint>]
let main argv =
    let a = task6 [@"D:\1.txt"; @"D:\2.txt"]
    0