#I @"packages\FSharp.Data.Toolbox.Twitter.0.6\lib\net40"
#I @"packages\FSharp.Data.2.1.1\lib\net40"

#r @".\packages\FSharp.Data.2.1.1\lib\net40\FSharp.Data.dll"
#r @".\packages\FSharp.Data.Toolbox.Twitter.0.6\lib\net40\FSharp.Data.Toolbox.Twitter.dll"

open FSharp.Data.Toolbox.Twitter

let key = "mKQL29XNemjQbLlQ8t0pBg"
let secret = "T27HLDve1lumQykBUgYAbcEkbDrjBe6gwbu0gqi4saM"
let twitter = Twitter.AuthenticateAppOnly(key, secret)

open System
open System.IO

let stopWords = File.ReadAllLines(__SOURCE_DIRECTORY__ + "\\StopWords.txt")

let words (text:string) = text.Split([|'#';'@';'.';',';';';':';'!';'?';'`';' ';'\r';'\n';'"';'\'';'“';'”';'(';')';'+';'-'|], StringSplitOptions.RemoveEmptyEntries)

let rec getWords id =
   let ndc =
      match id with
      | Some id -> twitter.Search.Tweets("#ndcoslo",count=100,sinceId=id)
      | None -> twitter.Search.Tweets("#ndcoslo",count=100)         
   let statuses = ndc.Statuses  
   let words =
      statuses
      |> Seq.map (fun tweet -> tweet.Text) 
      |> String.concat " " 
      |> words
      |> Seq.map (fun s -> s.ToLower())
      |> Seq.filter (fun word -> word <> "ndcoslo")
      |> Seq.filter (fun word -> word.StartsWith("http") |> not)
      |> Seq.filter (fun word -> word.StartsWith("co/") |> not)
      |> Seq.filter (fun word -> word.StartsWith("//t") |> not)
      |> Seq.filter (fun word -> word <> "rt")
      |> Seq.filter (fun word -> word <> "ht")
      |> Seq.filter (fun word -> word <> "co")
      |> Seq.filter (fun word -> word <> "cc")
      |> Seq.filter (fun word -> word <> "yo")
      |> Seq.filter (fun word -> word <> "bit")
      |> Seq.filter (fun word -> word <> "didn")
      |> Seq.filter (fun word -> word <> "dont")
      |> Seq.filter (fun word -> word <> "isn")
      |> Seq.filter (fun word -> word <> "htt")
      |> Seq.filter (fun word -> word <> "&amp")      
      |> Seq.filter (fun word -> word.Length > 2)
      |> Seq.filter (fun word -> word.EndsWith("…") |> not)
      |> Seq.filter (fun word -> word.EndsWith("%") |> not)
      |> Seq.filter (fun word -> let x,_ = Int32.TryParse(word) in not x)      
      |> Seq.filter (fun word -> stopWords |> Array.exists ((=) word) |> not)
      |> Seq.toList
   if statuses |> Seq.length > 0 then
      let last = statuses |> Seq.last
      let s = last.CreatedAt
      // "Fri Jun 19 13:11:05 +0000 2015" -> "Jun 19 2015"
      let date = s.[4..10] + s.[26..]
      printfn "%A" date
      if DateTime.Parse(date) >= System.DateTime(2015,6,15)
      then words @ getWords (Some last.Id)
      else []
   else []

let allWords = getWords None
let ndcoslo = allWords |> String.concat " "
File.WriteAllText(@"C:\temp\ndcoslo.txt", ndcoslo)

let myCounts = 
   allWords
   |> Seq.countBy id
   |> Seq.sortBy (fun (_, count) -> -count)
   |> Seq.take 50
   |> Seq.toArray