Advertisement
Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- #I @"packages\FSharp.Data.Toolbox.Twitter.0.6\lib\net40"
- #I @"packages\FSharp.Data.2.1.1\lib\net40"
- #r @".\packages\FSharp.Data.2.1.1\lib\net40\FSharp.Data.dll"
- #r @".\packages\FSharp.Data.Toolbox.Twitter.0.6\lib\net40\FSharp.Data.Toolbox.Twitter.dll"
- open FSharp.Data.Toolbox.Twitter
- let key = "mKQL29XNemjQbLlQ8t0pBg"
- let secret = "T27HLDve1lumQykBUgYAbcEkbDrjBe6gwbu0gqi4saM"
- let twitter = Twitter.AuthenticateAppOnly(key, secret)
- open System
- open System.IO
- let stopWords = File.ReadAllLines(__SOURCE_DIRECTORY__ + "\\StopWords.txt")
- let words (text:string) = text.Split([|'#';'@';'.';',';';';':';'!';'?';'`';' ';'\r';'\n';'"';'\'';'“';'”';'(';')';'+';'-'|], StringSplitOptions.RemoveEmptyEntries)
- let rec getWords id =
- let ndc =
- match id with
- | Some id -> twitter.Search.Tweets("#ndcoslo",count=100,sinceId=id)
- | None -> twitter.Search.Tweets("#ndcoslo",count=100)
- let statuses = ndc.Statuses
- let words =
- statuses
- |> Seq.map (fun tweet -> tweet.Text)
- |> String.concat " "
- |> words
- |> Seq.map (fun s -> s.ToLower())
- |> Seq.filter (fun word -> word <> "ndcoslo")
- |> Seq.filter (fun word -> word.StartsWith("http") |> not)
- |> Seq.filter (fun word -> word.StartsWith("co/") |> not)
- |> Seq.filter (fun word -> word.StartsWith("//t") |> not)
- |> Seq.filter (fun word -> word <> "rt")
- |> Seq.filter (fun word -> word <> "ht")
- |> Seq.filter (fun word -> word <> "co")
- |> Seq.filter (fun word -> word <> "cc")
- |> Seq.filter (fun word -> word <> "yo")
- |> Seq.filter (fun word -> word <> "bit")
- |> Seq.filter (fun word -> word <> "didn")
- |> Seq.filter (fun word -> word <> "dont")
- |> Seq.filter (fun word -> word <> "isn")
- |> Seq.filter (fun word -> word <> "htt")
- |> Seq.filter (fun word -> word <> "&")
- |> Seq.filter (fun word -> word.Length > 2)
- |> Seq.filter (fun word -> word.EndsWith("…") |> not)
- |> Seq.filter (fun word -> word.EndsWith("%") |> not)
- |> Seq.filter (fun word -> let x,_ = Int32.TryParse(word) in not x)
- |> Seq.filter (fun word -> stopWords |> Array.exists ((=) word) |> not)
- |> Seq.toList
- if statuses |> Seq.length > 0 then
- let last = statuses |> Seq.last
- let s = last.CreatedAt
- // "Fri Jun 19 13:11:05 +0000 2015" -> "Jun 19 2015"
- let date = s.[4..10] + s.[26..]
- printfn "%A" date
- if DateTime.Parse(date) >= System.DateTime(2015,6,15)
- then words @ getWords (Some last.Id)
- else []
- else []
- let allWords = getWords None
- let ndcoslo = allWords |> String.concat " "
- File.WriteAllText(@"C:\temp\ndcoslo.txt", ndcoslo)
- let myCounts =
- allWords
- |> Seq.countBy id
- |> Seq.sortBy (fun (_, count) -> -count)
- |> Seq.take 50
- |> Seq.toArray
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement