open System
open FSharp.Data
open Newtonsoft.Json
open System.Data.Entity
open System.ComponentModel.DataAnnotations

let format (source:string, page, language, date:string) = String.Format(source, page, language, date)

[<CLIMutable>]
type Repo = {
    [<Key>]
    id:int;
    language:string;
    name:string;
    full_name:string;
    html_url:string;
    description:string;
    fork:bool;
    created_at:DateTime;
    updated_at:DateTime;
    pushed_at:DateTime;
    size:int;
    stargazers_count:int;
    watchers_count:int;
    has_issues:bool;
    has_projects:bool;
    has_downloads:bool;
    has_wiki:bool;
    has_pages:bool;
    forks_count:int;
    open_issues_count:int;
    forks:int;
    open_issues:int;
    watchers:int;
}

type GithubResponse = {
    total_count:int;
    incomplete_results:bool;
    items:Repo[];
}

type RepositoryContext() =
    inherit DbContext()
         [<DefaultValue>] val mutable repositories: DbSet<Repo>
         member this.Repositories with get() = this.repositories and set f = this.repositories <- f

let context = new RepositoryContext()
let connectionString = "Data Source=(localdb)\MSSQLLocalDB;Initial Catalog=Github;Integrated Security=True;Connect Timeout=30;Encrypt=False;TrustServerCertificate=True;ApplicationIntent=ReadWrite;MultiSubnetFailover=False"
context.Database.Connection.ConnectionString <- connectionString

let addItem (item:Repo) =
    context.Repositories.Add(item) |> ignore
    context.SaveChanges() |> ignore

type GithubParser (language:string) =
    let userAgent = "extremecodetv (Microsoft Windows 10.0.15063 ; x64; ru-RU; Octokit 0.24.0)"
    let searchUrl = "https://api.github.com/search/repositories?page={0}&per_page=1000&order=desc&q=language:{1}+created:{2}..{2}"
    let event = new Event<_>()
    let span = TimeSpan.FromDays(1.0)

    let rec loader (date:DateTime) =
        async {
            if date.Year.Equals 2016 then
                Console.WriteLine("Done")

            let url = format (searchUrl, 1, language, date.ToString "yyyy-MM-dd" )
            do! Async.Sleep 6000 //10 requests in 1 minute
            Console.WriteLine (date.ToString "yyyy-MM-dd")
            try
                let! source = Http.AsyncRequestString(url, headers = ["User-Agent", userAgent] )

                let response = JsonConvert.DeserializeObject<GithubResponse> source
                for repo in response.items do
                    event.Trigger repo
            with
                | Failure msg -> loader date

            loader (date - span)
        } |> Async.RunSynchronously

    member this.Run(date:DateTime) = loader date
    member this.Abort() = Async.CancelDefaultToken
    member this.NewData = event.Publish

let parser = new GithubParser("CSharp")
parser.NewData
    |> Event.map(fun x -> addItem(x))
    |> ignore

parser.Run(DateTime.Now)