Siege Green Data Gatherer

package main

import (
    "bytes"
    "context"
    "encoding/json"
    "flag"
    "fmt"
    "io"
    "io/ioutil"
    "net/http"
    "strconv"
    "strings"
    "sync"
    "time"

    "golang.org/x/net/html"
    "golang.org/x/time/rate"
)

var zKillQPSFlag = flag.Float64("zkillqps", 1, "The QPS traffic to send to zKill")
var zKillBurstFlag = flag.Int("zkillburst", 1, "The burst of traffic allowed to send to zKill")
var outAggFile = flag.String("outstats", "agg_stats.csv", "The file to write aggregate stats csv to")
var srcDataFile = flag.String("srcdata", "src_data.csv", "The file to write data source csv to")

//
// ZKILL CLIENT
//

const (
    ccpGamesAstrahusShipTypeId = 35832
    ccpGamesAthanorShipTypeId  = 35835
    ccpGamesRaitaruShipTypeId  = 35825
    zkillLossPages             = "https://zkillboard.com/api/losses/shipTypeID/%d/year/%d/month/%d/page/%d/"
    // For some reason, the killmail details are extremely limited in JSON format. This
    // means we get to scrape HTML pages instead -- joy!
    zkillKillmail   = "https://zkillboard.com/kill/%d/"
    zKillTimeFormat = "2006-01-02 15:04"
)

// ZKillSummaries is a list of ZKillSummary.
type ZKillSummaries []ZKillSummary

// ZKillSummary contains killboard identifiers and summary details of a killmail.
type ZKillSummary struct {
    ID     int                `json:"killmail_id"`
    Detail ZKillSummaryDetail `json:"zkb"`
}

// ZKillSummaryDetail has summarizable details for each killmail.
type ZKillSummaryDetail struct {
    LocationID     int     `json:"locationID"`
    Hash           string  `json:"hash"`
    FittedValue    float64 `json:"fittedValue"`
    DroppedValue   float64 `json:"droppedValue"`
    DestroyedValue float64 `json:"destroyedValue"`
    TotalValue     float64 `json:"totalValue"`
    Points         int     `json:"points"`
    NPC            bool    `json:"npc"`
    Solo           bool    `json:"solo"`
    AWOX           bool    `json:"awox"`
}

type ZKillClient struct {
    client *http.Client
    r      *rate.Limiter
}

// get wraps HTTP GET calls and applies ZKill-specific logic to the requests.
func (c *ZKillClient) get(ctx context.Context, endpoint string) (*http.Response, error) {
    req, err := http.NewRequestWithContext(ctx, "GET", endpoint, nil)
    if err != nil {
        return nil, err
    }
    req.Header.Set("User-Agent", "Siege Green Historical Analysis")

    c.r.Wait(ctx)
    return c.client.Do(req)
}

// getZKillEndpoint obtains data from a ZKill API endpoint.
//
// dest must be a pointer.
func (c *ZKillClient) getZKillEndpoint(ctx context.Context, endpoint string, dest interface{}) error {
    r, err := c.get(ctx, endpoint)
    if err != nil {
        return err
    }
    defer r.Body.Close()
    b, err := ioutil.ReadAll(r.Body)
    if err != nil {
        return err
    }
    return json.Unmarshal(b, dest)
}

// getZKillHTML obtains the HTML page for a single killmail.
func (c *ZKillClient) getZKillHTML(ctx context.Context, endpoint string) (b *bytes.Buffer, err error) {
    r, err := c.get(ctx, endpoint)
    if err != nil {
        return
    }
    defer r.Body.Close()
    k, err := ioutil.ReadAll(r.Body)
    if err != nil {
        return
    }
    b = bytes.NewBuffer(k)
    return
}

// getZKillLossPages obtains a summary page of losses for a particular type.
//
// See https://github.com/zKillboard/zKillboard/wiki/API-(History)
func (c *ZKillClient) getZKillLossPages(ctx context.Context, typeId, year, month, page int) (d ZKillSummaries, err error) {
    err = c.getZKillEndpoint(
        ctx,
        fmt.Sprintf(
            zkillLossPages,
            typeId,
            year,
            month,
            page),
        &d)
    return
}

// GetZKillAstrahusLossPage gets a page of summaries of Astrahus losses.
func (c *ZKillClient) GetZKillAstrahusLossPage(ctx context.Context, year, month, page int) (ZKillSummaries, error) {
    return c.getZKillLossPages(ctx, ccpGamesAstrahusShipTypeId, year, month, page)
}

// GetZKillAthanorLossPage gets a page of summaries of Athanor losses.
func (c *ZKillClient) GetZKillAthanorLossPage(ctx context.Context, year, month, page int) (ZKillSummaries, error) {
    return c.getZKillLossPages(ctx, ccpGamesAthanorShipTypeId, year, month, page)
}

// GetZKillRaitaruLossPage gets a page of summaries of Raitaru losses.
func (c *ZKillClient) GetZKillRaitaruLossPage(ctx context.Context, year, month, page int) (ZKillSummaries, error) {
    return c.getZKillLossPages(ctx, ccpGamesRaitaruShipTypeId, year, month, page)
}

// GetZKillKillmail obtains data on a specific killmail.
func (c *ZKillClient) GetZKillKillmail(ctx context.Context, id int) (b *bytes.Buffer, err error) {
    return c.getZKillHTML(ctx, fmt.Sprintf(zkillKillmail, id))
}

//
// HTML PARSING
//

type HTMLKillmail struct {
    KillID         string
    Ship           string
    ShipID         string
    System         string
    SystemID       string
    SystemSecurity string
    Region         string
    RegionID       string
    Time           string
    Attackers      []*HTMLAttacker
}

func (k *HTMLKillmail) clean() {
    k.KillID = strings.TrimSpace(k.KillID)
    k.Ship = strings.TrimSpace(k.Ship)
    k.ShipID = strings.TrimSpace(k.ShipID)
    k.System = strings.TrimSpace(k.System)
    k.SystemID = strings.TrimSpace(k.SystemID)
    k.SystemSecurity = strings.TrimSpace(k.SystemSecurity)
    k.Region = strings.TrimSpace(k.Region)
    k.RegionID = strings.TrimSpace(k.RegionID)
    k.Time = strings.TrimSpace(k.Time)
    for _, a := range k.Attackers {
        a.clean()
    }
}

type HTMLAttacker struct {
    Pilot         string
    PilotID       string
    Ship          string
    ShipID        string
    Item          string
    ItemID        string
    Corporation   string
    CorporationID string
    Alliance      string
    AllianceID    string
    Damage        string
    Percent       string
}

func (k *HTMLAttacker) clean() {
    k.Pilot = strings.TrimSpace(k.Pilot)
    k.PilotID = strings.TrimSpace(k.PilotID)
    k.Ship = strings.TrimSpace(k.Ship)
    k.ShipID = strings.TrimSpace(k.ShipID)
    k.Item = strings.TrimSpace(k.Item)
    k.ItemID = strings.TrimSpace(k.ItemID)
    k.Corporation = strings.TrimSpace(k.Corporation)
    k.CorporationID = strings.TrimSpace(k.CorporationID)
    k.Alliance = strings.TrimSpace(k.Alliance)
    k.AllianceID = strings.TrimSpace(k.AllianceID)
    k.Damage = strings.TrimSpace(k.Damage)
    k.Percent = strings.TrimSpace(k.Percent)
}

func parseZKillmailHTML(r io.Reader, d *HTMLKillmail) error {
    doc, err := html.Parse(r)
    if err != nil {
        return err
    }
    err = processHTMLNode(doc, d)
    if err != nil {
        return err
    }
    d.clean()
    return nil
}

func processHTMLNode(n *html.Node, d *HTMLKillmail) error {
    // Visit
    maybeExtractKillID(n, d)
    maybeExtractShip(n, d)
    maybeExtractSystemAndRegion(n, d)
    maybeExtractTime(n, d)
    maybeExtractAttacker(n, d)

    // Recur
    for c := n.FirstChild; c != nil; c = c.NextSibling {
        if err := processHTMLNode(c, d); err != nil {
            return err
        }
    }
    return nil
}

func maybeExtractKillID(n *html.Node, d *HTMLKillmail) {
    if len(d.KillID) > 0 {
        return
    }
    if n.Data == "link" {
        hasRelCanonical := false
        for _, a := range n.Attr {
            hasRelCanonical = hasRelCanonical || (a.Key == "rel" && a.Val == "canonical")
        }
        if hasRelCanonical {
            for _, a := range n.Attr {
                if a.Key == "href" {
                    d.KillID = a.Val
                }
            }
        }
    }
}

func maybeExtractShip(n *html.Node, d *HTMLKillmail) {
    if len(d.Ship) > 0 {
        return
    }
    if n.Data == "Ship:" {
        n = n.Parent
        for n.NextSibling != nil && n.Data != "td" {
            n = n.NextSibling
        }
        for c := n.FirstChild; c != nil; c = c.NextSibling {
            if c.Data == "a" {
                n = c
            }
        }
        if n == nil {
            return
        }
        if n.FirstChild != nil {
            d.Ship = n.FirstChild.Data
        }
        for _, a := range n.Attr {
            if a.Key == "href" {
                d.ShipID = a.Val
            }
        }
    }
}

func maybeExtractSystemAndRegion(n *html.Node, d *HTMLKillmail) {
    if len(d.System) > 0 {
        return
    }
    if n.Data == "System:" {
        n = n.Parent
        for n.NextSibling != nil && n.Data != "td" {
            n = n.NextSibling
        }
        for c := n.FirstChild; c != nil; c = c.NextSibling {
            if c.Data == "a" {
                for _, a := range c.Attr {
                    if a.Key == "href" && strings.Contains(a.Val, "system") {
                        d.SystemID = a.Val
                        if c.FirstChild != nil {
                            d.System = c.FirstChild.Data
                        }
                    } else if a.Key == "href" && strings.Contains(a.Val, "region") {
                        d.RegionID = a.Val
                        if c.FirstChild != nil {
                            d.Region = c.FirstChild.Data
                        }
                    }
                }
            } else if c.Data == "span" {
                for cc := c.FirstChild; cc != nil; cc = cc.NextSibling {
                    if cc.Data == "span" {
                        if cc.FirstChild != nil {
                            d.SystemSecurity = cc.FirstChild.Data
                        }
                    }
                }
            }
        }
    }
}

func maybeExtractTime(n *html.Node, d *HTMLKillmail) {
    if len(d.Time) > 0 {
        return
    }
    if n.Data == "Time:" {
        n = n.Parent
        for n.NextSibling != nil && n.Data != "td" {
            n = n.NextSibling
        }
        for _, a := range n.Attr {
            if a.Key == "class" && a.Val == "info_kill_dttm" {
                if n.FirstChild != nil {
                    d.Time = n.FirstChild.Data
                }
            }
        }
    }
}

func maybeExtractAttacker(n *html.Node, d *HTMLKillmail) {
    hasAttacker := false
    if n.Data == "tr" {
        for _, a := range n.Attr {
            hasAttacker = hasAttacker || (a.Key == "class" && a.Val == "attacker")
        }
    }
    if !hasAttacker {
        return
    }
    var m HTMLAttacker
    processHTMLNodeAttacker(n, &m)
    d.Attackers = append(d.Attackers, &m)
}

func processHTMLNodeAttacker(n *html.Node, m *HTMLAttacker) {
    // Visit
    maybeExtractAttackerPilot(n, m)
    maybeExtractAttackerShip(n, m)
    maybeExtractAttackerItem(n, m)
    maybeExtractAttackerCorporation(n, m)
    maybeExtractAttackerAlliance(n, m)
    maybeExtractAttackerDamage(n, m)
    maybeExtractAttackerPercent(n, m)

    // Recur
    for c := n.FirstChild; c != nil; c = c.NextSibling {
        processHTMLNodeAttacker(c, m)
    }
}

func maybeExtractAttackerPilot(n *html.Node, m *HTMLAttacker) {
    if len(m.Pilot) > 0 {
        return
    }
    if n.Data == "a" {
        hasTooltip := false
        for _, a := range n.Attr {
            hasTooltip = hasTooltip || (a.Key == "rel" && a.Val == "tooltip")
        }
        if hasTooltip {
            return
        }
        for _, a := range n.Attr {
            if a.Key == "href" && strings.Contains(a.Val, "character") {
                if n.FirstChild != nil {
                    m.Pilot = n.FirstChild.Data
                }
                m.PilotID = a.Val
            }
        }
    }
}
func maybeExtractAttackerShip(n *html.Node, m *HTMLAttacker) {
    if len(m.Ship) > 0 {
        return
    }
    if n.Data == "a" {
        hasTooltip := false
        for _, a := range n.Attr {
            hasTooltip = hasTooltip || (a.Key == "rel" && a.Val == "tooltip")
        }
        if hasTooltip {
            return
        }
        for _, a := range n.Attr {
            if a.Key == "href" && strings.Contains(a.Val, "ship") {
                if n.FirstChild != nil {
                    m.Ship = n.FirstChild.Data
                }
                m.ShipID = a.Val
            }
        }
    }
}

func maybeExtractAttackerItem(n *html.Node, m *HTMLAttacker) {
    if len(m.Item) > 0 {
        return
    }
    if n.Data == "a" {
        hasTooltip := false
        for _, a := range n.Attr {
            hasTooltip = hasTooltip || (a.Key == "rel" && a.Val == "tooltip")
        }
        if hasTooltip {
            return
        }
        for _, a := range n.Attr {
            if a.Key == "href" && strings.Contains(a.Val, "item") {
                if n.FirstChild != nil {
                    m.Item = n.FirstChild.Data
                }
                m.ItemID = a.Val
            }
        }
    }
}

func maybeExtractAttackerCorporation(n *html.Node, m *HTMLAttacker) {
    if len(m.Corporation) > 0 {
        return
    }
    if n.Data == "a" {
        hasTooltip := false
        for _, a := range n.Attr {
            hasTooltip = hasTooltip || (a.Key == "rel" && a.Val == "tooltip")
        }
        if hasTooltip {
            return
        }
        for _, a := range n.Attr {
            if a.Key == "href" && strings.Contains(a.Val, "corporation") {
                if n.FirstChild != nil {
                    m.Corporation = n.FirstChild.Data
                }
                m.CorporationID = a.Val

            }
        }
    }
}

func maybeExtractAttackerAlliance(n *html.Node, m *HTMLAttacker) {
    if len(m.Alliance) > 0 {
        return
    }
    if n.Data == "a" {
        hasTooltip := false
        for _, a := range n.Attr {
            hasTooltip = hasTooltip || (a.Key == "rel" && a.Val == "tooltip")
        }
        if hasTooltip {
            return
        }
        for _, a := range n.Attr {
            if a.Key == "href" && strings.Contains(a.Val, "alliance") {
                if n.FirstChild != nil {
                    m.Alliance = n.FirstChild.Data
                }
                m.AllianceID = a.Val
            }
        }
    }
}

func maybeExtractAttackerDamage(n *html.Node, m *HTMLAttacker) {
    if len(m.Damage) > 0 {
        return
    }
    if n.Data == "td" {
        for _, a := range n.Attr {
            if a.Key == "class" && strings.Contains(a.Val, "damage") {
                if n.FirstChild != nil {
                    m.Damage = n.FirstChild.Data
                }
            }
        }
    }
}

func maybeExtractAttackerPercent(n *html.Node, m *HTMLAttacker) {
    if len(m.Percent) > 0 {
        return
    }
    if n.Data == "small" {
        if n.FirstChild != nil {
            m.Percent = n.FirstChild.Data
        }
    }
}

//
// SIGNALS ANALYSIS
//

type ParsedKillmail struct {
    KillID         int
    Ship           string
    ShipID         int
    System         string
    SystemID       int
    SystemSecurity float64
    Region         string
    RegionID       string
    T              time.Time
    Attackers      []ParsedAttacker
}

type ParsedAttacker struct {
    Pilot         string
    PilotID       int
    Ship          string
    ShipID        int
    Item          string
    ItemID        int
    Corporation   string
    CorporationID int
    Alliance      string
    AllianceID    int
    Damage        int
    Percent       float64
}

func parseID(s string) (int, error) {
    if len(s) == 0 {
        return -1, nil
    }
    ss := strings.Split(s, "/")
    if len(ss) != 4 {
        return -1, fmt.Errorf("Unexpected ID to parse: %s", s)
    }
    if ss[2] == "" {
        return -1, nil
    }
    return strconv.Atoi(ss[2])
}

func parseInt(s string) (int, error) {
    return strconv.Atoi(
        strings.ReplaceAll(s, ",", ""))
}

func parseFloat(s string) (float64, error) {
    if len(s) == 0 {
        return 0, nil
    }
    return strconv.ParseFloat(
        strings.ReplaceAll(
            strings.TrimSuffix(s, "%"),
            ",",
            ""),
        64)
}

func ParseKillmail(k *HTMLKillmail) (p ParsedKillmail, err error) {
    p.KillID, err = parseID(strings.TrimPrefix(k.KillID, "https://zkillboard.com"))
    if err != nil {
        return
    }
    p.Ship = k.Ship
    p.ShipID, err = parseID(k.ShipID)
    if err != nil {
        return
    }
    p.System = k.System
    p.SystemID, err = parseID(k.SystemID)
    if err != nil {
        return
    }
    p.SystemSecurity, err = parseFloat(k.SystemSecurity)
    if err != nil {
        return
    }
    p.Region = k.Region
    p.RegionID = k.RegionID
    p.T, err = time.Parse(zKillTimeFormat, k.Time)
    if err != nil {
        return
    }
    for _, a := range k.Attackers {
        var pa ParsedAttacker
        pa, err = parseAttacker(a)
        if err != nil {
            return
        }
        p.Attackers = append(p.Attackers, pa)
    }
    return
}

func parseAttacker(k *HTMLAttacker) (p ParsedAttacker, err error) {
    p.Pilot = k.Pilot
    p.PilotID, err = parseID(k.PilotID)
    if err != nil {
        return
    }
    p.Ship = k.Ship
    p.ShipID, err = parseID(k.ShipID)
    if err != nil {
        return
    }
    p.Item = k.Item
    p.ItemID, err = parseID(k.ItemID)
    if err != nil {
        return
    }
    p.Corporation = k.Corporation
    p.CorporationID, err = parseID(k.CorporationID)
    if err != nil {
        return
    }
    p.Alliance = k.Alliance
    p.AllianceID, err = parseID(k.AllianceID)
    if err != nil {
        return
    }
    p.Damage, err = parseInt(k.Damage)
    if err != nil {
        return
    }
    p.Percent, err = parseFloat(k.Percent)
    if err != nil {
        return
    }
    return
}

type AggregatedSignals struct {
    V  []*DateSignal
    mu *sync.Mutex
}

type DateSignal struct {
    Day time.Time
    S   *Signals
}

func (ds DateSignal) String() string {
    return fmt.Sprintf("%s: %s", ds.Day.Format("2006-01-02"), ds.S)
}

const (
    theInitiativeAllianceID = 1900696668
    briscRubalPilotID       = 883889312
)

type Signals struct {
    TypeToNLoss     map[int]int
    NKillmailsInit  int
    NKillmailsBrisc int
    NHighSec        int
    NLowSec         int
    NNullSec        int
    NWormhole       int
    NPochven        int
}

func (s *Signals) Add(km ParsedKillmail) {
    if s.TypeToNLoss == nil {
        s.TypeToNLoss = make(map[int]int)
    }
    s.TypeToNLoss[km.ShipID]++
    foundInit := false
    foundBrisc := false
    for _, a := range km.Attackers {
        if a.AllianceID == theInitiativeAllianceID {
            foundInit = true
        }
        if a.PilotID == briscRubalPilotID {
            foundBrisc = true
        }
    }
    if foundInit {
        s.NKillmailsInit++
    }
    if foundBrisc {
        s.NKillmailsBrisc++
    }
    // Colloquial "area of game" accounting
    if km.SystemSecurity >= 0.49 {
        s.NHighSec++
    } else if km.SystemSecurity >= 0.05 {
        s.NLowSec++
    } else if km.Region == "Pochven" {
        s.NPochven++
    } else if strings.HasPrefix(km.Region, "A-R") ||
        strings.HasPrefix(km.Region, "A-R0") ||
        strings.HasPrefix(km.Region, "ADR0") ||
        strings.HasPrefix(km.Region, "B-R0") ||
        strings.HasPrefix(km.Region, "C-R0") ||
        strings.HasPrefix(km.Region, "D-R0") ||
        strings.HasPrefix(km.Region, "E-R0") ||
        strings.HasPrefix(km.Region, "F-R0") ||
        strings.HasPrefix(km.Region, "G-R0") ||
        strings.HasPrefix(km.Region, "H-R0") ||
        strings.HasPrefix(km.Region, "K-R0") ||
        strings.HasPrefix(km.Region, "PR-01") {
        s.NWormhole++
    } else {
        s.NNullSec++
    }
}

func NewAggregatedSignals() *AggregatedSignals {
    return &AggregatedSignals{
        mu: &sync.Mutex{},
    }
}

func (a *AggregatedSignals) Add(km ParsedKillmail) {
    a.mu.Lock()
    defer a.mu.Unlock()
    day := time.Date(km.T.Year(), km.T.Month(), km.T.Day(), 0, 0, 0, 0, km.T.Location())
    var found *DateSignal
    for _, v := range a.V {
        if v.Day == day {
            found = v
            break
        }
    }
    if found == nil {
        found = &DateSignal{
            Day: day,
            S:   &Signals{},
        }
        a.V = append(a.V, found)
    }
    found.S.Add(km)
}

//
// COORDINATOR
//

type Coordinator struct {
    pivot  time.Time
    ships  []int
    client *ZKillClient
    ctx    context.Context
}

func NewCoordinator(
    pivot time.Time,
    ships []int,
    client *ZKillClient,
    ctx context.Context) *Coordinator {
    return &Coordinator{
        pivot:  pivot,
        ships:  ships,
        client: client,
        ctx:    ctx,
    }
}

func (c *Coordinator) Do() (as *AggregatedSignals, pkms []ParsedKillmail) {
    as = NewAggregatedSignals()
    end := time.Now()
    durAfterPivot := end.Sub(c.pivot)
    begin := c.pivot.Add(-durAfterPivot)

    fmt.Printf("Time Period Begin: %s\n", begin)
    fmt.Printf("Time Period End:   %s\n", end)

    // Get all years
    var years []int
    if begin.Year() == end.Year() {
        years = []int{end.Year()}
    } else {
        for y := begin.Year(); y <= end.Year(); y++ {
            years = append(years, y)
        }
    }
    // Get all months
    months := []int{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12}
    if len(years) == 1 {
        months = []int{}
        for m := begin.Month(); m <= end.Month(); m++ {
            months = append(months, int(m))
        }
    }
    // TODO: >1 year boundary will not work due to month overlap

    // Fetch lists and determine bounds
    nRequests := 0
    var zss []ZKillSummaries
    for _, y := range years {
        for _, m := range months {
            for _, s := range c.ships {
                for p := 1; p <= 10; p++ {
                    nRequests++
                    fmt.Printf("Making summary request %d for getting pages\n", nRequests)
                    zs, err := c.client.getZKillLossPages(c.ctx, s, y, m, p)
                    if err != nil {
                        fmt.Println(err)
                        continue
                    }
                    if len(zs) == 0 {
                        break
                    }
                    zss = append(zss, zs)
                }
            }
        }
    }

    // Convert list to killmails
    for _, zs := range zss {
        for _, zkm := range zs {
            nRequests++
            fmt.Printf("Making specific request %d for getting killmail %d\n", nRequests, zkm.ID)
            buf, err := c.client.GetZKillKillmail(c.ctx, zkm.ID)
            if err != nil {
                fmt.Println(err)
                continue
            }

            var km HTMLKillmail
            err = parseZKillmailHTML(buf, &km)
            if err != nil {
                fmt.Println(err)
                continue
            }
            pkm, err := ParseKillmail(&km)
            if err != nil {
                fmt.Println(err)
                continue
            }
            if pkm.T.Before(begin) {
                fmt.Printf("Skipping %d as %s is before %s", pkm.KillID, pkm.T, begin)
                continue
            }
            pkms = append(pkms, pkm)
            as.Add(pkm)
        }
    }
    return
}

//
// FILE WRITING
//

func (a *AggregatedSignals) Write(filename string, ships []int) error {
    a.mu.Lock()
    defer a.mu.Unlock()
    var buf bytes.Buffer

    fmt.Fprint(&buf, "date,n_hi,n_low,n_null,n_wh,n_poch,n_init,n_brisc")
    for _, s := range ships {
        fmt.Fprintf(&buf, ",ship_%d", s)
    }
    fmt.Fprintln(&buf)
    for _, d := range a.V {
        fmt.Fprintf(
            &buf,
            "%s,%d,%d,%d,%d,%d,%d,%d",
            d.Day,
            d.S.NHighSec,
            d.S.NLowSec,
            d.S.NNullSec,
            d.S.NWormhole,
            d.S.NPochven,
            d.S.NKillmailsInit,
            d.S.NKillmailsBrisc)
        for _, s := range ships {
            fmt.Fprintf(&buf, ",%d", d.S.TypeToNLoss[s])
        }
        fmt.Fprintln(&buf)
    }
    return ioutil.WriteFile(filename, buf.Bytes(), 0644)
}

func WriteParsedKillmails(pkms []ParsedKillmail, filename string) error {
    var buf bytes.Buffer

    fmt.Fprintln(&buf, "date,kill_id,ship,ship_id,system,system_id,sec,region,region_id,attackers")
    for _, pkm := range pkms {
        fmt.Fprintf(
            &buf,
            "%s,%d,%s,%d,%s,%d,%f,%s,%s,",
            pkm.T,
            pkm.KillID,
            pkm.Ship,
            pkm.ShipID,
            pkm.System,
            pkm.SystemID,
            pkm.SystemSecurity,
            pkm.Region,
            pkm.RegionID)
        for _, pa := range pkm.Attackers {
            fmt.Fprintf(
                &buf,
                ",%s,%d,%s,%d,%s,%d,%s,%d,%s,%d,%d,%f",
                pa.Pilot,
                pa.PilotID,
                pa.Ship,
                pa.ShipID,
                pa.Item,
                pa.ItemID,
                pa.Corporation,
                pa.CorporationID,
                pa.Alliance,
                pa.AllianceID,
                pa.Damage,
                pa.Percent)
        }
        fmt.Fprintln(&buf)
    }
    return ioutil.WriteFile(filename, buf.Bytes(), 0644)
}

func main() {

    zkc := &ZKillClient{
        &http.Client{},
        rate.NewLimiter(rate.Limit(*zKillQPSFlag), *zKillBurstFlag),
    }
    ships := []int{
        ccpGamesAstrahusShipTypeId,
        ccpGamesAthanorShipTypeId,
        ccpGamesRaitaruShipTypeId,
    }
    patchDay, err := time.Parse("2006-01-02 15:04" /*Siege Green Patch Time:*/, "2022-05-10 12:00")
    if err != nil {
        fmt.Println(err)
        return
    }
    c := NewCoordinator(patchDay,
        ships,
        zkc,
        context.Background())
    as, pkms := c.Do()
    err = as.Write(*outAggFile, ships)
    if err != nil {
        fmt.Println(err)
    }
    err = WriteParsedKillmails(pkms, *srcDataFile)
    if err != nil {
        fmt.Println(err)
    }
}