API tools faq
paste
Advertisement
Guest User

Untitled

a guest
Dec 7th, 2019
151
0
Never
Add comment
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
Python 19.89 KB | None | 0 0
raw download clone embed print report
  1. import collections
  2. import functools
  3. import itertools
  4. import json
  5. import math
  6. import urllib.parse
  7.  
  8. import pandas
  9. import scrapy
  10. REPORTS = ["fflogs report IDs goes here", "e.g.,", "YA4QvwPCft1qdxBX"]
  11. API_KEY = "fflogs API key goes here"
  12.  
  13. PHASES = ["Twin", "Nael", "Quickmarch", "Blackfire", "Fellruin", "Heavensfall", "Tenstrike", "Octet", "Adds", "Golden"] + \
  14.          ["Garuda (not woken)", "Garuda (woken)", "Ifrit (not woken)", "Ifrit (woken)", "Titan (not woken)", "Titan (woken)", "LB Phase", "Predation", "Annihilation", "Suppression", "Final"] + \
  15.          ["Pepsiman", "Limit Cut", "CC + BJ", "Time Stop + Inception", "Wormhole", "Perfect Alex"]
  16. TRIOS = {
  17.     9954: "Quickmarch",
  18.     9955: "Blackfire",
  19.     9956: "Fellruin",
  20.     9957: "Heavensfall",
  21.     9958: "Tenstrike",
  22.     9959: "Octet",
  23. }
  24. ULTIMATES = {
  25.     11126: "Predation",
  26.     11596: "Annihilation",
  27.     11597: "Suppression",
  28.     11151: "Final", # 3x Viscous Aetheroplasm
  29. }
  30. WOKEN_DEBUFF = 1000000 + 1529;
  31.  
  32. TRIO_FILTER = "type = 'cast' AND (" + " OR ".join(map(lambda t: "ability.id = " + str(t), TRIOS.keys())) + ")"
  33. ULTIMATE_FILTER = "(type = 'cast' AND (" + " OR ".join(map(lambda t: "ability.id = " + str(t), ULTIMATES.keys())) + "))" + \
  34.     " OR (type = 'applydebuff' AND ability.id = " + str(WOKEN_DEBUFF) + ")"
  35. TITAN_GAOL_FILTER = "(type = 'death' AND target.disposition = 'friendly') OR ability.id = 11115 OR ability.id = 11116 OR (type = 'applydebuff' AND ability.id = 1000292)"
  36.  
  37. class FFLogsSpider(scrapy.Spider):
  38.     name = "FFLogs"
  39.     custom_settings = {
  40.         "COOKIES_ENABLED": False,
  41.     }
  42.  
  43.     fightData = {}
  44.     uwuRoleDps = {}
  45.     ucobRoleDps = {}
  46.  
  47.     titanGaolData = {
  48.         "deaths": collections.Counter(),
  49.         "success": collections.Counter(),
  50.         "failPositioning": collections.Counter(),
  51.         "failDeathCount": 0,
  52.         "failPositioningCount": 0,
  53.     }
  54.  
  55.     def start_requests(self):
  56.         for report in REPORTS:
  57.             yield scrapy.Request(
  58.                 "https://www.fflogs.com/v1/report/fights/{}?api_key={}&translate=true".format(report, API_KEY),
  59.                 functools.partial(self.fights, report)
  60.             )
  61.  
  62.     def fights(self, reportId, response):
  63.         data = json.loads(response.body_as_unicode())
  64.  
  65.         data["actors"] = {}
  66.         for actor in itertools.chain(data["friendlies"], data["enemies"]):
  67.             data["actors"][actor["id"]] = actor
  68.  
  69.         uwuStart = math.inf
  70.         uwuEnd = -math.inf
  71.         uwuFights = []
  72.  
  73.         teaStart = math.inf
  74.         teaEnd = -math.inf
  75.         teaFights = []
  76.  
  77.         for fight in data["fights"]:
  78.             fight["report"] = data
  79.             fight["reportId"] = reportId
  80.             if fight["zoneName"] == "The Unending Coil Of Bahamut (Ultimate)":
  81.                 if "lastPhaseForPercentageDisplay" not in fight or fight["lastPhaseForPercentageDisplay"] == 1:
  82.                     self.fightData.setdefault(reportId, []).append((fight, "Twin"))
  83.                 elif fight["lastPhaseForPercentageDisplay"] == 2:
  84.                     self.fightData.setdefault(reportId, []).append((fight, "Nael"))
  85.                 elif fight["lastPhaseForPercentageDisplay"] == 3:
  86.                     yield response.follow(
  87.                         "https://www.fflogs.com/v1/report/events/{}?".format(reportId) +
  88.                             urllib.parse.urlencode({
  89.                                 "api_key": API_KEY,
  90.                                 "start": fight["start_time"],
  91.                                 "end": fight["end_time"],
  92.                                 "filter": TRIO_FILTER
  93.                             }),
  94.                         functools.partial(self.ucobEvents, fight)
  95.                     )
  96.                 elif fight["lastPhaseForPercentageDisplay"] == 4:
  97.                     self.fightData.setdefault(reportId, []).append((fight, "Adds"))
  98.                 elif fight["lastPhaseForPercentageDisplay"] == 5:
  99.                     self.fightData.setdefault(reportId, []).append((fight, "Golden"))
  100.  
  101.                     #if fight["bossPercentage"] <= 2000:
  102.                     if "kill" in fight and fight["kill"]:
  103.                         yield scrapy.Request(
  104.                             "https://www.fflogs.com/v1/report/tables/damage-done/{}?".format(fight["reportId"]) +
  105.                                 urllib.parse.urlencode({
  106.                                     "api_key": API_KEY,
  107.                                     "start": fight["start_time"],
  108.                                     "end": fight["end_time"],
  109.                                     "filter": "IN RANGE FROM type = 'begincast' AND ability.id = 9964 TO type = 'cast' AND ability.id = 9965 END"
  110.                                 }),
  111.                             functools.partial(self.ucobDamageDone, fight)
  112.                         )
  113.                 else:
  114.                     assert(False)
  115.             elif fight["zoneName"] == "The Weapon's Refrain (Ultimate)":
  116.                 if "lastPhaseForPercentageDisplay" not in fight:
  117.                     self.fightData.setdefault(reportId, []).append((fight, "Garuda (not woken)"))
  118.                 else:
  119.                     uwuStart = min(uwuStart, fight["start_time"])
  120.                     uwuEnd = max(uwuEnd, fight["end_time"])
  121.                     uwuFights.append(fight)
  122.             elif fight["zoneName"] == "The Epic of Alexander (Ultimate)":
  123.                 if "lastPhaseForPercentageDisplay" not in fight or fight["lastPhaseForPercentageDisplay"] == 1:
  124.                     if not fight.get("lastPhaseIsIntermission"):
  125.                         self.fightData.setdefault(reportId, []).append((fight, "Pepsiman"))
  126.                     else:
  127.                         self.fightData.setdefault(reportId, []).append((fight, "Limit Cut"))
  128.                 elif (fight["lastPhaseForPercentageDisplay"] == 4 or
  129.                     (fight["lastPhaseForPercentageDisplay"] == 3 and fight.get("lastPhaseIsIntermission"))):
  130.                     self.fightData.setdefault(reportId, []).append((fight, "Perfect Alex"))
  131.                 else:
  132.                     teaStart = min(teaStart, fight["start_time"])
  133.                     teaEnd = max(teaEnd, fight["end_time"])
  134.                     teaFights.append(fight)
  135.  
  136.         if uwuStart < uwuEnd:
  137.             yield self.getAllEvents(
  138.                 reportId, uwuStart, uwuEnd,
  139.                 {"filter": "({}) OR ({})".format(ULTIMATE_FILTER, TITAN_GAOL_FILTER)},
  140.                 functools.partial(self.uwuEvents, uwuFights)
  141.             )
  142.         if teaStart < teaEnd:
  143.             yield self.getAllEvents(
  144.                 reportId, teaStart, teaEnd,
  145.                 {"filter": "type = 'cast' AND (" +
  146.                     "ability.id = 18494 OR " + # Judgement Nisi
  147.                     "ability.id = 18522 OR " + # Temporal Stasis
  148.                     "ability.id = 18542" + # Wormhole
  149.                 ")"},
  150.                 functools.partial(self.teaEvents, teaFights)
  151.             )
  152.  
  153.     def getAllEvents(self, reportId, start, end, args, callback):
  154.         args.update({
  155.             "api_key": API_KEY,
  156.             "start": start,
  157.             "end": end,
  158.         })
  159.         events = []
  160.  
  161.         def continuation(response):
  162.             nonlocal args, events
  163.             data = json.loads(response.body_as_unicode())
  164.             events.extend(data["events"])
  165.             if len(data["events"]) != 0:
  166.                 args["start"] = data["events"][-1]["timestamp"] + 1
  167.                 return response.follow(
  168.                     "https://www.fflogs.com/v1/report/events/{}?".format(reportId) +
  169.                         urllib.parse.urlencode(args),
  170.                     continuation
  171.                 )
  172.             else:
  173.                 events = pandas.Series(events, index = map(lambda e: e["timestamp"], events))
  174.                 return callback(events)
  175.  
  176.         return scrapy.Request(
  177.             "https://www.fflogs.com/v1/report/events/{}?".format(reportId) +
  178.                 urllib.parse.urlencode(args),
  179.             continuation
  180.         )
  181.  
  182.     def ucobEvents(self, fight, response):
  183.         data = json.loads(response.body_as_unicode())
  184.         assert("nextPageTimestamp" not in data)
  185.  
  186.         if len(data["events"]) == 0:
  187.             # Probably died to phase transition
  188.             self.fightData.setdefault(fight["reportId"], []).append((fight, "Nael"))
  189.         else:
  190.             self.fightData.setdefault(fight["reportId"], []).append((fight, TRIOS[data["events"][-1]["ability"]["guid"]]))
  191.  
  192.     def ucobDamageDone(self, fight, response):
  193.         data = json.loads(response.body_as_unicode())
  194.  
  195.         activeTime = max(e["activeTime"] for e in data["entries"]) / 1000
  196.         print("\nFight {}:{} Golden -> 1st Enrage hit DPS".format(fight["reportId"], fight["id"]))
  197.         for entry in data["entries"]:
  198.             if entry["type"] == "LimitBreak" and entry["name"] != "Limit Break":
  199.                 continue
  200.             self.ucobRoleDps.setdefault(entry["type"], []).append(entry["total"] / activeTime)
  201.             print("{}: {:.0f} dps".format(entry["type"], entry["total"] / activeTime))
  202.         print("")
  203.  
  204.     def uwuEvents(self, fights, allEvents):
  205.         def rfindAbility(events, abilities):
  206.             for e, event in events.iloc[::-1].items():
  207.                 if "ability" not in event:
  208.                     continue
  209.                 if event["ability"]["guid"] in abilities:
  210.                     return event
  211.             return None
  212.  
  213.         for fight in fights:
  214.             events = allEvents.loc[fight["start_time"]:fight["end_time"]]
  215.  
  216.             phase = None
  217.             lastWokenTarget = None
  218.             if fight["lastPhaseForPercentageDisplay"] == 4:
  219.                 phase = "LB Phase"
  220.             elif fight["lastPhaseForPercentageDisplay"] == 5:
  221.                 ultimateEvent = rfindAbility(events, ULTIMATES)
  222.                 phase = ULTIMATES[ultimateEvent["ability"]["guid"]] if ultimateEvent else "LB Phase"
  223.  
  224.                 #if fight["bossPercentage"] <= 2000:
  225.                 if "kill" in fight and fight["kill"]:
  226.                     yield scrapy.Request(
  227.                         "https://www.fflogs.com/v1/report/tables/damage-done/{}?".format(fight["reportId"]) +
  228.                             urllib.parse.urlencode({
  229.                                 "api_key": API_KEY,
  230.                                 "start": fight["start_time"],
  231.                                 "end": fight["end_time"],
  232.                                 "filter": "IN RANGE FROM encounterPhase = 5 AND type = 'begincast' AND ability.id = 11147 TO ability.id = 1000201 END"
  233.                             }),
  234.                         functools.partial(self.uwuDamageDone, fight)
  235.                     )
  236.             else:
  237.                 lastWokenEvent = rfindAbility(events, {WOKEN_DEBUFF})
  238.                 lastWokenTarget = fight["report"]["actors"][lastWokenEvent["targetID"]]["name"] if lastWokenEvent else None
  239.  
  240.                 if fight["lastPhaseForPercentageDisplay"] == 1:
  241.                     phase = "Garuda (woken)" if lastWokenTarget == "Garuda" else "Garuda (not woken)"
  242.                 elif fight["lastPhaseForPercentageDisplay"] == 2:
  243.                     phase = "Ifrit (woken)" if lastWokenTarget == "Ifrit" else "Ifrit (not woken)"
  244.                 elif fight["lastPhaseForPercentageDisplay"] == 3:
  245.                     phase = "Titan (woken)" if lastWokenTarget == "Titan" else "Titan (not woken)"
  246.  
  247.             if fight["lastPhaseForPercentageDisplay"] >= 3:
  248.                 isTitanWoken = lastWokenTarget == "Titan" or fight["lastPhaseForPercentageDisplay"] == 5
  249.                 self.titanGaolEvents(fight, events.tolist(), isTitanWoken)
  250.  
  251.             assert(phase != None)
  252.             self.fightData.setdefault(fight["reportId"], []).append((fight, phase))
  253.  
  254.     def uwuDamageDone(self, fight, response):
  255.         data = json.loads(response.body_as_unicode())
  256.  
  257.         activeTime = max(e["activeTime"] for e in data["entries"]) / 1000
  258.         print("\nFight {}:{} Ultima -> Stun DPS".format(fight["reportId"], fight["id"]))
  259.         for entry in data["entries"]:
  260.             if entry["type"] == "LimitBreak" and entry["name"] != "Limit Break":
  261.                 continue
  262.             self.uwuRoleDps.setdefault(entry["type"], []).append(entry["total"] / activeTime)
  263.             print("{}: {:.0f} dps".format(entry["type"], entry["total"] / activeTime))
  264.         print("")
  265.  
  266.     def titanGaolEvents(self, fight, allEvents, isWoken):
  267.         start = -1
  268.         end = -1
  269.         gaols = 0
  270.         for e, event in enumerate(allEvents):
  271.             if "ability" not in event:
  272.                 continue
  273.             if start == -1 and event["ability"]["guid"] in {11115, 11116}:
  274.                 start = e
  275.             if gaols < 3 and event["ability"]["guid"] == 1000292:
  276.                 gaols += 1
  277.                 end = e + 1
  278.         if start == -1 or end < start + 6:
  279.             return
  280.         events = allEvents[start:end]
  281.  
  282.         targeted = set()
  283.         for event in events[:3]:
  284.             if "ability" not in event or event["ability"]["guid"] not in {11115, 11116}:
  285.                 return
  286.             targeted.add(event["targetID"])
  287.  
  288.         deaths = set()
  289.         for event in events[3:-3]:
  290.             assert(event["type"] == "death")
  291.             deaths.add(event["targetID"])
  292.         for event in reversed(events[:start]):
  293.             if event["timestamp"] + 5000 < allEvents[start]["timestamp"]:
  294.                 break
  295.             if event["type"] == "death":
  296.                 deaths.add(event["targetID"])
  297.  
  298.         gaoled = set()
  299.         for event in events[-3:]:
  300.             if "ability" not in event or event["ability"]["guid"] != 1000292:
  301.                 return
  302.             gaoled.add(event["targetID"])
  303.  
  304.         self.titanGaolData["deaths"].update(map(lambda id: fight["report"]["actors"][id]["name"], deaths))
  305.         if targeted == gaoled:
  306.             if isWoken:
  307.                 self.titanGaolData["success"].update(map(lambda id: fight["report"]["actors"][id]["name"], targeted))
  308.             else:
  309.                 self.titanGaolData["failPositioning"].update(map(lambda id: fight["report"]["actors"][id]["name"], targeted))
  310.                 self.titanGaolData["failPositioningCount"] += 1
  311.         else:
  312.             self.titanGaolData["failDeathCount"] += 1
  313.  
  314.     def teaEvents(self, fights, allEvents):
  315.         for fight in fights:
  316.             events = allEvents.loc[fight["start_time"]:fight["end_time"]]
  317.  
  318.             if len(events) == 0:
  319.                 # Before Nisi
  320.                 self.fightData.setdefault(fight["reportId"], []).append((fight, "Limit Cut"))
  321.             elif events.iloc[-1]["ability"]["guid"] == 18494: # Judgement Nisi
  322.                 self.fightData.setdefault(fight["reportId"], []).append((fight, "CC + BJ"))
  323.             elif events.iloc[-1]["ability"]["guid"] == 18522: # Temporal Stasis
  324.                 self.fightData.setdefault(fight["reportId"], []).append((fight, "Time Stop + Inception"))
  325.             elif events.iloc[-1]["ability"]["guid"] == 18542: # Wormhole
  326.                 self.fightData.setdefault(fight["reportId"], []).append((fight, "Wormhole"))
  327.             else:
  328.                 assert(False)
  329.  
  330.     def closed(self, reason):
  331.         MSPERHOUR = 1000 * 60 * 60
  332.  
  333.         totalDuration = 0
  334.         phaseCounts = dict.fromkeys(PHASES, 0)
  335.         phaseDurations = dict.fromkeys(PHASES, 0)
  336.         phaseIntervals = {}
  337.         weekStarts = [0]
  338.         lastWeek = None
  339.         clears = []
  340.         for report in REPORTS:
  341.             for fight, phase in sorted(self.fightData[report], key = lambda e: e[0]["start_time"]):
  342.                 duration = fight["end_time"] - fight["start_time"]
  343.                 phaseCounts[phase] += 1
  344.                 phaseDurations[phase] += duration
  345.                 phaseIntervals.setdefault(phase, []).append((totalDuration, totalDuration + duration))
  346.  
  347.                 if lastWeek is None:
  348.                     lastWeek = fight["report"]["start"] + fight["start_time"]
  349.                 if lastWeek + MSPERHOUR * 24 * 7 < fight["report"]["start"] + fight["end_time"]:
  350.                     weekStarts.append(totalDuration)
  351.                     while lastWeek + MSPERHOUR * 24 * 7 < fight["report"]["start"] + fight["end_time"]:
  352.                         lastWeek += MSPERHOUR * 24 * 7
  353.  
  354.                 isClear = "kill" in fight and fight["kill"]
  355.                 if isClear:
  356.                     clears.append(totalDuration + duration)
  357.  
  358.                 totalDuration += duration
  359.  
  360.                 print("{}:{}: {}{}".format(fight["reportId"], fight["id"], phase, " (Clear)" if isClear else ""))
  361.         print("")
  362.         print("legends = {{{}}};".format(
  363.             ",".join(map(
  364.                 lambda p: "\"{}\"".format(p),
  365.                 filter(lambda p: p in phaseIntervals, PHASES)
  366.             ))
  367.         ))
  368.         print("data = {{{}}};".format(
  369.             ",".join(map(
  370.                 lambda p: "{{{}}}".format(
  371.                     ",".join(map(lambda i: "{{{:.2f}, {:.2f}}}".format(i[0] / MSPERHOUR, i[1] / MSPERHOUR), phaseIntervals[p]))
  372.                 ),
  373.                 filter(lambda p: p in phaseIntervals, PHASES)
  374.             ))
  375.         ))
  376.         print("gridLines = {{{{{}}}, None}};".format(
  377.             ",".join(itertools.chain(
  378.                 map(lambda c: "{{{:.2f}, Directive[Red, Thick]}}".format(c / MSPERHOUR), clears),
  379.                 map(lambda w: "{:.2f}".format(w / MSPERHOUR), weekStarts)
  380.             ))
  381.         ))
  382.         print("""
  383. NumberLinePlot[
  384.    Interval @@ # & /@ data, PlotLegends -> legends,
  385.    PlotStyle -> Directive[Thickness[0.01], CapForm[None]],
  386.    AspectRatio -> 1 / 5, ImageSize -> 1000
  387. ] /. Point[a_] -> Point[{-100, 1}]""")
  388.         print("""
  389. filling = Join[{{1 -> Axis}}, Table[{n -> {n - 1}}, {n, 2, Length[data]}]];
  390. windowSize = 4;
  391. smooth[fn_] := MovingAverage[Table[If[fn, 1, 0], {x, 0 - windowSize/2, Max[data] + windowSize/2, 0.01}], 100 windowSize];
  392. envelope = smooth[0 <= x < Max[data]];
  393. ParallelMap[smooth[Or @@ (#[[1]] <= x < #[[2]] & /@ #)] / envelope &, data];
  394. ListLinePlot[
  395.    Accumulate[%], Filling -> filling, DataRange -> {0, Max[data]},
  396.    PlotRange -> All, PlotLegends -> legends,
  397.    GridLines -> gridLines, GridLinesStyle -> Dashed,
  398.    PlotLabel -> "Time spent in pull wiping to phase x",
  399.    AxesLabel -> {"Hours in pull", "Ratio"},
  400.    ImageSize -> 1000
  401. ]""")
  402.  
  403.         print("")
  404.         for phase in PHASES:
  405.             print("{}: {} pulls, {:.1f}% in duration".format(phase, phaseCounts[phase], phaseDurations[phase] / totalDuration * 100))
  406.  
  407.         if len(self.ucobRoleDps) > 0:
  408.             print("")
  409.             print("{{{}}}".format(",".join("plot[{0}, \"{0}\", {{{1}}}]".format(role, ",".join(map(str, dps))) for role, dps in self.ucobRoleDps.items())))
  410.  
  411.         if len(self.uwuRoleDps) > 0:
  412.             print("")
  413.             print("{{{}}}".format(",".join("plot[{0}, \"{0}\", {{{1}}}]".format(role, ",".join(map(str, dps))) for role, dps in self.uwuRoleDps.items())))
  414.  
  415.         if len(self.titanGaolData["deaths"]) > 0:
  416.             print("\nTitan Knockback Deaths")
  417.             print("\n".join(map(lambda e: "{}: {}".format(e[0], e[1]), self.titanGaolData["deaths"].most_common())))
  418.         if len(self.titanGaolData["failPositioning"]) > 0:
  419.             failureRate = collections.Counter()
  420.             for id, failCount in self.titanGaolData["failPositioning"].items():
  421.                 successCount = self.titanGaolData["success"][id]
  422.                 failureRate[id] = failCount / (failCount + successCount)
  423.  
  424.             print("\nTitan Gaol Positioning Failure Rate")
  425.             print("\n".join(map(lambda e: "{}: {:.3f}".format(e[0], e[1]), failureRate.most_common())))
  426.         totalFailCount = self.titanGaolData["failDeathCount"] + self.titanGaolData["failPositioningCount"]
  427.         if totalFailCount > 0:
  428.             print("\nTitan Non-woken Reason:")
  429.             print("Knockback Death: {:.3f}".format(self.titanGaolData["failDeathCount"] / totalFailCount))
  430.             print("Gaol Positioning: {:.3f}".format(self.titanGaolData["failPositioningCount"] / totalFailCount))
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement
Public Pastes
Advertisement
We use cookies for various purposes including analytics. By continuing to use Pastebin, you agree to our use of cookies as described in the Cookies Policy.  OK, I Understand
Not a member of Pastebin yet?
Sign Up, it unlocks many cool features!