LSC graph

import string
import re
import numpy as np
import matplotlib.pyplot as plt
import codecs

print "---- LSC log parser ----"

## input

input_files = [
    #"input/log_01.txt","input/log_02.txt","input/log_03.txt",
    #"input/log_04.txt","input/log_05.txt",
    #"input/log_06.txt","input/log_07.txt",
    "input/log_08.txt","input/log_09.txt","input/log_10.txt",
    "input/log_11.txt","input/log_12.txt","input/log_13.txt",
    "input/log_14.txt","input/log_15.txt","input/log_16.txt",
    "input/log_17.txt","input/log_18.txt",
    "input/log_19.txt",
    "input/log_20.txt","input/log_21.txt",
    "input/actual.txt"
    ]

ship_plot_groups = [
     {'jap':[u'大鳳',u'大凰'], 'eng':'Taihou', 'color':'#acd8bd'}
    ,{'jap':u'大和', 'eng':'Yamato', 'color':'#b66572'}
    ,{'jap':[u'あきつ丸',u'あきつ'], 'eng':'Akitsumaru', 'color':'#c7c6c7'}
    ,{'jap':[u'まるゆ',u'まゆる'], 'eng':'Maruyu', 'color':'#fcf1d6'}

    ,{'jap':u'矢矧', 'eng':'Yahagi', 'color':'#5b6268'},{'jap':u'阿賀野', 'eng':'Agano', 'color':'#aa2845'},{'jap':u'能代', 'eng':'Noshiro', 'color':'#6a4143'}
    #,{'jap':[u'矢矧',u'阿賀野',u'能代'], 'eng':'Agano class', 'color':'#aa2845'}

    ,{'jap':u'三隈', 'eng':'Mikuma', 'color':'#86362d'},{'jap':u'鈴谷', 'eng':'Suzuya', 'color':'#b3cbc4'},{'jap':u'熊野', 'eng':'Kumano', 'color':'#d2c1aa'},{'jap':u'最上', 'eng':'Mogami', 'color':'#666666'}
    #,{'jap':[u'三隈',u'鈴谷',u'熊野',u'最上'], 'eng':'Mogami class', 'color':'#86362d'}

    ,{'jap':u'長門', 'eng':'Nagato', 'color':'#605044'},{'jap':[u'陸奥',u'陸奧'], 'eng':'Mutsu', 'color':'#b5916c'}
    #,{'jap':[u'長門',u'陸奥',u'陸奧'], 'eng':'Nagato class', 'color':'#b5916c'}

    #,{'jap':u'赤城', 'eng':'Akagi', 'color':'#666666'}
    ,{'jap':u'加賀', 'eng':'Kaga', 'color':'#473f8a'}

    #,{'jap':[u'赤城',u'加賀'], 'eng':'1kou', 'color':'#473f8a'}

    ,{'jap':u'翔鶴', 'eng':'Shoukaku', 'color':'#dee8ed'}
    ,{'jap':u'瑞鶴', 'eng':'Zuikaku', 'color':'#99948e'}

    #,{'jap':[u'翔鶴',u'瑞鶴'], 'eng':'5kou', 'color':'#dee8ed'}

    #,{'jap':u'蒼龍', 'eng':'Souryuu', 'color':'#556959'}
    #,{'jap':u'飛龍', 'eng':'Hiryuu', 'color':'#555969'}

    #,{'jap':[u'蒼龍',u'飛龍'], 'eng':'2kou', 'color':'#666666'}

    ,{'jap':[u'赤城',u'蒼龍',u'飛龍'], 'eng':'Other CV', 'color':'#667766'}

    ,{'jap':[u'瑞鳳',u'飛鷹',u'隼鷹',u'準鷹',u'祥鳳',u'龍驤',u'翔鳳',u'龍嬢',u'出雲丸',u'龍穣'], 'eng':'CVL', 'color':'#776666'}

    ,{'jap':[u'金剛',u'比叡',u'榛名',u'霧島'], 'eng':'Kongou class', 'color':'#595555'}

    #,{'jap':[u'扶桑',u'山城'], 'eng':'Fusou class', 'color':'#666666'}
    #,{'jap':[u'伊勢',u'日向'], 'eng':'Ise class', 'color':'#666666'}
    ,{'jap':[u'扶桑',u'山城',u'伊勢',u'日向'], 'eng':'BBV', 'color':'#555955'}

    #,{'jap':[u'金剛',u'比叡',u'榛名',u'霧島',u'扶桑',u'山城',u'伊勢',u'日向'], 'eng':'Other BB', 'color':'#666677'}

    ,{'jap':[u'利根',u'筑摩'], 'eng':'Tone class', 'color':'#444444'}
    ]

all_ship = [
    u"長門",u"陸奥",u"伊勢",u"日向",u"雪風",u"赤城",u"加賀",u"蒼龍",u"飛龍",u"島風"
   ,u"吹雪",u"白雪",u"初雪",u"深雪",u"叢雲",u"磯波",u"綾波",u"敷波",u"大井",u"北上"
   ,u"金剛",u"比叡",u"榛名",u"霧島",u"鳳翔",u"扶桑",u"山城",u"天龍",u"龍田",u"龍驤"
   ,u"睦月",u"如月",u"皐月",u"文月",u"長月",u"菊月",u"三日月",u"望月",u"球磨",u"多摩"
   ,u"木曾",u"長良",u"五十鈴",u"名取",u"由良",u"川内",u"神通",u"那珂",u"千歳",u"千代田"
   ,u"最上",u"古鷹",u"加古",u"青葉",u"妙高",u"那智",u"足柄",u"羽黒",u"高雄",u"愛宕"
   ,u"摩耶",u"鳥海",u"利根",u"筑摩",u"飛鷹",u"隼鷹",u"朧",u"曙",u"漣",u"潮"
   ,u"暁",u"響",u"雷",u"電",u"初春",u"子日",u"若葉",u"初霜",u"白露",u"時雨"
   ,u"村雨",u"夕立",u"五月雨",u"涼風",u"朝潮",u"大潮",u"満潮",u"荒潮",u"霰",u"霞"
   ,u"陽炎",u"不知火",u"黒潮",u"祥鳳"
   ,u"翔鶴",u"瑞鶴",u"鬼怒"
   ,u"阿武隈",u"夕張",u"瑞鳳",u"三隈",u"初風",u"舞風"
   ,u"衣笠",u"伊19",u"鈴谷",u"熊野",u"伊168",u"伊58",u"伊8"
   ,u"大和",u"秋雲",u"夕雲",u"巻雲",u"長波",u"阿賀野",u"能代",u"矢矧"
   ,u"武蔵"
   ,u"大鳳",u"伊401",u"あきつ丸",u"まるゆ"
   # name variants
   ,u'大凰',u'あきつ',u'まゆる',u'準鷹',u'陸奧',u'翔鳳', u'龍嬢',u'龍穣',u'出雲丸'
   ]

## plotting config

#Taihou min 3000,2000,2000,3000
#CV only (4000,1500,2000,5000,1)

#Yamato min 3000,2000,3500,1000
#No Yamato 4000,1500,2000,5000

min_resources = (1500,1500,2000,1000,1)
max_resources = (7000,7000,7000,7000,100)

min_samples = 100
legend = 1

label_dist = -0.3
label_min_chance = 2.5
label_name_min_chance = 6.0
label_short_name_min_chance = 5.0
short_name_length = 8

x_max = 100
bar_y_start_pos = 0
bar_spacing = 1.25
bar_height = 1

yellow_rate = 0.25

figure_size = (16, 6)

font = 'Segoe UI'
font_size = 'xx-small'

legend_columns = 12

subplots_adjustment = {'left':0.16, 'right':0.99, 'bottom':0.11, 'top':0.94}

lsc_regex = u".*\D+(\d{4})\D*(\d{4})\D*(\d{4})\D*(\d{4}).*[開発資材財源料費」]+\s*[：:・]?\s*(\d+).*結果\s*[：:・]?\s*([^-]*) --"

## functions

def fix_rec(rec):
    m = {u"０":u"0", u"１":u"1", u"２":u"2", u"３":u"3", u"４":u"4", u"５":u"5", u"６":u"6", u"７":u"7", u"８":u"8", u"９":u"9"}
    for k,v in m.items():
        rec = rec.replace(k,v)
    return rec

def parse_ships(ships_str):
    res = re.findall("[\w]+", ships_str, re.UNICODE)
    filtered_res = [s for s in res if s in all_ship]
    #bad_res = [s for s in res if s not in all_ship]
    #for s in bad_res:
    #    print "bad ship report: " + s
    return filtered_res

def cmp_rec(a, b):
    return a[0] < b[0] or a[1] < b[1] or a[2] < b[2] or a[3] < b[3] or a[4] < b[4]

## main

all_possible = []
for ship in ship_plot_groups:
    s = ship['jap']
    if isinstance(s, list):
        all_possible.extend(s)
    else:
        all_possible.append(s)

print "---- Reading - start ----"

all_input = ""
for file in input_files:
    print "Reading - " + file
    f = codecs.open(file, 'r', encoding='utf-8')
    all_input += f.read()

print "---- Reading - end ----"

print "---- Parsing - start ----"

all_input_splitted = string.split(all_input, "\n")
data = {}
total = 0
bad_count = 0
times = []
for line in all_input_splitted:
    time_match = re.match(".*(\d{4}-\d{2}-\d{2}).*(\d{2}:\d{2}:\d{2})", line, re.UNICODE)
    if time_match is not None:
        time = time_match.group(1) + " " + time_match.group(2)
        times.append(time)

    m = re.match(lsc_regex, line, re.UNICODE)
    if m is None:
        if line.find(u"結果") != -1:
            #print "no match: " + line
            bad_count += 1
        continue
    rec = m.group(1) + "/" + m.group(2) + "/" + m.group(3) + "/" + m.group(4) + "/" + m.group(5)
    rec = fix_rec(rec)
    rec_val = (int(fix_rec(m.group(1))), int(fix_rec(m.group(2))), int(fix_rec(m.group(3))), int(fix_rec(m.group(4))), int(fix_rec(m.group(5))))

    if cmp_rec(rec_val,min_resources) or cmp_rec(max_resources,rec_val):
        continue

    ship_str = m.group(6)
    rec_ships = parse_ships(ship_str)
    for ship in rec_ships:
        #if ship == u"大和" and rec_val[3] > rec_val[1]:
        #    print rec + u" 大和"
        total += 1
        if rec not in data:
            data[rec] = {'total':0,'rec_val':rec_val}
        data[rec]['total'] += 1
        if ship not in data[rec]:
            data[rec][ship] = 0
        data[rec][ship] += 1

print "---- Parsing - end ----"

total_stat_str = "Reports: " + str(total) + " (" + ("%.1lf"%(100*float(total) / float(total + bad_count))) + "%) - Unparsable: " + str(bad_count) + " (" + ("%.1lf"%(100*(float(bad_count) / float(total + bad_count)))) + "%)"
print total_stat_str
times_sorted = sorted(times)
time_span_str = times_sorted[0] + " - " + times_sorted[len(times_sorted)-1]
print "Time span: " + time_span_str

print "---- Generating - start ----"

## ordering
to_order = {}
max_count = 0
for rec, value in data.items():
    if value['total'] > max_count:
        max_count = value['total']
    order_key = value['total'] #value['rec_val'][3]
    if order_key not in to_order:
        to_order[order_key] = []
    to_order[order_key].append({'rec':rec, 'val':value})
ordered = sorted(to_order)

recs = []
chances = {}
colors = []
for key in ordered:
    for data in to_order[key]:
        rec = data['rec']
        value = data['val']
        rec_str = rec + u" (" + str(value['total']) + u")"

        if value['total'] < min_samples:
            #print u"too few data: " + rec_str
            continue

        for k in value.keys():
            if k not in ['total','rec_val'] and k not in all_possible:
                print "bad ship: " + k + " cnt: " + str(value[k]) + " rec: " + rec

        rec_rate = float(value['total']) / float(max_count)
        green_rate = 0.0
        red_rate = 0.0
        if rec_rate < yellow_rate:
            green_rate = rec_rate / yellow_rate
            red_rate = 1.0
        else:
            green_rate = 1.0
            red_rate = (1.0 - rec_rate) / (1.0 - yellow_rate)
        color = '#' + ('%02x'%(red_rate*225)) + ('%02x'%(green_rate*225)) + '00'

        for i in range(0,len(ship_plot_groups)):
            ship_names = ship_plot_groups[i]['jap']
            if not isinstance(ship_names, list):
                ship_names = [ship_names]
            chance = 0.0
            for ship_name in ship_names:
                if ship_name not in value:
                    value[ship_name] = 0
                    #print u"no " + ship_name + " with: " + rec_str
                #print u"" + ship_name + " get: " + rec_str
                chance += float(value[ship_name]) / float(value['total']) * 100
            if i not in chances:
                chances[i] = []
            chances[i].append(chance)

        recs.append(rec_str)
        colors.append(color)

print "---- Generating - end ----"
print "---- Plotting - start ----"

plt.rc('font', family=font)

y_pos = np.arange(len(recs)) * bar_spacing + bar_y_start_pos
fig, ax1 = plt.subplots(figsize=figure_size)

bars = []
prev_chance = []
for v in chances.values()[0]:
    prev_chance.append(0)

for i, chance in chances.items():
    rects = ax1.barh(y_pos, chance, align='center', alpha=0.8, color=ship_plot_groups[i]['color'], left=prev_chance, height=bar_height)
    for j in range(0,len(chance)):
        prev_chance[j] += chance[j]
        if chance[j] > label_min_chance:
            label = ('%.1lf'%chance[j]) + '%'
            if chance[j] > label_name_min_chance or (chance[j] > label_short_name_min_chance and len(ship_plot_groups[i]['eng']) < short_name_length):
                label = ship_plot_groups[i]['eng'] + " " + label
            ax1.text(label_dist + prev_chance[j], rects[j].get_y()+rects[j].get_height()/2., label, ha='right', va='center', fontsize=font_size)
    bars.append(rects[0])

ax1.set_xticklabels([])

plt.yticks(y_pos, recs, fontsize='small')
for i in range(0,len(colors)):
    ticks = ax1.get_yticklabels()
    ticks[i].set_color(colors[i])
plt.tick_params(axis='y',which='both',left='off',right='off')
plt.tick_params(axis='x',which='both',top='off',bottom='off')

titles = []
for v in ship_plot_groups:
    titles.append(v['eng'])

plt.xlim(0, x_max)
plt.ylim(-1,y_pos[len(y_pos)-1]+1)

fig.canvas.set_window_title('LSC')
plt.title("LSC report (" + time_span_str + ") - Total: " + str(total))
if legend != 0:
    plt.legend(bars, titles, fontsize='x-small', loc='lower center', mode="expand", ncol=legend_columns, bbox_to_anchor=(0, -0.1, 1, 0))
plt.subplots_adjust(left=subplots_adjustment['left'], right=subplots_adjustment['right'], bottom=subplots_adjustment['bottom'], top=subplots_adjustment['top'])

print "---- Plotting - end ----"

print "---- Display - start ----"

plt.show()

print "---- Display - end ----"

print "Bye."