bevstat.py

#!/bin/python3.4
import numpy as np
from os import path
from bokeh.io import curdoc
from bokeh.layouts import row, column, widgetbox, layout
from bokeh.models import ColumnDataSource, formatters, BoxAnnotation, BoxSelectTool, HoverTool, Span, Label
from bokeh.models.widgets import Slider, PreText, RadioGroup
from bokeh.plotting import figure
from functools import lru_cache


data_dir = "/home/user/bokeh/bev_daten/"

age_data_m = np.loadtxt(path.join(data_dir,"bev_daten_ch_m.csv"), dtype=int)
age_data_f = np.loadtxt(path.join(data_dir,"bev_daten_ch_f.csv"), dtype=int)
age_data_ma = np.loadtxt(path.join(data_dir,"bev_daten_alle_m.csv"), dtype=int)
age_data_fa = np.loadtxt(path.join(data_dir,"bev_daten_alle_f.csv"), dtype=int)

prediction_data_low_m_ch = np.loadtxt(path.join(data_dir,"low_prediction_m_ch.csv"), dtype=int)
prediction_data_low_f_ch = np.loadtxt(path.join(data_dir,"low_prediction_f_ch.csv"), dtype=int)
prediction_data_low_m_au = np.loadtxt(path.join(data_dir,"low_prediction_m_au.csv"), dtype=int)
prediction_data_low_f_au = np.loadtxt(path.join(data_dir,"low_prediction_f_au.csv"), dtype=int)

prediction_data_reference_m_ch = np.loadtxt(path.join(data_dir,"reference_prediction_m_ch.csv"), dtype=int)
prediction_data_reference_f_ch = np.loadtxt(path.join(data_dir,"reference_prediction_f_ch.csv"), dtype=int)
prediction_data_reference_m_au = np.loadtxt(path.join(data_dir,"reference_prediction_m_au.csv"), dtype=int)
prediction_data_reference_f_au = np.loadtxt(path.join(data_dir,"reference_prediction_f_au.csv"), dtype=int)

prediction_data_high_m_ch = np.loadtxt(path.join(data_dir,"high_prediction_m_ch.csv"), dtype=int)
prediction_data_high_f_ch = np.loadtxt(path.join(data_dir,"high_prediction_f_ch.csv"), dtype=int)
prediction_data_high_m_au = np.loadtxt(path.join(data_dir,"high_prediction_m_au.csv"), dtype=int)
prediction_data_high_f_au = np.loadtxt(path.join(data_dir,"high_prediction_f_au.csv"), dtype=int)

additional_stats = {}
additional_source = {}

for key in ["historical_ch","historical_au","low_ch","low_au","ref_ch","ref_au","high_ch","high_au"]:
    additional_stats[key] = np.rot90(np.fliplr(np.loadtxt(path.join(data_dir,key + ".stats"), dtype=int)))
    additional_stats[key][2] *= -1
    additional_stats[key][4] *= -1
    #x_axis_offset = 0.75 if "au" in key else 0.25
    #additional_stats[key][0] += x_axis_offset



age_data_ma = age_data_ma - age_data_m
age_data_fa = age_data_fa - age_data_f


#since were working without a stacked bar:
age_data_m = np.loadtxt(path.join(data_dir,"bev_daten_alle_m.csv"), dtype=int)
age_data_f = np.loadtxt(path.join(data_dir,"bev_daten_alle_f.csv"), dtype=int)

age_data_f *= -1
age_data_fa *= -1
prediction_data_low_f_ch *= -1
prediction_data_low_f_au *= -1
prediction_data_reference_f_ch *= -1
prediction_data_reference_f_au *= -1
prediction_data_high_f_ch *= -1
prediction_data_high_f_au *= -1

#load initial graph data
display_data_m = age_data_m[0]
display_data_ma = age_data_ma[0]
display_data_f = age_data_f[0]
display_data_fa = age_data_fa[0]

display_stats_ch = np.hstack((additional_stats["historical_ch"],additional_stats["ref_ch"]))
display_stats_au = np.hstack((additional_stats["historical_au"],additional_stats["ref_au"]))

# print(len(display_stats_au[0]),len(display_stats_ch[0]))
# exit()

y = [k for k in range(101)]
x_scatter = np.zeros(101)
laborage_min = 18
laborage_max = 67
historical_years = len(age_data_m)-1



source_m = ColumnDataSource(data=dict(y=y, display=display_data_m))
source_ma = ColumnDataSource(data=dict(y=y, display=display_data_ma))
source_f = ColumnDataSource(data=dict(y=y, display=display_data_f))
source_fa = ColumnDataSource(data=dict(y=y, display=display_data_fa))

source_ch_stats = ColumnDataSource(data=dict(years=display_stats_ch[0]+0.25,
                                             births=display_stats_ch[1],
                                             deaths=display_stats_ch[2],
                                             immigration=display_stats_ch[3],
                                             migration=display_stats_ch[4]))

source_au_stats = ColumnDataSource(data=dict(years=display_stats_au[0]+0.75,
                                             births=display_stats_au[1],
                                             deaths=display_stats_au[2],
                                             immigration=display_stats_au[3],
                                             migration=display_stats_au[4]))

dependency_ratio_textfield = PreText(text="", width=500)
total_population_textfield = PreText(text="", width=500)

hovertool_births = HoverTool(tooltips=[("Geburtenüberschuss", "@y"),("Jahr","@x")])
hovertool_migration = HoverTool(tooltips=[("Wanderungssaldo", "@y"),("Jahr","@x")])

plot = figure(plot_height=400, plot_width=600, title="Ständige Wohnbevölkerung Schweiz: 2010",
              tools=["save", "box_select"],
              x_range=[-85000, 85000],
              y_range=[0, 101])

plot_birth = figure(plot_height=400, plot_width=600, title="Geburtenüberschuss",
              tools=[hovertool_births],
              x_range=[display_stats_ch[0][0],display_stats_ch[0][-1]],
              y_range=[-100000, 100000])

plot_migration = figure(plot_height=400, plot_width=600, title="Wanderungssaldo",
              tools=[hovertool_migration],
              x_range=[display_stats_ch[0][0],display_stats_ch[0][-1]],
              y_range=[-180000, 180000])


m_plot = plot.hbar(right='display', y='y', source=source_m, line_width=6, line_alpha=0.5, color="blue", legend="Schweizer")
ma_plot = plot.hbar(right='display', y='y', source=source_ma, line_width=6, line_alpha=0.5, color="red", legend="Ausländer")
f_plot = plot.hbar(right='display', y='y', source=source_f, line_width=6, line_alpha=0.5, color="blue")
fa_plot = plot.hbar(right='display', y='y', source=source_fa, line_width=6, line_alpha=0.5, color="red")


plot_birth.vbar(x='years',
                     top='births',
                     source=source_ch_stats,
                     bottom=0,
                     width=0.5,
                     alpha=0.1,
                     color="blue")
plot_birth.vbar(x='years',
                     top='births',
                     source=source_au_stats,
                     bottom=0,
                     width=0.5,
                     alpha=0.1,
                     color="red")
plot_birth.vbar(x='years',
                     top='deaths',
                     source=source_ch_stats,
                     bottom=0,
                     width=0.5,
                     alpha=0.1,
                     color="blue")
plot_birth.vbar(x='years',
                     top='deaths',
                     source=source_au_stats,
                     bottom=0,
                     width=0.5,
                     alpha=0.1,
                     color="red")

a = np.array(display_stats_ch[0])

glyph_birth_ch = plot_birth.line(x=a,
                     y=display_stats_ch[1]+display_stats_ch[2],
                     line_width=4,
                     color="blue",
                     legend="Geburtenüberschuss (Schweizer)")

glyph_birth_au = plot_birth.line(x=a,
                     y=display_stats_au[1]+display_stats_au[2],
                     line_width=4,
                     color="red",
                     legend="Geburtenüberschuss (Ausländer)")

hovertool_births.renderers.append(glyph_birth_ch)
hovertool_births.renderers.append(glyph_birth_au)


plot_migration.vbar(x='years',
                     top='migration',
                     source=source_ch_stats,
                     bottom=0,
                     width=0.5,
                     alpha=0.1,
                     color="blue")

plot_migration.vbar(x='years',
                     top='migration',
                     source=source_au_stats,
                     bottom=0,
                     width=0.5,
                     alpha=0.1,
                     color="red")

plot_migration.vbar(x='years',
                     top='immigration',
                     source=source_ch_stats,
                     bottom=0,
                     width=0.5,
                     alpha=0.1,
                     color="blue")

plot_migration.vbar(x='years',
                     top='immigration',
                     source=source_au_stats,
                     bottom=0,
                     width=0.5,
                     alpha=0.1,
                     color="red")

#plot doesn't seem to work without this
a = np.array(display_stats_ch[0])

glyph_migration_ch = plot_migration.line(x=a,
                     y=display_stats_ch[3]+display_stats_ch[4],
                     line_width=4,
                     color="blue",
                     legend="Migrationssaldo (Schweizer)")

glyph_migration_au = plot_migration.line(x=a,
                     y=display_stats_au[3]+display_stats_au[4],
                     line_width=4,
                     color="red",
                     legend="Migrationssaldo (Ausländer)")

hovertool_migration.renderers.append(glyph_migration_ch)
hovertool_migration.renderers.append(glyph_migration_au)

#only scatterplots can be selectec with boxselect
#invisible scatter along x=0
scatter = plot.scatter(x=x_scatter, y=y, size=0, fill_color="red")

plot.legend.location = "top_right"
plot_birth.legend.location = "bottom_left"
plot_migration.legend.location = "bottom_left"

jugend_box = BoxAnnotation(top=laborage_min, fill_alpha=0.1, fill_color='red')
arbeiter_box = BoxAnnotation(bottom=laborage_min, top=laborage_max, fill_alpha=0.1, fill_color='green')
rentner_box = BoxAnnotation(bottom=laborage_max, fill_alpha=0.1, fill_color='red')
current_year_births_box = BoxAnnotation(left=1971, right=1972, fill_alpha=0.2, fill_color='yellow')
current_year_migration_box = BoxAnnotation(left=1971, right=1972, fill_alpha=0.2, fill_color='yellow')

m_f_separator = Span(location=0, dimension='height', line_dash='dashed', line_color='black', line_width=1)

annotation_female = Label(x=85, y=35, x_units='screen', y_units='screen',
                 text='Weiblich', render_mode='css',
                 border_line_color='black', border_line_alpha=0.4,
                 background_fill_color='white', background_fill_alpha=0.7)

annotation_male = Label(x=495, y=35, x_units='screen', y_units='screen',
                 text='Männlich', render_mode='css',
                 border_line_color='black', border_line_alpha=0.4,
                 background_fill_color='white', background_fill_alpha=0.7)

plot.yaxis.axis_label = "Alter"
plot.xaxis.formatter = formatters.NumeralTickFormatter(format="(0,0)")

plot_birth.yaxis.axis_label = "Todesfälle (-)  Geburten (+)"
plot_birth.yaxis.formatter = formatters.PrintfTickFormatter(format="%d")
plot_birth.xgrid.minor_grid_line_alpha = 0.5

plot_migration.yaxis.axis_label = "Auswanderung (-)  Einwanderung (+)"
plot_migration.yaxis.formatter = formatters.PrintfTickFormatter(format="%d")
plot_migration.xgrid.minor_grid_line_alpha = 0.5

offset = Slider(title="", value=0, start=0, end=44+30, step=1,
                callback_throttle=15000000, callback_policy="throttle",
        sizing_mode="scale_height", orientation="horizontal")


prediction_radio_group = RadioGroup(
        labels=["Szenario: \"Tief\"", "Referenzszenario", "Szenario: \"Hoch\""], active=1)

def update_stat_plots(_):
    for stat, num in [('births',1),('deaths',2),('immigration',3),('migration',4)]:
        if prediction_radio_group.active == 0:
            source_ch_stats.data[stat] = np.hstack((additional_stats["historical_ch"][num], additional_stats["low_ch"][num]))
            source_au_stats.data[stat] = np.hstack((additional_stats["historical_au"][num], additional_stats["low_au"][num]))
        if prediction_radio_group.active == 1:
            source_ch_stats.data[stat] = np.hstack((additional_stats["historical_ch"][num], additional_stats["ref_ch"][num]))
            source_au_stats.data[stat] = np.hstack((additional_stats["historical_au"][num], additional_stats["ref_au"][num]))
        if prediction_radio_group.active == 2:
            source_ch_stats.data[stat] = np.hstack((additional_stats["historical_ch"][num], additional_stats["high_ch"][num]))
            source_au_stats.data[stat] = np.hstack((additional_stats["historical_au"][num], additional_stats["high_au"][num]))

        update_data(None, None, offset.value)

def update_dependency_box(attr, old, new):
    global laborage_max, laborage_min
    laborage_max = max(new['1d']['indices'])
    laborage_min = min(new['1d']['indices'])
    jugend_box.update(top=laborage_min)
    arbeiter_box.update(bottom=laborage_min, top=laborage_max)
    rentner_box.update(bottom=laborage_max)
    update_dependency_text()

def update_dependency_text():
    display_data_total = source_m.data['display'] + (-1*source_f.data['display'])
    dependency_ratio_textfield.update(text="Abhängigenquotient: {dependency_ratio:.2f} \n"
                                           "\t- Jugendquotient: {dependency_ratio_minor:.2f} bei Eintrittsalter: {working_age}\n"
                                           "\t- Altersquotient: {dependency_ratio_major:.2f} bei Rentenalter: {retirement_age}".format(
        retirement_age=laborage_max,
        working_age=laborage_min,
        dependency_ratio=(sum(display_data_total[0:laborage_min])+sum(display_data_total[laborage_max:]))/sum(display_data_total[laborage_min:laborage_max]),
        dependency_ratio_minor=(sum(display_data_total[0:laborage_min]))/sum(display_data_total[laborage_min:]),
        dependency_ratio_major=(sum(display_data_total[laborage_max:]))/sum(display_data_total[:laborage_max])))

def update_population_text():
    source_m_sum = sum(source_m.data['display'])
    source_ma_sum = sum(source_ma.data['display'])
    source_f_sum = sum(source_f.data['display']) * -1
    source_fa_sum = sum(source_fa.data['display']) * -1

    total_population_textfield.update(text="Gesamtbevölkerung: {total_pop:,}\n"
                                           "\t- Männlich: {total_pop_m:,}\n"
                                           "\t\t- Schweizer: {total_pop_m_ch:,}\n"
                                           "\t\t- Ausländer: {total_pop_m_au:,}\n"
                                           "\t- Weiblich: {total_pop_f:,}\n"
                                           "\t\t- Schweizerinnen: {total_pop_f_ch:,}\n"
                                           "\t\t- Ausländerinnen: {total_pop_f_au:,}".format(
        total_pop=source_m_sum + source_f_sum,
        total_pop_m=source_m_sum,
        total_pop_m_ch=source_m_sum - source_ma_sum,
        total_pop_m_au=source_ma_sum,
        total_pop_f=source_f_sum,
        total_pop_f_ch=source_f_sum - source_fa_sum,
        total_pop_f_au=source_fa_sum))

def get_new_display_data(slider):
    if slider <= historical_years:
        return (age_data_m[slider],age_data_ma[slider],age_data_f[slider],age_data_fa[slider])
    elif slider > historical_years and prediction_radio_group.active == 0:
        slider -= historical_years+1
        return (prediction_data_low_m_ch[slider],prediction_data_low_m_au[slider],prediction_data_low_f_ch[slider],prediction_data_low_f_au[slider])
    elif slider > historical_years and prediction_radio_group.active == 1:
        slider -= historical_years+1
        return (prediction_data_reference_m_ch[slider],prediction_data_reference_m_au[slider],prediction_data_reference_f_ch[slider],prediction_data_reference_f_au[slider])
    elif slider > historical_years and prediction_radio_group.active == 2:
        slider -= historical_years+1
        return (prediction_data_high_m_ch[slider],prediction_data_high_m_au[slider],prediction_data_high_f_ch[slider],prediction_data_high_f_au[slider])

def update_current_year_box(slider):
    current_year_births_box.update(left=1971+slider-1,right=1971+slider)
    current_year_migration_box.update(left=1971+slider-1,right=1971+slider)

def update_data(attrname, old, new):
    source_m.data['display'], source_ma.data['display'], source_f.data['display'], source_fa.data['display'] = get_new_display_data(new)
    update_population_text()
    update_dependency_text()
    update_current_year_box(new)
    plot.title.text = "Ständige Wohnbevölkerung: {}".format(str(1971+new))


update_dependency_text()
update_population_text()

offset.on_change('value', update_data)
prediction_radio_group.on_click(update_stat_plots)
scatter.data_source.on_change('selected', update_dependency_box)

inputs = widgetbox(dependency_ratio_textfield, total_population_textfield, prediction_radio_group, width=600)

curdoc().add_root(column(row(plot, column(inputs,offset)), row(plot_birth, plot_migration)))
curdoc().title = "Wohnbevölkerung der Schweiz, Aufteilung nach Alter und Geschlecht"


for my_plot in [plot, plot_birth, plot_migration]:
    my_plot.toolbar.logo = None

for my_layout in [jugend_box, arbeiter_box, rentner_box, m_f_separator, annotation_male, annotation_female]:
    plot.add_layout(my_layout)

plot_birth.add_layout(current_year_births_box)
plot_migration.add_layout(current_year_migration_box)