!pip install --upgrade -q gspread
from tensorboardcolab import *
import shutil
#clean out the directory
shutil.rmtree('./Graph', ignore_errors=True)
os.mkdir('./Graph')
#tf.reset_default_graph()
#will start the tunneling and will print out a link:
tbc=TensorBoardColab()


!pip show tensorflow

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

# Dependencies
import os
import warnings
#from absl import flags
import matplotlib
import numpy as np
import tensorflow as tf
import tensorflow_probability as tfp
import math
import pandas as pd
tfd = tfp.distributions
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
#from hyperopt import fmin, tpe, hp, STATUS_OK, STATUS_FAIL, Trials
#import python_utils


# clear graph (if any) before running
tf.reset_default_graph()

####Delete all flags before declare#####

def del_all_flags(FLAGS):
    flags_dict = FLAGS._flags()
    keys_list = [keys for keys in flags_dict]
    for keys in keys_list:
        FLAGS.__delattr__(keys)

del_all_flags(tf.flags.FLAGS)

flags = tf.app.flags
FLAGS = tf.app.flags.FLAGS
flags.DEFINE_float("learning_rate", default = 0.0001, help = "Initial learning rate.")
flags.DEFINE_integer("epochs", default = 700, help = "Number of epochs to train for")
flags.DEFINE_integer("batch_size", default =128, help = "Batch size.")
flags.DEFINE_integer("eval_freq", default = 400, help =" Frequency at which to validate the model.")
flags.DEFINE_float("kernel_posterior_scale_mean", default = -0.9, help = "Initial kernel posterior mean of the scale (log var) for q(w)")
flags.DEFINE_float("kernel_posterior_scale_constraint", default = 0.2, help = "Posterior kernel constraint for the scale (log var) for q(w)")
flags.DEFINE_float("kl_annealing", default = 50, help = "Epochs to anneal the KL term (anneals from 0 to 1)")
flags.DEFINE_integer("num_hidden_layers", default = 4, help = "Number of hidden layers")
flags.DEFINE_integer("num_monte_carlo",

                     default=50,

                     help="Network draws to compute predictive probabilities.")
tf.app.flags.DEFINE_string('f', '', 'kernel')
#initialize flags
#FLAGS = flags.FLAGS
print(FLAGS.learning_rate)
print(FLAGS.epochs)
print(FLAGS.num_monte_carlo)

def build_input_pipeline(X_train,X_test,y_train, y_test, batch_size, valid_size):
  #Build an iterator over training batches
  training_dataset = tf.data.Dataset.from_tensor_slices((X_train, y_train))
  #Shuffle the dataset (note shuffle argument much larger than training size)
  # and form batches of size batch_size
  training_batches = training_dataset.shuffle(20000, reshuffle_each_iteration =True).repeat().batch(batch_size)
  training_iterator = tf.data.make_one_shot_iterator(training_batches)

  #Building iterator over the heldout set with batch_size = heldout_size,
  # i.e., return the entire heldout set as a constant.
  heldout_dataset = tf.data.Dataset.from_tensor_slices((X_test, y_test))
  heldout_batches = heldout_dataset.repeat().batch(valid_size)
  heldout_iterator = tf.data.make_one_shot_iterator(heldout_batches)

  #Combine these into a feasible iterator that can switch between training
  # and validation inputs.
  # Here should be minibatch increment be defined
  handle = tf.placeholder(tf.string, shape = [])
  feedable_iterator = tf.data.Iterator.from_string_handle(handle, training_batches.output_types, training_batches.output_shapes)
  features_final, labels_final = feedable_iterator.get_next()

  return features_final, labels_final, handle, training_iterator, heldout_iterator
 from google.colab import drive
 drive.mount("/content/gdrive")
# Read in the dataset
df = pd.read_csv('/content/gdrive/My Drive/work2.csv').astype(np.float32)
change = df.query('Speed>0').sample(frac = .1).index
df.loc[change, 'Speed'] = 0
df.loc[change, 'Class'] = 0
df.to_csv('work2.csv', header = True, index =False)
df.shape

X = df.iloc[:,:-1].values
y = df.iloc[:,-1].values
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state =1)

#reshape y-data to become column vector
y_train = np.reshape(y_train, [-1,1])
y_test = np.reshape(y_test, [-1,1])

# Standardize the dataset
scalar_x_train = StandardScaler().fit(X_train)
scalar_x_test = StandardScaler().fit(X_test)
X_train = scalar_x_train.transform(X_train)
X_test = scalar_x_test.transform(X_test)

def main(argv):
  # extract the activation function from the hyperopt spec as an attribute from the tf.nn module
  #activation = getattr(tf.nn, FLAGS.activation_function)
  # define the graph
  #with tf.Graph().as_default():
  (features_final, labels_final, handle, training_iterator, heldout_iterator) = build_input_pipeline(X_train,X_test, y_train,y_test, FLAGS.batch_size, 500)


  # Building the Bayesian Neural Network
  # we are Gaussian Reparametrization Trick
  # to compute the stochastic gradients as described in the paper
  with tf.name_scope("bayesian_neural_net", values =[features_final]):
    neural_net = tf.keras.Sequential()
    for i in range(FLAGS.num_hidden_layers):
      layer = tfp.layers.DenseReparameterization(
          units = 10,
          activation = tf.nn.relu,
          trainable = True,
          kernel_prior_fn=tfp.layers.default_multivariate_normal_fn, # NormalDiag
          kernel_posterior_fn=tfp.layers.default_mean_field_normal_fn(),
          #kernel_posterior_fn=tfp_layers_util.default_mean_field_normal_fn(), # softplus(sigma)
          kernel_posterior_tensor_fn=lambda x: x.sample(),
          bias_prior_fn=tfp.layers.default_multivariate_normal_fn, # NormalDiag
          bias_posterior_fn=tfp.layers.default_mean_field_normal_fn(), # softplus(sigma)
          bias_posterior_tensor_fn=lambda x: x.sample()
          )
      neural_net.add(layer)
  neural_net.add(tfp.layers.DenseReparameterization(
      units=2, # one dimensional output
      activation= tf.nn.sigmoid, # since regression (outcome not bounded)
      trainable=True, # i.e subject to optimization
      kernel_prior_fn=tfp.layers.default_multivariate_normal_fn, # NormalDiag with hyperopt sigma
      kernel_posterior_fn=tfp.layers.default_mean_field_normal_fn(), # softplus(sigma)
      kernel_posterior_tensor_fn=lambda x: x.sample(),
      bias_prior_fn =tfp.layers.default_multivariate_normal_fn, # NormalDiag with hyperopt sigma
      bias_posterior_fn=tfp.layers.default_mean_field_normal_fn(), # softplus(sigma)
      bias_posterior_tensor_fn=lambda x: x.sample()
      ))
  logits = neural_net(features_final)
  labels_distribution = tfd.Categorical(logits=logits)
  # Perform KL annealing. The optimal number of annealing steps
  # depends on the dataset and architecture.
  t = tf.Variable(0.0)
  kl_regularizer = t / (FLAGS.kl_annealing * len(X_train) / FLAGS.batch_size)

  #Compute the -ELBO as the loss. The kl term is annealed from 1 to 1 over
  # the epochs specified by the kl_annealing flag.
  log_likelihood = labels_distribution.log_prob(labels_final)
  #neg_log_likelihood = tf.reduce_mean(tf.squared_difference(logits,labels_final))
  neg_log_likelihood = -tf.reduce_mean(input_tensor = log_likelihood)
  kl = sum(neural_net.losses)/len(X_train) * tf.minimum(1.0, kl_regularizer)
  elbo_loss = neg_log_likelihood + kl

  # Build metrics for evaluation. Predictions are formed from single forward
  # pass of the probablisitic layers . They are cheap but noisy predictions
  predictions = tf.argmax(input = logits, axis=1)
  with tf.name_scope("train"):
    train_accuracy, train_accuracy_update_op = tf.metrics.accuracy(labels=labels_final,predictions =predictions)
    opt = tf.train.AdamOptimizer(FLAGS.learning_rate)
    train_op = opt.minimize(elbo_loss)
    update_step_op = tf.assign(t, t+1)

  with tf.name_scope("valid"):
    valid_accuracy, validation_accuracy_update_op = tf.metrics.accuracy(labels= labels_final,predictions = predictions)


  init_op = tf.group(tf.global_variables_initializer(),
                     tf.local_variables_initializer())

  stream_vars_valid = [ v for v in tf.local_variables() if "valid" in v.name]

  reset_valid_op = tf.variables_initializer(stream_vars_valid)

  with tf.Session() as sess:
    sess.run(init_op)

    # Run the training loop

    train_handle = sess.run(training_iterator.string_handle())

    heldout_handle = sess.run(heldout_iterator.string_handle())

    training_steps = int(

        round(FLAGS.epochs * (len(X_train) / FLAGS.batch_size)))

    for step in range(training_steps):

      _ = sess.run([train_op,train_accuracy_update_op, update_step_op],feed_dict={handle: train_handle})

      # Manually print the frequency
      if step % 100 == 0:
        loss_value, accuracy_value, kl_value = sess.run([elbo_loss, train_accuracy, kl], feed_dict= {handle:train_handle})
        print("Step:{:>3d} loss : {:.3f} KL: {:.3f}" .format(step , loss_value, accuracy_value, kl_value))

      if (step +1) % FLAGS.eval_freq ==0:
        # Compute log prob of heldout set by averaging draws from the model:
        # p(heldout | train) = int_model p(heldout|model) p(model|train) ~= 1/n * sum_{i=1}^n p(heldout | model_i)
        # where model_i is a draw from the posterior
        #p(model|train)
        probs = np.asarray([sess.run((labels_distribution.probs),
                                     feed_dict ={handle: heldout_handle})

                            for _ in range(FLAGS.num_monte_carlo)])

        mean_probs = np.mean(probs, axis =0).astype(np.int32)
        #print(mean_probs)

        _, label_vals = sess.run((features_final, labels_final), feed_dict = {handle: heldout_handle})

        heldout_lp = np.mean(np.log(mean_probs[np.arange(mean_probs.shape[0]), label_vals]))


        print(" ...Held_out nats: {:.3f}".format(heldout_lp))

       # Calculate validation accuracy

        for _ in range(20):

          sess.run(valid_accuracy_update_op, feed_dict={handle: heldout_handle})

        valid_value = sess.run(

            valid_accuracy, feed_dict={handle: heldout_handle})

        print(

            " ... Validation Accuracy: {:.3f}".format(valid_value))

        sess.run(reset_valid_op)
if __name__ == "__main__":
  tf.app.run()


Step:  0 loss : 0.670 KL: 0.883
Step:100 loss : 0.667 KL: 0.908
Step:200 loss : 0.681 KL: 0.909
Step:300 loss : 0.658 KL: 0.909
---------------------------------------------------------------------------
IndexError                                Traceback (most recent call last)
<ipython-input-190-637b9364d64b> in <module>()
    130         sess.run(reset_valid_op)
    131 if __name__ == "__main__":
--> 132   tf.app.run()
    133

1 frames
<ipython-input-190-637b9364d64b> in main(argv)
    109         _, label_vals = sess.run((features_final, labels_final), feed_dict = {handle: heldout_handle})
    110
--> 111         heldout_lp = np.mean(np.log(mean_probs[np.arange(mean_probs.shape[0]), label_vals]))
    112
    113

IndexError: arrays used as indices must be of integer (or boolean) type