SHARE
TWEET

Untitled

a guest Apr 26th, 2019 79 Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
  1. !pip install --upgrade -q gspread
  2. from tensorboardcolab import *
  3. import shutil
  4. #clean out the directory
  5. shutil.rmtree('./Graph', ignore_errors=True)
  6. os.mkdir('./Graph')
  7. #tf.reset_default_graph()
  8. #will start the tunneling and will print out a link:
  9. tbc=TensorBoardColab()
  10.  
  11.  
  12. !pip show tensorflow
  13.  
  14. from __future__ import absolute_import
  15. from __future__ import division
  16. from __future__ import print_function
  17. from __future__ import unicode_literals
  18.  
  19. # Dependencies
  20. import os
  21. import warnings
  22. #from absl import flags
  23. import matplotlib
  24. import numpy as np
  25. import tensorflow as tf
  26. import tensorflow_probability as tfp
  27. import math
  28. import pandas as pd
  29. tfd = tfp.distributions
  30. from sklearn.model_selection import train_test_split
  31. from sklearn.preprocessing import StandardScaler
  32. #from hyperopt import fmin, tpe, hp, STATUS_OK, STATUS_FAIL, Trials
  33. #import python_utils
  34.  
  35.  
  36. # clear graph (if any) before running
  37. tf.reset_default_graph()
  38.  
  39. ####Delete all flags before declare#####
  40.  
  41. def del_all_flags(FLAGS):
  42.     flags_dict = FLAGS._flags()    
  43.     keys_list = [keys for keys in flags_dict]    
  44.     for keys in keys_list:
  45.         FLAGS.__delattr__(keys)
  46.  
  47. del_all_flags(tf.flags.FLAGS)
  48.  
  49. flags = tf.app.flags
  50. FLAGS = tf.app.flags.FLAGS
  51. flags.DEFINE_float("learning_rate", default = 0.0001, help = "Initial learning rate.")
  52. flags.DEFINE_integer("epochs", default = 700, help = "Number of epochs to train for")
  53. flags.DEFINE_integer("batch_size", default =128, help = "Batch size.")
  54. flags.DEFINE_integer("eval_freq", default = 400, help =" Frequency at which to validate the model.")
  55. flags.DEFINE_float("kernel_posterior_scale_mean", default = -0.9, help = "Initial kernel posterior mean of the scale (log var) for q(w)")
  56. flags.DEFINE_float("kernel_posterior_scale_constraint", default = 0.2, help = "Posterior kernel constraint for the scale (log var) for q(w)")
  57. flags.DEFINE_float("kl_annealing", default = 50, help = "Epochs to anneal the KL term (anneals from 0 to 1)")
  58. flags.DEFINE_integer("num_hidden_layers", default = 4, help = "Number of hidden layers")
  59. flags.DEFINE_integer("num_monte_carlo",
  60.  
  61.                      default=50,
  62.  
  63.                      help="Network draws to compute predictive probabilities.")
  64. tf.app.flags.DEFINE_string('f', '', 'kernel')
  65. #initialize flags
  66. #FLAGS = flags.FLAGS
  67. print(FLAGS.learning_rate)
  68. print(FLAGS.epochs)
  69. print(FLAGS.num_monte_carlo)
  70.  
  71. def build_input_pipeline(X_train,X_test,y_train, y_test, batch_size, valid_size):
  72.   #Build an iterator over training batches
  73.   training_dataset = tf.data.Dataset.from_tensor_slices((X_train, y_train))
  74.   #Shuffle the dataset (note shuffle argument much larger than training size)
  75.   # and form batches of size batch_size
  76.   training_batches = training_dataset.shuffle(20000, reshuffle_each_iteration =True).repeat().batch(batch_size)
  77.   training_iterator = tf.data.make_one_shot_iterator(training_batches)
  78.  
  79.   #Building iterator over the heldout set with batch_size = heldout_size,
  80.   # i.e., return the entire heldout set as a constant.
  81.   heldout_dataset = tf.data.Dataset.from_tensor_slices((X_test, y_test))
  82.   heldout_batches = heldout_dataset.repeat().batch(valid_size)
  83.   heldout_iterator = tf.data.make_one_shot_iterator(heldout_batches)
  84.  
  85.   #Combine these into a feasible iterator that can switch between training
  86.   # and validation inputs.
  87.   # Here should be minibatch increment be defined
  88.   handle = tf.placeholder(tf.string, shape = [])
  89.   feedable_iterator = tf.data.Iterator.from_string_handle(handle, training_batches.output_types, training_batches.output_shapes)
  90.   features_final, labels_final = feedable_iterator.get_next()
  91.  
  92.   return features_final, labels_final, handle, training_iterator, heldout_iterator
  93.  from google.colab import drive
  94.  drive.mount("/content/gdrive")
  95. # Read in the dataset
  96. df = pd.read_csv('/content/gdrive/My Drive/work2.csv').astype(np.float32)
  97. change = df.query('Speed>0').sample(frac = .1).index
  98. df.loc[change, 'Speed'] = 0
  99. df.loc[change, 'Class'] = 0
  100. df.to_csv('work2.csv', header = True, index =False)
  101. df.shape
  102.  
  103. X = df.iloc[:,:-1].values
  104. y = df.iloc[:,-1].values
  105. X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state =1)
  106.  
  107. #reshape y-data to become column vector
  108. y_train = np.reshape(y_train, [-1,1])
  109. y_test = np.reshape(y_test, [-1,1])
  110.  
  111. # Standardize the dataset
  112. scalar_x_train = StandardScaler().fit(X_train)
  113. scalar_x_test = StandardScaler().fit(X_test)
  114. X_train = scalar_x_train.transform(X_train)
  115. X_test = scalar_x_test.transform(X_test)
  116.  
  117. def main(argv):
  118.   # extract the activation function from the hyperopt spec as an attribute from the tf.nn module
  119.   #activation = getattr(tf.nn, FLAGS.activation_function)
  120.   # define the graph
  121.   #with tf.Graph().as_default():
  122.   (features_final, labels_final, handle, training_iterator, heldout_iterator) = build_input_pipeline(X_train,X_test, y_train,y_test, FLAGS.batch_size, 500)
  123.  
  124.  
  125.   # Building the Bayesian Neural Network
  126.   # we are Gaussian Reparametrization Trick
  127.   # to compute the stochastic gradients as described in the paper
  128.   with tf.name_scope("bayesian_neural_net", values =[features_final]):
  129.     neural_net = tf.keras.Sequential()
  130.     for i in range(FLAGS.num_hidden_layers):
  131.       layer = tfp.layers.DenseReparameterization(
  132.           units = 10,
  133.           activation = tf.nn.relu,
  134.           trainable = True,
  135.           kernel_prior_fn=tfp.layers.default_multivariate_normal_fn, # NormalDiag
  136.           kernel_posterior_fn=tfp.layers.default_mean_field_normal_fn(),
  137.           #kernel_posterior_fn=tfp_layers_util.default_mean_field_normal_fn(), # softplus(sigma)
  138.           kernel_posterior_tensor_fn=lambda x: x.sample(),
  139.           bias_prior_fn=tfp.layers.default_multivariate_normal_fn, # NormalDiag
  140.           bias_posterior_fn=tfp.layers.default_mean_field_normal_fn(), # softplus(sigma)
  141.           bias_posterior_tensor_fn=lambda x: x.sample()
  142.           )
  143.       neural_net.add(layer)
  144.   neural_net.add(tfp.layers.DenseReparameterization(
  145.       units=2, # one dimensional output
  146.       activation= tf.nn.sigmoid, # since regression (outcome not bounded)
  147.       trainable=True, # i.e subject to optimization
  148.       kernel_prior_fn=tfp.layers.default_multivariate_normal_fn, # NormalDiag with hyperopt sigma
  149.       kernel_posterior_fn=tfp.layers.default_mean_field_normal_fn(), # softplus(sigma)
  150.       kernel_posterior_tensor_fn=lambda x: x.sample(),
  151.       bias_prior_fn =tfp.layers.default_multivariate_normal_fn, # NormalDiag with hyperopt sigma
  152.       bias_posterior_fn=tfp.layers.default_mean_field_normal_fn(), # softplus(sigma)
  153.       bias_posterior_tensor_fn=lambda x: x.sample()
  154.       ))
  155.   logits = neural_net(features_final)
  156.   labels_distribution = tfd.Categorical(logits=logits)
  157.   # Perform KL annealing. The optimal number of annealing steps
  158.   # depends on the dataset and architecture.
  159.   t = tf.Variable(0.0)
  160.   kl_regularizer = t / (FLAGS.kl_annealing * len(X_train) / FLAGS.batch_size)
  161.  
  162.   #Compute the -ELBO as the loss. The kl term is annealed from 1 to 1 over
  163.   # the epochs specified by the kl_annealing flag.
  164.   log_likelihood = labels_distribution.log_prob(labels_final)
  165.   #neg_log_likelihood = tf.reduce_mean(tf.squared_difference(logits,labels_final))
  166.   neg_log_likelihood = -tf.reduce_mean(input_tensor = log_likelihood)
  167.   kl = sum(neural_net.losses)/len(X_train) * tf.minimum(1.0, kl_regularizer)
  168.   elbo_loss = neg_log_likelihood + kl
  169.  
  170.   # Build metrics for evaluation. Predictions are formed from single forward
  171.   # pass of the probablisitic layers . They are cheap but noisy predictions
  172.   predictions = tf.argmax(input = logits, axis=1)
  173.   with tf.name_scope("train"):
  174.     train_accuracy, train_accuracy_update_op = tf.metrics.accuracy(labels=labels_final,predictions =predictions)
  175.     opt = tf.train.AdamOptimizer(FLAGS.learning_rate)
  176.     train_op = opt.minimize(elbo_loss)
  177.     update_step_op = tf.assign(t, t+1)
  178.  
  179.   with tf.name_scope("valid"):
  180.     valid_accuracy, validation_accuracy_update_op = tf.metrics.accuracy(labels= labels_final,predictions = predictions)
  181.    
  182.  
  183.   init_op = tf.group(tf.global_variables_initializer(),
  184.                      tf.local_variables_initializer())
  185.  
  186.   stream_vars_valid = [ v for v in tf.local_variables() if "valid" in v.name]
  187.  
  188.   reset_valid_op = tf.variables_initializer(stream_vars_valid)
  189.  
  190.   with tf.Session() as sess:
  191.     sess.run(init_op)
  192.    
  193.     # Run the training loop
  194.  
  195.     train_handle = sess.run(training_iterator.string_handle())
  196.  
  197.     heldout_handle = sess.run(heldout_iterator.string_handle())
  198.  
  199.     training_steps = int(
  200.  
  201.         round(FLAGS.epochs * (len(X_train) / FLAGS.batch_size)))
  202.  
  203.     for step in range(training_steps):
  204.  
  205.       _ = sess.run([train_op,train_accuracy_update_op, update_step_op],feed_dict={handle: train_handle})
  206.      
  207.       # Manually print the frequency
  208.       if step % 100 == 0:
  209.         loss_value, accuracy_value, kl_value = sess.run([elbo_loss, train_accuracy, kl], feed_dict= {handle:train_handle})
  210.         print("Step:{:>3d} loss : {:.3f} KL: {:.3f}" .format(step , loss_value, accuracy_value, kl_value))
  211.        
  212.       if (step +1) % FLAGS.eval_freq ==0:
  213.         # Compute log prob of heldout set by averaging draws from the model:
  214.         # p(heldout | train) = int_model p(heldout|model) p(model|train) ~= 1/n * sum_{i=1}^n p(heldout | model_i)
  215.         # where model_i is a draw from the posterior
  216.         #p(model|train)
  217.         probs = np.asarray([sess.run((labels_distribution.probs),
  218.                                      feed_dict ={handle: heldout_handle})
  219.      
  220.                             for _ in range(FLAGS.num_monte_carlo)])
  221.      
  222.         mean_probs = np.mean(probs, axis =0).astype(np.int32)
  223.         #print(mean_probs)
  224.        
  225.         _, label_vals = sess.run((features_final, labels_final), feed_dict = {handle: heldout_handle})
  226.      
  227.         heldout_lp = np.mean(np.log(mean_probs[np.arange(mean_probs.shape[0]), label_vals]))
  228.        
  229.        
  230.         print(" ...Held_out nats: {:.3f}".format(heldout_lp))
  231.        
  232.        # Calculate validation accuracy
  233.  
  234.         for _ in range(20):
  235.  
  236.           sess.run(valid_accuracy_update_op, feed_dict={handle: heldout_handle})
  237.  
  238.         valid_value = sess.run(
  239.  
  240.             valid_accuracy, feed_dict={handle: heldout_handle})
  241.  
  242.         print(
  243.  
  244.             " ... Validation Accuracy: {:.3f}".format(valid_value))
  245.  
  246.         sess.run(reset_valid_op)
  247. if __name__ == "__main__":
  248.   tf.app.run()
  249.  
  250.  
  251. Step:  0 loss : 0.670 KL: 0.883
  252. Step:100 loss : 0.667 KL: 0.908
  253. Step:200 loss : 0.681 KL: 0.909
  254. Step:300 loss : 0.658 KL: 0.909
  255. ---------------------------------------------------------------------------
  256. IndexError                                Traceback (most recent call last)
  257. <ipython-input-190-637b9364d64b> in <module>()
  258.     130         sess.run(reset_valid_op)
  259.     131 if __name__ == "__main__":
  260. --> 132   tf.app.run()
  261.     133
  262.  
  263. 1 frames
  264. <ipython-input-190-637b9364d64b> in main(argv)
  265.     109         _, label_vals = sess.run((features_final, labels_final), feed_dict = {handle: heldout_handle})
  266.     110
  267. --> 111         heldout_lp = np.mean(np.log(mean_probs[np.arange(mean_probs.shape[0]), label_vals]))
  268.     112
  269.     113
  270.  
  271. IndexError: arrays used as indices must be of integer (or boolean) type
RAW Paste Data
We use cookies for various purposes including analytics. By continuing to use Pastebin, you agree to our use of cookies as described in the Cookies Policy. OK, I Understand
 
Top