daily pastebin goal
34%
SHARE
TWEET

Untitled

a guest Jan 20th, 2018 52 Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
  1. import os
  2.  
  3. n_samples = mnist.train.num_examples
  4.  
  5. learning_rate = 0.001
  6. batch_size = 100
  7.  
  8. n_hidden_recog_1=200 # 1 sloj enkodera
  9. n_hidden_recog_2=200 # 2 sloj enkodera
  10. n_hidden_gener_1=200 # 1 sloj dekodera
  11. n_hidden_gener_2=200 # 2 sloj dekodera
  12. n_z=20 # broj skrivenih varijabli
  13. n_input=784 # MNIST data input (img shape: 28*28)
  14. in_shape = (28,28)
  15.  
  16. def get_canvas(Z, ind, nx, ny, in_shape, batch_size, sess):
  17.     """Crtanje rekonstrukcija na odgovarajućim pozicijama u 2D prostoru skrivenih varijabli
  18.     Z -- skriveni vektori raspoređeni u gridu oko ishodišta
  19.     ind -- indeksi za rezanje Z-a na batch_size blokove za slanje u graf -zbog problema sa random generatorom
  20.     nx -- raspon grida po x osi - skrivena varijabla z0
  21.     ny -- raspon grida po y osi - skrivena varijabla z1
  22.     in_shape -- dimenzije jedne rekonstrukcije i.e. ulazne sličice
  23.     batch_size -- veličina minibatcha na koji je graf naviknut
  24.     sess -- session grafa mreže
  25.     """
  26.     # get reconstructions for visualiations
  27.     X = np.empty((0,in_shape[0]*in_shape[1])) # empty array for concatenation
  28.     # split hidden vectors into minibatches of batch_size due to TF random generator limitation
  29.     for batch in np.array_split(Z,ind):
  30.         # fill up last batch to full batch_size if neccessary
  31.         # this addition will not be visualized, but is here to avoid TF error
  32.         if batch.shape[0] < batch_size:
  33.             batch = np.concatenate((batch, np.zeros((batch_size-batch.shape[0], batch.shape[1]))), 0)
  34.         # get batch_size reconstructions and add them to array of previous reconstructions
  35.         X = np.vstack((X, sess.run(x_reconstr_mean_out, feed_dict={z: batch})))
  36.     # make canvas with reconstruction tiles arranged by the hidden state coordinates of each reconstruction
  37.     # this is achieved for all reconstructions by clever use of reshape, swap axes and axis inversion
  38.     return (X[0:nx*ny,:].reshape((nx*ny,in_shape[0],in_shape[1])).swapaxes(0,1)
  39.             .reshape((in_shape[0],ny,nx*in_shape[1])).swapaxes(0,1)[::-1,:,:]
  40.             .reshape((ny*in_shape[0],nx*in_shape[1])))
  41.  
  42. def draw_reconstructions(ins, outs, states, shape_in, shape_state):
  43.     """Vizualizacija ulaza i pripadajućih rekonstrkcija i stanja skrivenog sloja
  44.     ins -- ualzni vektori
  45.     outs -- rekonstruirani vektori
  46.     states -- vektori stanja skrivenog sloja
  47.     shape_in -- dimezije ulaznih slika npr. (28,28)
  48.     shape_state -- dimezije za 2D prikaz stanja (npr. za 100 stanja (10,10)
  49.     """
  50.     plt.figure(figsize=(8, 12*4))
  51.     for i in range(20):
  52.  
  53.         plt.subplot(20, 4, 4*i + 1)
  54.         plt.imshow(ins[i].reshape(shape_in), vmin=0, vmax=1, interpolation="nearest")
  55.         plt.title("Test input")
  56.         plt.subplot(20, 4, 4*i + 2)
  57.         plt.imshow(outs[i][0:784].reshape(shape_in), vmin=0, vmax=1, interpolation="nearest")
  58.         plt.title("Reconstruction")
  59.         plt.subplot(20, 4, 4*i + 3)
  60.         plt.imshow(states[i][0:(shape_state[0] * shape_state[1])].reshape(shape_state),
  61.                    vmin=-4, vmax=4, interpolation="nearest")
  62.         plt.colorbar()
  63.         plt.title("States")
  64.     plt.tight_layout()
  65.    
  66. def plot_latent(inmat, labels):
  67.     """Crtanje pozicija uzoraka u 2D latentnom prostoru
  68.     inmat -- matrica latentnih stanja
  69.     labels -- labela klas
  70.     """
  71.     plt.figure(figsize=(8, 6))
  72.     plt.axis([-4, 4, -4, 4])
  73.     plt.gca().set_autoscale_on(False)
  74.  
  75.     plt.scatter(inmat[:, 0], inmat[:, 1], c=np.argmax(labels, 1))
  76.     plt.colorbar()
  77.     plt.xlabel('z0')
  78.     plt.ylabel('z1')
  79.  
  80. def save_latent_plot(name):
  81.     """Spremanje trenutnog figure-a
  82.     name -- ime datoteke
  83.     """
  84.     plt.savefig(name)
  85.    
  86. def weight_variable(shape, name):
  87.     """Kreiranje težina"""
  88.     # http://andyljones.tumblr.com/post/110998971763/an-explanation-of-xavier-initialization
  89.     return tf.get_variable(name, shape=shape,
  90.             initializer=tf.contrib.layers.xavier_initializer())
  91.  
  92. def bias_variable(shape):
  93.     """Kreiranje pomaka"""
  94.     initial = tf.zeros(shape, dtype=tf.float32)
  95.     return tf.Variable(initial)
  96.  
  97. def variable_summaries(var, name):
  98.     """Prikupljanje podataka za Tensorboard"""
  99.     with tf.name_scope(name):
  100.         mean = tf.reduce_mean(var)
  101.         tf.summary.scalar('mean', mean)
  102.         stddev = tf.sqrt(tf.reduce_mean(tf.square(var - mean)))
  103.         tf.summary.scalar('stddev', stddev)
  104.         tf.summary.scalar('max', tf.reduce_max(var))
  105.         tf.summary.scalar('min', tf.reduce_min(var))
  106.         tf.summary.histogram(name, var)
  107.  
  108. def vae_layer(input_tensor, input_dim, output_dim, layer_name, act=tf.nn.softplus):
  109.     """Kreiranje jednog skrivenog sloja"""
  110.     # Adding a name scope ensures logical grouping of the layers in the graph.
  111.     with tf.name_scope(layer_name):
  112.         # This Variable will hold the state of the weights for the layer
  113.         weights = weight_variable([input_dim, output_dim], layer_name + '/weights')
  114.         variable_summaries(weights,'weights')
  115.         tf.summary.tensor_summary('weightsT', weights)
  116.         biases = bias_variable([output_dim])
  117.         variable_summaries(biases, 'biases')
  118.         preactivate = tf.matmul(input_tensor, weights) + biases
  119.         tf.summary.histogram('pre_activations', preactivate)
  120.         activations = act(preactivate, name='activation')
  121.         tf.summary.histogram('activations', activations)
  122.     return activations
  123.  
  124. tf.reset_default_graph()
  125.    
  126. sess = tf.InteractiveSession()
  127.        
  128. # definicije ulaznog tenzora
  129. x = tf.placeholder("float", [None, 784])
  130.  
  131. # definirajte enkoderski dio
  132. layer_e1 = vae_layer(x, n_input, n_hidden_recog_1, 'layer_e1')
  133. layer_e2 = vae_layer(layer_e1, n_hidden_recog_1, n_hidden_recog_2, 'layer_e2')
  134.  
  135. with tf.name_scope('z'):
  136. # definirajte skrivene varijable i pripadajući generator šuma
  137.     z_mean = vae_layer(layer_e2, n_hidden_recog_2, n_z, 'z_mean', act=tf.identity)
  138.     z_log_sigma_sq = vae_layer(layer_e2, n_hidden_recog_2, n_z, 'z_log_sigma_sq', act=tf.identity)
  139.     eps = tf.random_normal((batch_size, n_z), 0, 1, dtype=tf.float32)
  140.                          
  141.     z = tf.add(z_mean, tf.multiply(tf.sqrt(tf.exp(z_log_sigma_sq)), eps))
  142.     tf.summary.histogram('activations', z)
  143.  
  144. # definirajte dekoderski dio
  145. layer_d1 = vae_layer(z, n_z, n_hidden_gener_1, 'layer_d1')
  146. layer_d2 = vae_layer(layer_d1, n_hidden_gener_1, n_hidden_gener_2, 'layer_d2')
  147.            
  148. # definirajte srednju vrijednost rekonstrukcije
  149. x_reconstr_mean = vae_layer(layer_d2, n_hidden_gener_2, n_input, 'x_reconstr_mean', act=tf.identity)
  150.  
  151. x_reconstr_mean_out = tf.nn.sigmoid(x_reconstr_mean)
  152.  
  153. # definirajte dvije komponente funkcije cijene
  154. with tf.name_scope('cost'):
  155.     cost1 = tf.reduce_sum(tf.nn.sigmoid_cross_entropy_with_logits(labels=x, logits=x_reconstr_mean), 1)
  156.     tf.summary.histogram('cross_entropy', cost1)
  157.     cost2 = -1/2 * tf.reduce_sum(1 + z_log_sigma_sq - tf.square(z_mean) - tf.exp(z_log_sigma_sq), 1)  
  158.     tf.summary.histogram('D_KL', cost2)
  159.     cost = tf.reduce_mean(cost1 + cost2)   # average over batch
  160.     tf.summary.histogram('cost', cost)
  161.                          
  162. # ADAM optimizer
  163. with tf.name_scope('train'):
  164.     optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost)
  165.  
  166. # Prikupljanje podataka za Tensorboard
  167. merged = tf.summary.merge_all()
  168.  
  169. init = tf.global_variables_initializer()
  170.  
  171. saver = tf.train.Saver()
  172.  
  173. n_epochs = 100
  174. train_writer = tf.summary.FileWriter('train', sess.graph)
  175.  
  176. sess.run(init)
  177.  
  178. total_batch = int(n_samples / batch_size)
  179. step = 0
  180. for epoch in range(n_epochs):
  181.     avg_cost = 0.
  182.        
  183.     for i in range(total_batch):
  184.         batch_xs, _ = mnist.train.next_batch(batch_size)
  185.         # Fit training using batch data
  186.         opt, cos = sess.run((optimizer, cost), feed_dict={x: batch_xs})
  187.         # Compute average loss
  188.         avg_cost += cos / n_samples * batch_size
  189.        
  190.     # Display logs per epoch step
  191.     if epoch%(int(n_epochs/10)) == 0:
  192.         print("Epoch:", '%04d' % (epoch+1),
  193.               "cost=", "{:.9f}".format(avg_cost))
  194.         run_options = tf.RunOptions(trace_level=tf.RunOptions.FULL_TRACE)
  195.         run_metadata = tf.RunMetadata()
  196.         summary, _ = sess.run([merged, optimizer], feed_dict={x: batch_xs},
  197.                               options=run_options, run_metadata=run_metadata)
  198.         train_writer.add_run_metadata(run_metadata, 'epoch%03d' % epoch)
  199.         train_writer.add_summary(summary, i)
  200.        
  201.         saver.save(sess, os.path.join('train', "model.ckpt"), epoch)
  202.  
  203. train_writer.close()
  204.  
  205. # vizualizacija rekonstrukcije i stanja
  206. x_sample = mnist.test.next_batch(100)[0]
  207. x_reconstruct, z_out = sess.run([x_reconstr_mean_out, z], feed_dict={x: x_sample})
  208.  
  209. draw_reconstructions(x_sample, x_reconstruct, z_out, (28, 28), (4,5)) # prilagodite dimenzije prema potrebi
  210.  
  211. # Vizualizacija raspored testnih uzoraka u 2D prostoru skrivenih varijabli - 1. način
  212. x_sample, y_sample = mnist.test.next_batch(5000)
  213. z_mu, z_sigma = sess.run((z_mean, z_log_sigma_sq), feed_dict={x: x_sample})
  214.    
  215. plot_latent(z_mu, y_sample)
  216. #save_latent_plot('trt.png')
  217.  
  218. # Vizualizacija raspored testnih uzoraka u 2D prostoru skrivenih varijabli - 2. način
  219.  
  220. nx = ny = 21
  221. x_values = np.linspace(-3, 3, nx)
  222. y_values = np.linspace(-3, 3, ny)
  223.  
  224. canvas = np.empty((28*ny, 28*nx))
  225.  
  226. # Trikovito popunjavanje rezultata za grid zbog fiksirane veličine z batcha u grafu
  227. # Valjda će se to riješiti u nekoj budućoj verziji TF
  228. Xi, Yi = np.meshgrid(x_values, y_values)
  229. Z = np.column_stack((Xi.flatten(), Yi.flatten()))
  230. X = np.empty((0,28*28))
  231. ind = list(range(batch_size, nx*ny, batch_size))
  232. for i in np.array_split(Z,ind):
  233.     if i.shape[0] < batch_size:
  234.         i = np.concatenate((i, np.zeros((batch_size-i.shape[0], i.shape[1]))), 0)
  235.     X = np.vstack((X, sess.run(x_reconstr_mean_out, feed_dict={z: i})))
  236.    
  237. for i, yi in enumerate(y_values):
  238.     for j, xi in enumerate(x_values):
  239.         canvas[(nx-i-1)*28:(nx-i)*28, j*28:(j+1)*28] = X[i*nx+j].reshape(28, 28)
  240.  
  241. plt.figure(figsize=(8, 10))
  242. plt.imshow(canvas, origin="upper")
  243. plt.xticks( np.linspace(14,588-14,11), np.round(np.linspace(-3,3,11), 2) )
  244. plt.yticks( np.linspace(14,588-14,11), np.round(np.linspace(3,-3,11), 2) )
  245. plt.xlabel('z0')
  246. plt.ylabel('z1')
  247. plt.tight_layout()
  248.  
  249. # Vizualizacija ugašenih elemenata skrivenog sloja - 1. način
  250.  
  251. # Pomoćna funkcija za crtanje boxplot grafova
  252. def boxplot_vis(pos, input_data, label_x, label_y):
  253.     ax = fig.add_subplot(130+pos)
  254.     plt.boxplot(input_data, 0, '', 0, 0.75)
  255.     ax.set_xlabel(label_x)
  256.     ax.set_ylabel(label_y)
  257.     return ax
  258.    
  259. fig = plt.figure(figsize=(15,4))
  260.  
  261. # Vizualizacija statistike za z_mean
  262. boxplot_vis(1,z_mu, 'Z mean values', 'Z elemets')
  263.  
  264. # Vizualizacija statistike za z_sigma
  265. ax = boxplot_vis(2, np.square(np.exp(z_sigma)), 'Z sigma values', 'Z elemets')
  266. ax.set_xlim([-0.05,1.1])
  267.  
  268. # Vizualizacija statistike za težine ulaza u dekoder
  269. test = tf.get_default_graph().get_tensor_by_name("layer_d1/weights:0")
  270. weights_d1 = test.eval(session=sess)
  271. boxplot_vis(3, weights_d1.T, 'Weights to decoder', 'Z elemets')
  272.  
  273.  
  274. # Vizualizacija ugašenih elemenata skrivenog sloja - 2. način
  275.  
  276. from mpl_toolkits.mplot3d import Axes3D
  277.  
  278. # Funkcija za crtanje 3D bar grafa
  279. def bargraph_vis(pos, input_data, dims, color, labels):
  280.     ax = fig.add_subplot(120+pos, projection='3d')
  281.     xpos, ypos = np.meshgrid(range(dims[0]), range(dims[1]))
  282.     xpos = xpos.flatten('F')
  283.     ypos = ypos.flatten('F')
  284.     zpos = np.zeros_like(xpos)
  285.    
  286.     dx = np.ones_like(zpos)
  287.     dy = np.ones_like(zpos) * 0.5
  288.     dz = input_data.flatten()
  289.    
  290.     ax.bar3d(xpos, ypos, zpos, dx, dy, dz, color=color)
  291.     ax.view_init(elev=30., azim=5)
  292.     ax.set_xlabel(labels[0])
  293.     ax.set_ylabel(labels[1])
  294.     ax.set_zlabel(labels[2])
  295.                              
  296. fig = plt.figure(figsize=(15,7))
  297.  
  298. # 3D bar graf za z_mean
  299. labels = ('Samples', 'Hidden elements', 'Z mean')
  300. bargraph_vis(1, z_mu, [200, z_mu.shape[1]], 'g', labels)
  301.  
  302. # 3D bar graf za težine iz z_mena u dekoder
  303. labels = ('Decoder elements', 'Hidden elements Z', 'Weights')
  304. bargraph_vis(2, weights_d1.T, weights_d1.T.shape, 'y', labels)
RAW Paste Data
Top