LostInMyThoughts137

Assignment3 - Single Layer Perceptron Leaky Relu

Jun 22nd, 2021
1,050
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
  1. """
  2. @author: chrispellegrino
  3. Class: CS 767
  4. Assignment: 3
  5.  
  6. Using single layer Perceptron neural network which is connected to “Leaky ReLU”
  7. activation function with (a=0.05) to predict salary of baseball players using the data set
  8. “Assignment_3_Hitters.csv”.
  9. Use batch gradient descent to adjust the weights and predict salary
  10. with L2 regularization and Lasso with lambda = 0.01 and 10.
  11. (i) Input data is Assignment_3_Hitters.csv data, which is available in Blackboard.
  12. (ii)    Write a code and build a single layer Perceptron with Leaky ReLU activation function as follows.
  13. (iii)   Use all the features to predict Salary.
  14. (iv)    Assume anything that is needed to solve the problem. Make sure to state your assumptions.
  15.  
  16. a. (30 points) Choose a learning rate. Show details of your work and all the steps that you take to choose a suitable learning rate.
  17.  
  18.  
  19. b. (20 points) Plot total cost (MSE - Mean Square Error + regularization cost) as a function of iterations for both regularizations.
  20.  
  21.  
  22. c. (50 points) Create a table of the weights and show the final weights of the solution
  23.  
  24. i. Without regularization
  25.  
  26.  
  27. ii. With L2 regularization and two different lambdas
  28.  
  29.  
  30. iii. With Lasso regularization and two different lambdas
  31.  
  32. """
  33.  
  34. import pandas as pd
  35. import numpy as np
  36. import matplotlib.pyplot as plt
  37. import traceback
  38.  
  39. hitters = pd.read_csv('/Users/chrispellegrino/Desktop/CS 767 Machine Learning/Assignments/Assignment 3/Assignment_3_Hitters.csv')
  40. hitters = hitters.dropna()
  41. print(hitters)
  42. print(hitters.dtypes)
  43.  
  44. #converts column to category so python knows each unique value is category
  45. hitters['League'] = hitters.League.astype('category').cat.codes
  46. hitters['NewLeague'] = hitters.NewLeague.astype('category').cat.codes
  47. hitters['Division'] = hitters.Division.astype('category').cat.codes
  48.  
  49. #iloc means accessing by number, and loc is by name
  50. x = hitters.loc[:, ['AtBat', 'Hits', 'HmRun', 'Runs', 'RBI', 'Walks', 'Years',
  51.        'CAtBat', 'CHits', 'CHmRun', 'CRuns', 'CRBI', 'CWalks', 'League',
  52.        'Division', 'PutOuts', 'Assists', 'Errors', 'NewLeague']]
  53.  
  54. at_bat = hitters.loc[:, ['AtBat']]
  55. hits = hitters.loc[:, ['Hits']]
  56. home_runs = hitters.loc[:, ['HmRun']]
  57. runs = hitters.loc[:, ['Runs']]
  58. rbi = hitters.loc[:, ['RBI']]
  59. walks = hitters.loc[:, ['Walks']]
  60. years = hitters.loc[:, ['Years']]
  61. catbat = hitters.loc[:, ['CAtBat']]
  62. chits = hitters.loc[:, ['CHits']]
  63. chmrun = hitters.loc[:, ['CHmRun']]
  64. cruns = hitters.loc[:, ['CRuns']]
  65. crbi = hitters.loc[:, ['CRBI']]
  66. cwalks = hitters.loc[:, ['CWalks']]
  67. league = hitters.loc[:, ['League']]
  68. division = hitters.loc[:, ['League']]
  69. putouts = hitters.loc[:, ['PutOuts']]
  70. assists = hitters.loc[:, ['Assists']]
  71. errors = hitters.loc[:, ['Errors']]
  72. new_league = hitters.loc[:, ['NewLeague']]
  73. y = hitters.loc[:, ['Salary']]
  74. x = x.to_numpy()
  75. y = y.to_numpy()
  76.  
  77.  
  78. #convert every row into a numpy array
  79. x = [
  80.      np.array(at_bat).reshape(1, 263),
  81.      np.array(hits).reshape(1, 263),
  82.      np.array(home_runs).reshape(1, 263),
  83.      np.array(rbi).reshape(1, 263),
  84.      np.array(walks).reshape(1, 263),
  85.      np.array(years).reshape(1, 263),
  86.      np.array(catbat).reshape(1, 263),
  87.      np.array(chits).reshape(1, 263),
  88.      np.array(chmrun).reshape(1, 263),
  89.      np.array(cruns).reshape(1, 263),
  90.      np.array(crbi).reshape(1, 263),
  91.      np.array(cwalks).reshape(1, 263),
  92.      np.array(league).reshape(1, 263),
  93.      np.array(division).reshape(1, 263),
  94.      np.array(putouts).reshape(1, 263),
  95.      np.array(assists).reshape(1, 263),
  96.      np.array(errors).reshape(1, 263),
  97.      np.array(new_league).reshape(1, 263)
  98.      ]
  99.  
  100.  
  101.  
  102.  
  103. def generate_weight(x, y):
  104.     list1 =[]
  105.     for i in range(x * y):
  106.         list1.append(np.random.randn())
  107.     return(np.array(list1).reshape(x, y)) #reshape keeps dimensions of weight (20, 5)
  108.  
  109. #20 features and 5 selected arbitrarily
  110. weight1 = generate_weight(20, 5)
  111. weight2 = generate_weight(5, 1) #1 because we are just trying to predict one output
  112.  
  113.  
  114.  
  115. #leaky relu
  116. def leaky_relu(x):
  117.     x = np.where(x > 0, x, x * 0.01)    
  118.     return x
  119.  
  120. #feed foward, x is the feature
  121. def feed_forward(x, weight1, weight2):
  122.     # input layer to hidden layer
  123.     input_layer_input = x.dot(weight1)# input from layer 1
  124.     input_layer_output = leaky_relu(input_layer_input)# output of layer 2 (output is the hidden layer)
  125.      
  126.     # hidden layer to output layer
  127.     hidden_layer_input = input_layer_output.dot(weight2)# input of 1st output layer
  128.     hidden_layer_output = leaky_relu(hidden_layer_input)# output of out layer
  129.     return(hidden_layer_output)
  130.  
  131.  
  132. # for loss we will be using mean square error(MSE)
  133. def loss(out, Y): #out is the output from feed forward, and Y is actualy salary
  134.     s =(np.square(out-Y))
  135.     s = np.sum(s)/len(Y)
  136.     return(s)
  137.  
  138. # Back propagation of error
  139. def back_prop(x, y, weight1, weight2, learning_rate):
  140.      
  141.     # output layer back to hidden layer
  142.     output_layer_input = x.dot(weight1)# input from layer 1
  143.     output_layer_output = leaky_relu(output_layer_input)# output of layer 2
  144.      
  145.     # hidden layer back to input layer
  146.     hidden_layer_input2 = output_layer_output.dot(weight2)# input of out layer
  147.     hidden_layer_output2 = leaky_relu(hidden_layer_input2)# output of out layer
  148.     # error in output layer
  149.     d2 =(hidden_layer_output2-y)
  150.     d1 = np.multiply((weight2.dot((d2.transpose()))).transpose(),
  151.                                    (np.multiply(output_layer_output, 1-output_layer_output)))
  152.  
  153.     # Gradient for weight1 and weight2
  154.     weight1_adj = x.transpose().dot(d1)
  155.     weight2_adj = output_layer_output.transpose().dot(d2)
  156.      
  157.     # Updating parameters
  158.     weight1 = weight1-(learning_rate*(weight1_adj))
  159.     weight2 = weight2-(learning_rate*(weight2_adj))
  160.      
  161.     return(weight1, weight2)
  162.  
  163.  
  164. #train is method to measure accuracy of model
  165. def train(x, Y, weight1, weight2, learning_rate = 0.01, epoch = 10):
  166.     accuracy_training_set =[]
  167.     test_set =[]
  168.     for j in range(epoch):
  169.         list2 =[]
  170.         for i in range(len(x)): #looping through all features
  171.             out = feed_forward(x[i], weight1, weight2) #running through feed forward
  172.             list2.append((test_set(out, Y[i]))) #adding output to the list
  173.             weight1, weight2 = back_prop(x[i], Y[i], weight1, weight2, learning_rate) #back propogate to get new weights and update network
  174.         accuracy_training_set.append((1-(sum(list2)/len(x)))*100)
  175.         test_set.append(sum(list2)/len(x))
  176.     return(accuracy_training_set, test_set, weight1, weight2)
  177.  
  178.  
  179. def predict_salary(x, weight1, weight2):
  180.     Out = feed_forward(x, weight1, weight2)
  181.     for i in range(len(Out[0])):
  182.         Out[0[i]]
  183.        
  184. # x is take all the features except salary, which is Y        
  185. accuracy_training_set, test_set, weight1, weight2 = train(x, y, weight1, weight2, 0.01, 100)
  186.  
  187. # ploting accuracy
  188. plt.plot(accuracy_training_set)
  189. plt.ylabel('Accuracy Training Set')
  190. plt.xlabel('Epochs/Iterations:')
  191. plt.show()
  192.  
  193. # plotting Loss
  194. plt.plot(test_set)
  195. plt.ylabel('Test Set')
  196. plt.xlabel('Epochs/Iterations:')
  197. plt.show()
  198.  
  199. #predict salary of players
  200. predict_salary(x[i], weight1, weight2)
  201.  
RAW Paste Data