# Assignment3 - Single Layer Perceptron Leaky Relu

Jun 22nd, 2021
1,050
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
1. """
2. @author: chrispellegrino
3. Class: CS 767
4. Assignment: 3
5.
6. Using single layer Perceptron neural network which is connected to “Leaky ReLU”
7. activation function with (a=0.05) to predict salary of baseball players using the data set
8. “Assignment_3_Hitters.csv”.
9. Use batch gradient descent to adjust the weights and predict salary
10. with L2 regularization and Lasso with lambda = 0.01 and 10.
11. (i) Input data is Assignment_3_Hitters.csv data, which is available in Blackboard.
12. (ii)    Write a code and build a single layer Perceptron with Leaky ReLU activation function as follows.
13. (iii)   Use all the features to predict Salary.
14. (iv)    Assume anything that is needed to solve the problem. Make sure to state your assumptions.
15.
16. a. (30 points) Choose a learning rate. Show details of your work and all the steps that you take to choose a suitable learning rate.
17.
18.
19. b. (20 points) Plot total cost (MSE - Mean Square Error + regularization cost) as a function of iterations for both regularizations.
20.
21.
22. c. (50 points) Create a table of the weights and show the final weights of the solution
23.
24. i. Without regularization
25.
26.
27. ii. With L2 regularization and two different lambdas
28.
29.
30. iii. With Lasso regularization and two different lambdas
31.
32. """
33.
34. import pandas as pd
35. import numpy as np
36. import matplotlib.pyplot as plt
37. import traceback
38.
39. hitters = pd.read_csv('/Users/chrispellegrino/Desktop/CS 767 Machine Learning/Assignments/Assignment 3/Assignment_3_Hitters.csv')
40. hitters = hitters.dropna()
41. print(hitters)
42. print(hitters.dtypes)
43.
44. #converts column to category so python knows each unique value is category
45. hitters['League'] = hitters.League.astype('category').cat.codes
46. hitters['NewLeague'] = hitters.NewLeague.astype('category').cat.codes
47. hitters['Division'] = hitters.Division.astype('category').cat.codes
48.
49. #iloc means accessing by number, and loc is by name
50. x = hitters.loc[:, ['AtBat', 'Hits', 'HmRun', 'Runs', 'RBI', 'Walks', 'Years',
51.        'CAtBat', 'CHits', 'CHmRun', 'CRuns', 'CRBI', 'CWalks', 'League',
52.        'Division', 'PutOuts', 'Assists', 'Errors', 'NewLeague']]
53.
54. at_bat = hitters.loc[:, ['AtBat']]
55. hits = hitters.loc[:, ['Hits']]
56. home_runs = hitters.loc[:, ['HmRun']]
57. runs = hitters.loc[:, ['Runs']]
58. rbi = hitters.loc[:, ['RBI']]
59. walks = hitters.loc[:, ['Walks']]
60. years = hitters.loc[:, ['Years']]
61. catbat = hitters.loc[:, ['CAtBat']]
62. chits = hitters.loc[:, ['CHits']]
63. chmrun = hitters.loc[:, ['CHmRun']]
64. cruns = hitters.loc[:, ['CRuns']]
65. crbi = hitters.loc[:, ['CRBI']]
66. cwalks = hitters.loc[:, ['CWalks']]
67. league = hitters.loc[:, ['League']]
68. division = hitters.loc[:, ['League']]
69. putouts = hitters.loc[:, ['PutOuts']]
70. assists = hitters.loc[:, ['Assists']]
71. errors = hitters.loc[:, ['Errors']]
72. new_league = hitters.loc[:, ['NewLeague']]
73. y = hitters.loc[:, ['Salary']]
74. x = x.to_numpy()
75. y = y.to_numpy()
76.
77.
78. #convert every row into a numpy array
79. x = [
80.      np.array(at_bat).reshape(1, 263),
81.      np.array(hits).reshape(1, 263),
82.      np.array(home_runs).reshape(1, 263),
83.      np.array(rbi).reshape(1, 263),
84.      np.array(walks).reshape(1, 263),
85.      np.array(years).reshape(1, 263),
86.      np.array(catbat).reshape(1, 263),
87.      np.array(chits).reshape(1, 263),
88.      np.array(chmrun).reshape(1, 263),
89.      np.array(cruns).reshape(1, 263),
90.      np.array(crbi).reshape(1, 263),
91.      np.array(cwalks).reshape(1, 263),
92.      np.array(league).reshape(1, 263),
93.      np.array(division).reshape(1, 263),
94.      np.array(putouts).reshape(1, 263),
95.      np.array(assists).reshape(1, 263),
96.      np.array(errors).reshape(1, 263),
97.      np.array(new_league).reshape(1, 263)
98.      ]
99.
100.
101.
102.
103. def generate_weight(x, y):
104.     list1 =[]
105.     for i in range(x * y):
106.         list1.append(np.random.randn())
107.     return(np.array(list1).reshape(x, y)) #reshape keeps dimensions of weight (20, 5)
108.
109. #20 features and 5 selected arbitrarily
110. weight1 = generate_weight(20, 5)
111. weight2 = generate_weight(5, 1) #1 because we are just trying to predict one output
112.
113.
114.
115. #leaky relu
116. def leaky_relu(x):
117.     x = np.where(x > 0, x, x * 0.01)
118.     return x
119.
120. #feed foward, x is the feature
121. def feed_forward(x, weight1, weight2):
122.     # input layer to hidden layer
123.     input_layer_input = x.dot(weight1)# input from layer 1
124.     input_layer_output = leaky_relu(input_layer_input)# output of layer 2 (output is the hidden layer)
125.
126.     # hidden layer to output layer
127.     hidden_layer_input = input_layer_output.dot(weight2)# input of 1st output layer
128.     hidden_layer_output = leaky_relu(hidden_layer_input)# output of out layer
129.     return(hidden_layer_output)
130.
131.
132. # for loss we will be using mean square error(MSE)
133. def loss(out, Y): #out is the output from feed forward, and Y is actualy salary
134.     s =(np.square(out-Y))
135.     s = np.sum(s)/len(Y)
136.     return(s)
137.
138. # Back propagation of error
139. def back_prop(x, y, weight1, weight2, learning_rate):
140.
141.     # output layer back to hidden layer
142.     output_layer_input = x.dot(weight1)# input from layer 1
143.     output_layer_output = leaky_relu(output_layer_input)# output of layer 2
144.
145.     # hidden layer back to input layer
146.     hidden_layer_input2 = output_layer_output.dot(weight2)# input of out layer
147.     hidden_layer_output2 = leaky_relu(hidden_layer_input2)# output of out layer
148.     # error in output layer
149.     d2 =(hidden_layer_output2-y)
150.     d1 = np.multiply((weight2.dot((d2.transpose()))).transpose(),
151.                                    (np.multiply(output_layer_output, 1-output_layer_output)))
152.
153.     # Gradient for weight1 and weight2
156.
157.     # Updating parameters
160.
161.     return(weight1, weight2)
162.
163.
164. #train is method to measure accuracy of model
165. def train(x, Y, weight1, weight2, learning_rate = 0.01, epoch = 10):
166.     accuracy_training_set =[]
167.     test_set =[]
168.     for j in range(epoch):
169.         list2 =[]
170.         for i in range(len(x)): #looping through all features
171.             out = feed_forward(x[i], weight1, weight2) #running through feed forward
172.             list2.append((test_set(out, Y[i]))) #adding output to the list
173.             weight1, weight2 = back_prop(x[i], Y[i], weight1, weight2, learning_rate) #back propogate to get new weights and update network
174.         accuracy_training_set.append((1-(sum(list2)/len(x)))*100)
175.         test_set.append(sum(list2)/len(x))
176.     return(accuracy_training_set, test_set, weight1, weight2)
177.
178.
179. def predict_salary(x, weight1, weight2):
180.     Out = feed_forward(x, weight1, weight2)
181.     for i in range(len(Out[0])):
182.         Out[0[i]]
183.
184. # x is take all the features except salary, which is Y
185. accuracy_training_set, test_set, weight1, weight2 = train(x, y, weight1, weight2, 0.01, 100)
186.
187. # ploting accuracy
188. plt.plot(accuracy_training_set)
189. plt.ylabel('Accuracy Training Set')
190. plt.xlabel('Epochs/Iterations:')
191. plt.show()
192.
193. # plotting Loss
194. plt.plot(test_set)
195. plt.ylabel('Test Set')
196. plt.xlabel('Epochs/Iterations:')
197. plt.show()
198.
199. #predict salary of players
200. predict_salary(x[i], weight1, weight2)
201.
RAW Paste Data