# Untitled

a guest
Nov 19th, 2019
93
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
1. import numpy as np
2. import csv
3. import Circle_data
4. import matplotlib.pyplot as plt
5.
6. class Data(): #這個類別協助我們把資料轉成向量以及上好標籤
7.     def __init__(self, x, z):
8.         self.x = self.setX(x)
9.         self.z = self.setZ(z)
10.
11.     def setX(self, x):
12.         return np.array([x]).T
13.
14.     def setZ(self, z):
15.         return np.array([z]).T
16.
18.     data = []
19.     with open("output.csv", newline = "") as csvfile:
21.         for row in rows:
22.             x = [float(row['x1']), float(row['x2'])]
23.             z = [float(row['z'])]
24.             data.append(Data(x, z))
25.     return data
26.
27. def f(x, function_name = 'ReLU'):
28.     if function_name == 'ReLU':
29.         return np.maximum(x, 0)
30.     elif function_name == 'tanh':
31.         return np.tanh(x)
32.     elif function_name == 'sigmoid':
33.         return 1 / (1 + np.exp(-x))
34.
35. def df(x, Hidden_nodes, function_name = 'ReLU'):
36.     if function_name == 'ReLU':
37.         return np.eye(Hidden_nodes, dtype = 'float')*(np.where(x > 0, 1, 0))
38.     elif function_name == 'tanh':
39.         return np.eye(Hidden_nodes, dtype = 'float')*(1 - f(x, function_name)*f(x, function_name))
40.     elif function_name == 'sigmoid':
41.         return np.eye(Hidden_nodes, dtype = 'float')*(f(x, function_name)*(1 - f(x, function_name)))
42.
43. def loss(N, Layer, data, loss_name ='MSE'):
44.     if loss_name == 'MSE':
45.         result = 0
46.         for i in range(N):
47.             result = result + (Layer[2][i] - data[i].z)**2
48.         return result / (2*N)
49.
50. def forward(W1, W2, b, N, data, function_name = 'ReLU'): #向前傳播
51.     Layer1_1 = []
52.     Layer1_2 = []
53.     output = []
54.     Layer = []
55.     for i in range(N):
56.         Layer1_1.append(W1.dot(data[i].x)+b)  #計算每一層的值，並存取下來
57.         Layer1_2.append(f(Layer1_1[i], function_name))
58.         output.append(W2.dot(Layer1_2[i]))
59.     Layer.append(Layer1_1)
60.     Layer.append(Layer1_2)
61.     Layer.append(output)
62.     return Layer
63.
64. def backward(W2, N, Layer, data, Hidden_nodes, function_name = 'ReLU'): #反向傳播
65.     dW2 = np.zeros((1, Hidden_nodes)) #計算W2的梯度
66.     for i in range(N):
67.         dW2 = dW2 + (Layer[2][i] - data[i].z)*(Layer[1][i].T)
68.     dW2 = dW2 / N
69.
70.     dW1 = np.zeros((Hidden_nodes, 2)) #計算W1和b的梯度
71.     db = np.zeros((Hidden_nodes, 1))
72.     for i in range(N):
73.         temp = (Layer[2][i] - data[i].z)*W2.dot(df(Layer[0][i], Hidden_nodes, function_name))
74.         db = db + temp.T
75.         dW1 = dW1 + np.kron(temp.T, data[i].x.T)
76.     dW1 = dW1 / N
77.     db = db / N
78.     return dW1, dW2, db
79.
80. def Train(W1, W2, b, data, Hidden_nodes, N, learning_rate = 0.3, beta = 0.1, gamma = 0.9, beta1 = 0.9, beta2 = 0.999, Method = 'Adam', function_name = 'ReLU', loss_name ='MSE', train_size = 20000):
81.     TOL=1e-7
82.     k = 1
83.     Layer = forward(W1, W2, b, N, data)
84.     best_result = 0
85.     mW1 = sW1 = vW1 = np.zeros(W1.shape)
86.     mW2 = sW2 = vW2 = np.zeros(W2.shape)
87.     mb = sb = vb = np.zeros(b.shape)
88.     train = []
89.     test = []
90.     G = [[], [], []]
91.     dG = [np.zeros(W1.shape), np.zeros(W2.shape), np.zeros(b.shape)]
92.     y = []
93.     for i in range(train_size):
94.         train.append(data[i])
95.     for i in range(N-train_size):
96.         test.append(data[train_size+i])
97.     Layer = forward(W1, W2, b, train_size, data, function_name = 'ReLU')
98.     while k <= 200:
99.         dW1, dW2, db = backward(W2, train_size, Layer, train, Hidden_nodes, function_name = 'ReLU')
101.             W1 = W1 - learning_rate*dW1
102.             W2 = W2 - learning_rate*dW2
103.             b = b - learning_rate*db
104.         elif Method == 'Momentum':
105.             mW1 = beta*dW1 + gamma*mW1
106.             mW2 = beta*dW2 + gamma*mW2
107.             mb = beta*db + gamma*mb
108.             W1 = W1 - learning_rate*mW1
109.             W2 = W2 - learning_rate*mW2
110.             b = b - learning_rate*mb
112.             G[0].append(dW1)
113.             G[1].append(dW2)
114.             G[2].append(db)
115.             for i in range(k):
116.                 dG[0] += G[0][i]**2
117.                 dG[1] += G[1][i]**2
118.                 dG[2] += G[2][i]**2
119.             dG[0] = dG[0]**(1/2)
120.             dG[1] = dG[1]**(1/2)
121.             dG[2] = dG[2]**(1/2)
122.             W1 = W1 - (learning_rate/(dG[0] + TOL))*dW1
123.             W2 = W2 - (learning_rate/(dG[1] + TOL))*dW2
124.             b = b - (learning_rate/(dG[2] + TOL))*db
125.         elif Method == 'RMSProp':
126.             G[0].append(dW1)
127.             G[1].append(dW2)
128.             G[2].append(db)
129.             for i in range(k):
130.                 dG[0] += G[0][i]**2
131.                 dG[1] += G[1][i]**2
132.                 dG[2] += G[2][i]**2
133.             dG[0] = (1/k)*dG[0]
134.             dG[1] = (1/k)*dG[1]
135.             dG[2] = (1/k)*dG[2]
136.             dG[0] = dG[0]**(1/2)
137.             dG[1] = dG[1]**(1/2)
138.             dG[2] = dG[2]**(1/2)
139.             W1 = W1 - (learning_rate/(dG[0] + TOL))*dW1
140.             W2 = W2 - (learning_rate/(dG[1] + TOL))*dW2
141.             b = b - (learning_rate/(dG[2] + TOL))*db
142.
144.             vW1 = beta1*vW1 + (1 - beta1)*dW1
145.             vW2 = beta1*vW2 + (1 - beta1)*dW2
146.             vb = beta1*vb + (1 - beta1)*db
147.             sW1 = beta2*sW1 + (1 - beta2)*dW1*dW1
148.             sW2 = beta2*sW2 + (1 - beta2)*dW2*dW2
149.             sb = beta2*sb + (1 - beta2)*db*db
150.             vW1_hat = vW1/(1 - beta1**k)
151.             vW2_hat = vW2/(1 - beta1**k)
152.             vb_hat = vb/(1 - beta1**k)
153.             sW1_hat = sW1/(1 - beta2**k)
154.             sW2_hat = sW2/(1 - beta2**k)
155.             sb_hat = sb/(1 - beta2**k)
156.             new_dW1 = learning_rate*vW1_hat/(np.sqrt(sW1_hat)+TOL)
157.             new_dW2 = learning_rate*vW2_hat/(np.sqrt(sW2_hat)+TOL)
158.             new_db = learning_rate*vb_hat/(np.sqrt(sb_hat)+TOL)
159.             W1 = W1 - new_dW1
160.             W2 = W2 - new_dW2
161.             b = b - new_db
162.         Layer = forward(W1, W2, b, train_size, train, function_name = 'ReLU')
163.         result = ANN(W1, W2, b, test, function_name = 'ReLU')
164.         y.append(float(loss(train_size, Layer, train, loss_name ='MSE')))
165.         print(k)
166.         print(y[k-1])
167.         if result[1] > best_result:
168.             best_result = result[1]
169.             bestW1 = W1
170.             bestW2 = W2
171.             bestb = b
172.         k = k + 1
173.     return bestW1, bestW2, bestb, y
174.
175. def ANN(W1, W2, b, data, function_name = 'ReLU'):
176.     M = len(data)
177.     output = forward(W1, W2, b, M, data, function_name)
178.     ans = []
179.     correct = 0
180.     for i in range(M):
181.         if output[2][i] < 0:
182.             ans.append(-1)
183.         else:
184.             ans.append(1)
185.
186.         if ans[i] == data[i].z:
187.             correct = correct + 1
188.     return [ans, correct / M]
189.
190. def GenerateWeight(Hidden_nodes):
191.     W1 = np.random.rand(Hidden_nodes, 2) #權重矩陣W1和W2
192.     W2 = np.random.rand(1,Hidden_nodes)
193.     b = np.random.rand(Hidden_nodes,1) #偏置向量b
194.     return W1, W2, b
195.
196. def GenerateTestData(M):
197.     temp = []
198.     test_data = []
199.     for i in range(M):
200.         r = np.random.randint(0, 2)
201.         if r == 1:
202.             temp.append(Circle_data.Data("blue"))
203.         else:
204.             temp.append(Circle_data.Data("red"))
205.         test_data.append(Data(temp[i].x, temp[i].z))
206.     return test_data
207.
208. def main():
210.     N = len(data)  #取得資料的個數
211.     Hidden_nodes = 4
212.     t = np.arange(100)
213.     """
214.    learning_rate = 0.3
215.    beta = 0.1
216.    gamma = 0.9
217.    beta1 = 0.9
218.    beta2 = 0.999
220.    function_name = 'ReLU'
221.    loss_name ='MSE'
222.    train_size = 20000
223.    """
224.     W1, W2, b = GenerateWeight(Hidden_nodes)
225.     W1_1, W2_1, b_1, y_1 = Train(W1, W2, b, data, Hidden_nodes, N, learning_rate = 0.01, Method = 'GradientDescent')
226.     W1_2, W2_2, b_2, y_2 = Train(W1, W2, b, data, Hidden_nodes, N, learning_rate = 0.01, Method = 'Momentum')
227.     W1_3, W2_3, b_3, y_3 = Train(W1, W2, b, data, Hidden_nodes, N, learning_rate = 0.01, Method = 'Adagrad')
228.     W1_4, W2_4, b_4, y_4 = Train(W1, W2, b, data, Hidden_nodes, N, learning_rate = 0.01, Method = 'RMSProp')
229.     W1_5, W2_5, b_5, y_5 = Train(W1, W2, b, data, Hidden_nodes, N, learning_rate = 0.01, Method = 'Adam')
230.
231.     plt.plot(t, y_1, 'b-', t, y_2, 'r-', t, y_3, 'g-', t, y_4, 'k-', t, y_5, 'y-')
232.     plt.show()
233.     plt.savefig('plot1.png')
234.
235.     M = 1000
236.     test_data = GenerateTestData(M)
237.
238.     result = ANN(W1_1, W2_1, b_1, test_data)
239.     print("命中率=",100*result[1],'%')
240.     result = ANN(W1_2, W2_2, b_2, test_data)
241.     print("命中率=",100*result[1],'%')
242.     result = ANN(W1_3, W2_3, b_3, test_data)
243.     print("命中率=",100*result[1],'%')
244.     result = ANN(W1_4, W2_4, b_4, test_data)
245.     print("命中率=",100*result[1],'%')
246.     result = ANN(W1_5, W2_5, b_5, test_data)
247.     print("命中率=",100*result[1],'%')
248.
249.
250. if __name__ == "__main__":
251.     main()
RAW Paste Data