Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- #Understanding neural networks
- #Random approach
- def forward_multiply_gate(x,y):
- return x*y
- x = -2
- y = 3
- tweak_amount = 0.01
- import random
- best_out = -999999999999999999
- for var in range(100):
- x_try = x + tweak_amount*(random.uniform(0, 1)*2 -1)
- y_try = y + tweak_amount*(random.uniform(0,1)*2-1)
- out = forward_multiply_gate(x_try,y_try)
- if (out > best_out):
- best_out = out
- best_x = x_try
- best_y = y_try
- print (x_try)
- print (y_try)
- print (best_out)
- strategy approach
- compute dervivative
- out = forward_multiply_gate(x,y)
- h = 0.001
- x_h = x + h
- out2 = forward_multiply_gate(x_h,y)
- x_derivative = (out2-out)/h
- y_h = y + h
- out3 = forward_multiply_gate(x,y_h)
- y_derivative = (out3-out)/h
- print (x_derivative)
- print (y_derivative)
- for var in range(100):
- x_try = x + tweak_amount*x_derivative
- y_try = y + tweak_amount*y_derivative
- out = forward_multiply_gate(x_try,y_try)
- if (out > best_out):
- best_out = out
- best_x = x_try
- best_y = y_try
- print (x_try)
- print (y_try)
- print (best_out)
- x_try = x + tweak_amount*y
- y_try = y + tweak_amount*x
- print (forward_multiply_gate(x_try,y_try))
- def forward_multiply_gate(x,y):
- return x*y
- def forward_add_gate(x,y):
- return x+y
- def forward_circuit(x,y,z):
- q = forward_add_gate(x,y)
- out = forward_multiply_gate(q,z)
- return out
- x = -2
- y = 5
- z = -4
- q = forward_add_gate(x,y)
- f = forward_circuit(x,y,z)
- derivative_f_wrt_z = q
- derivative_f_wrt_q = z
- derivative_q_wrt_y = 1.0
- derivative_q_wrt_x = 1.0
- derivative_f_wrt_x = derivative_q_wrt_x * derivative_f_wrt_q
- derivative_f_wrt_y = derivative_q_wrt_y * derivative_f_wrt_q
- # print derivative_f_wrt_x
- # print derivative_f_wrt_y
- # print derivative_f_wrt_z
- step_zize = 0.01
- x = x + step_zize*derivative_f_wrt_x
- y = y + step_zize*derivative_f_wrt_y
- z = z + step_zize*derivative_f_wrt_z
- q = forward_add_gate(x,y)
- f = forward_multiply_gate(q,z)
- print(q)
- print(f)
- print forward_circuit(x,y,z)
- print forward_circuit(x,y,z)
- print forward_circuit(x_try,y_try,z_try)
- numerical derivative
- x,y,z = -2,5,-4
- h = 0.0001
- x_derivative = (forward_circuit(x+h,y,z)-forward_circuit(x,y,z))/h
- y_derivative = (forward_circuit(x,y+h,z)-forward_circuit(x,y,z))/h
- z_derivative = (forward_circuit(x,y,z+h)-forward_circuit(x,y,z))/h
- print x_derivative
- print y_derivative
- print z_derivative
- computing analytical derivative properly has made mme correct the function
- import math
- def sigmoid(x):
- return 1 / (1 + math.exp(-x))
- class unit():
- def __init__(self,value,grad):
- self.value = value
- self.grad = grad
- class multiply_gate():
- def forward(self,u0,u1):
- self.u0 = u0
- self.u1 = u1
- self.utop = unit(u0.value*u1.value,0.0)
- return self.utop
- def backward(self):
- self.u0.grad += self.u1.grad*self.utop.grad
- self.u1.grad += self.u0.grad*self.utop.grad
- class add_gate():
- def forward(self,u0,u1):
- self.u0 = u0
- self.u1 = u1
- self.utop = unit(u0.value+u1.value,0.0)
- return self.utop
- def backward(self):
- self.u0.grad += 1*self.utop.grad
- self.u1.grad += 1*self.utop.grad
- class sigmoid_gate():
- def forward(self,u0):
- self.u0 = u0
- self.utop = unit(sigmoid(self.u0.value),0.0)
- return self.utop
- def backward(self):
- s = sigmoid(self.u0.value)
- self.u0.grad += s*(1-s)*self.utop.grad
- a = unit(1,0)
- b = unit(2,0)
- c = unit(-3,0)
- x = unit(-1,0)
- y = unit(3,0)
- mulg0 = multiply_gate()
- mulg1 = multiply_gate()
- addg0 = add_gate()
- addg1 = add_gate()
- sg0 = sigmoid_gate()
- ax = mulg0.forward(a,x)
- by = mulg1.forward(b,y)
- axpby = addg0.forward(ax,by)
- axpbypc = addg1.forward(axpby,c)
- s = sg0.forward(axpbypc)
- # print s.value
- # print s.grad
- s.grad = 1.0
- sg0.backward()
- addg1.backward()
- addg0.backward()
- mulg1.backward()
- mulg0.backward()
- print a.grad
- print b.grad
- print c.grad
- print x.grad
- print y.grad
- step_zize = 0.01
- a.value += step_zize*a.grad
- b.value += step_zize*b.grad
Add Comment
Please, Sign In to add comment