Guest User

Untitled

a guest
Dec 15th, 2018
107
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 3.88 KB | None | 0 0
  1. def update_parameters(parameters, grads,learning_rate,iter_no,method = 'SGD',opt_parameters=None,beta1=0.9,beta2=0.999):
  2. """
  3. Update parameters using gradient descent
  4.  
  5. Arguments:
  6. parameters -- python dictionary containing your parameters
  7. grads -- python dictionary containing your gradients, output of L_model_backward
  8. method -- method for updation of weights
  9. 'SGD','SGDM','RMSP','ADAM'
  10. learning rate -- learning rate alpha value
  11. beta1 -- weighted avg parameter for SGDM and ADAM
  12. beta2 -- weighted avg parameter for RMSP and ADAM
  13.  
  14. Returns:
  15. parameters -- python dictionary containing your updated parameters
  16. parameters["W" + str(l)] = ...
  17. parameters["b" + str(l)] = ...
  18. opt_parameters
  19. """
  20.  
  21. L = len(parameters) // 2 # number of layers in the neural network
  22. if method == 'SGD':
  23. for l in range(L):
  24. parameters["W" + str(l+1)] = parameters["W" + str(l+1)] - learning_rate*grads["dW" + str(l + 1)]
  25. parameters["b" + str(l+1)] = parameters["b" + str(l+1)] - learning_rate*grads["db" + str(l + 1)]
  26. opt_parameters = None
  27. elif method == 'SGDM':
  28. for l in range(L):
  29. opt_parameters['vdb'+str(l+1)] = beta1*opt_parameters['vdb'+str(l+1)] + (1-beta1)*grads["db" + str(l + 1)]
  30. opt_parameters['vdw'+str(l+1)] = beta1*opt_parameters['vdw'+str(l+1)] + (1-beta1)*grads["dW" + str(l + 1)]
  31. parameters["W" + str(l+1)] = parameters["W" + str(l+1)] - learning_rate*opt_parameters['vdw'+str(l+1)]
  32. parameters["b" + str(l+1)] = parameters["b" + str(l+1)] - learning_rate*opt_parameters['vdb'+str(l+1)]
  33. elif method == 'RMSP':
  34. for l in range(L):
  35. opt_parameters['sdb'+str(l+1)] = beta2*opt_parameters['sdb'+str(l+1)] + \
  36. (1-beta2)*np.square(grads["db" + str(l + 1)])
  37. opt_parameters['sdw'+str(l+1)] = beta2*opt_parameters['sdw'+str(l+1)] + \
  38. (1-beta2)*np.square(grads["dW" + str(l + 1)])
  39. parameters["W" + str(l+1)] = parameters["W" + str(l+1)] - \
  40. learning_rate*(grads["dW" + str(l + 1)]/(np.sqrt(opt_parameters['sdw'+str(l+1)])+10**-8))
  41. parameters["b" + str(l+1)] = parameters["b" + str(l+1)] - \
  42. learning_rate*(grads["db" + str(l + 1)]/(np.sqrt(opt_parameters['sdb'+str(l+1)])+10**-8))
  43. elif method == 'ADAM':
  44. for l in range(L):
  45. opt_parameters['vdb'+str(l+1)] = beta1*opt_parameters['vdb'+str(l+1)] + (1-beta1)*grads["db" + str(l + 1)]
  46. opt_parameters['vdw'+str(l+1)] = beta1*opt_parameters['vdw'+str(l+1)] + (1-beta1)*grads["dW" + str(l + 1)]
  47. opt_parameters['sdb'+str(l+1)] = beta2*opt_parameters['sdb'+str(l+1)] + \
  48. (1-beta2)*np.square(grads["db" + str(l + 1)])
  49. opt_parameters['sdw'+str(l+1)] = beta2*opt_parameters['sdw'+str(l+1)] + \
  50. (1-beta2)*np.square(grads["dW" + str(l + 1)])
  51.  
  52. learning_rate = learning_rate * np.sqrt((1-beta2**iter_no)/((1-beta1**iter_no)+10**-8))
  53. parameters["W" + str(l+1)] = parameters["W" + str(l+1)] - \
  54. learning_rate*(opt_parameters['vdw'+str(l+1)]/\
  55. (np.sqrt(opt_parameters['sdw'+str(l+1)])+10**-8))
  56. parameters["b" + str(l+1)] = parameters["b" + str(l+1)] - \
  57. learning_rate*(opt_parameters['vdb'+str(l+1)]/\
  58. (np.sqrt(opt_parameters['sdb'+str(l+1)])+10**-8))
  59.  
  60. return parameters,opt_parameters
Add Comment
Please, Sign In to add comment