Advertisement
Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- class Softmax:
- # ...
- def backprop(self, d_L_d_out):
- '''
- Performs a backward pass of the softmax layer.
- Returns the loss gradient for this layer's inputs.
- - d_L_d_out is the loss gradient for this layer's outputs.
- '''
- # We know only 1 element of d_L_d_out will be nonzero
- for i, gradient in enumerate(d_L_d_out):
- if gradient == 0:
- continue
- # e^totals
- t_exp = np.exp(self.last_totals)
- # Sum of all e^totals
- S = np.sum(t_exp)
- # Gradients of out[i] against totals
- d_out_d_t = -t_exp[i] * t_exp / (S ** 2)
- d_out_d_t[i] = t_exp[i] * (S - t_exp[i]) / (S ** 2)
- # Gradients of totals against weights/biases/input
- d_t_d_w = self.last_input
- d_t_d_b = 1
- d_t_d_inputs = self.weights
- # Gradients of loss against totals
- d_L_d_t = gradient * d_out_d_t
- # Gradients of loss against weights/biases/input
- d_L_d_w = d_t_d_w[np.newaxis].T @ d_L_d_t[np.newaxis]
- d_L_d_b = d_L_d_t * d_t_d_b
- d_L_d_inputs = d_t_d_inputs @ d_L_d_t
- # ... to be continued
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement