Advertisement
Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- def block(x, scope, *, past, hparams):
- with tf.variable_scope(scope):
- nx = x.shape[-1].value
- a, present = attn(norm(x, 'ln_1'), 'attn', nx, past=past, hparams=hparams)
- x = x + a
- m = mlp(norm(x, 'ln_2'), 'mlp', nx*4, hparams=hparams)
- x = x + m
- return x, present
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement