Advertisement
Guest User

Untitled

a guest
Sep 20th, 2019
121
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 0.51 KB | None | 0 0
  1. class RobertaTokenizeProcessor(TokenizeProcessor):
  2. def __init__(self, tokenizer):
  3. super().__init__(tokenizer=tokenizer, include_bos=False, include_eos=False)
  4.  
  5. class RobertaNumericalizeProcessor(NumericalizeProcessor):
  6. def __init__(self, *args, **kwargs):
  7. super().__init__(*args, vocab=fastai_roberta_vocab, **kwargs)
  8.  
  9. def get_roberta_processor(tokenizer:Tokenizer=None, vocab:Vocab=None):
  10. return [RobertaTokenizeProcessor(tokenizer=tokenizer), NumericalizeProcessor(vocab=vocab)]
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement