Advertisement
Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- class RobertaTokenizeProcessor(TokenizeProcessor):
- def __init__(self, tokenizer):
- super().__init__(tokenizer=tokenizer, include_bos=False, include_eos=False)
- class RobertaNumericalizeProcessor(NumericalizeProcessor):
- def __init__(self, *args, **kwargs):
- super().__init__(*args, vocab=fastai_roberta_vocab, **kwargs)
- def get_roberta_processor(tokenizer:Tokenizer=None, vocab:Vocab=None):
- return [RobertaTokenizeProcessor(tokenizer=tokenizer), NumericalizeProcessor(vocab=vocab)]
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement