Advertisement
Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- def processNlToks(nlToks):
- return [tok.encode('ascii', 'replace').decode().strip() for tok in nlToks \
- if tok != "-RCB-" and \
- tok != "-LCB-" and \
- tok != "-LSB-" and \
- tok != "-RSB-" and \
- tok != "-LRB-" and \
- tok != "-RRB-" and \
- tok != "@link" and \
- tok != "@code" and \
- tok != "@inheritDoc" and \
- tok.encode('ascii', 'replace').decode().strip() != '']
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement