Advertisement
Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- def tokenize(line):
- """Convert a string into a list of tokens.
- >>> tokenize('2.3')
- [2.3]
- >>> tokenize('(2 3)')
- ['(', 2, 3, ')']
- >>> tokenize('<2 3)')
- ['<', 2, 3, ')']
- >>> tokenize('<[2{12.5 6.0}](3 -4 5)>')
- ['<', '[', 2, '{', 12.5, 6.0, '}', ']', '(', 3, -4, 5, ')', '>']
- >>> tokenize('2.3.4')
- Traceback (most recent call last):
- ...
- ValueError: invalid token 2.3.4
- >>> tokenize('?')
- Traceback (most recent call last):
- ...
- ValueError: invalid token ?
- >>> tokenize('hello')
- Traceback (most recent call last):
- ...
- ValueError: invalid token hello
- >>> tokenize('<(GO BEARS)>')
- Traceback (most recent call last):
- ...
- ValueError: invalid token GO
- """
- # Surround all brackets by spaces so that they are separated by split.
- for b in all_brackets:
- line = line.replace(b, ' ' + b + ' ')
- # Convert numerals to numbers and raise ValueErrors for invalid tokens.
- tokens = []
- for t in line.split():
- if t in all_brackets:
- tokens.append(t)
- elif coerce_to_number(t) != None:
- tokens.append(coerce_to_number(t))
- else:
- raise ValueError('invalid token %s' % (t) )
- return tokens
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement