Advertisement
Guest User

Untitled

a guest
Nov 25th, 2014
171
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 1.29 KB | None | 0 0
  1. def tokenize(line):
  2. """Convert a string into a list of tokens.
  3.  
  4. >>> tokenize('2.3')
  5. [2.3]
  6. >>> tokenize('(2 3)')
  7. ['(', 2, 3, ')']
  8. >>> tokenize('<2 3)')
  9. ['<', 2, 3, ')']
  10. >>> tokenize('<[2{12.5 6.0}](3 -4 5)>')
  11. ['<', '[', 2, '{', 12.5, 6.0, '}', ']', '(', 3, -4, 5, ')', '>']
  12.  
  13. >>> tokenize('2.3.4')
  14. Traceback (most recent call last):
  15. ...
  16. ValueError: invalid token 2.3.4
  17.  
  18. >>> tokenize('?')
  19. Traceback (most recent call last):
  20. ...
  21. ValueError: invalid token ?
  22.  
  23. >>> tokenize('hello')
  24. Traceback (most recent call last):
  25. ...
  26. ValueError: invalid token hello
  27.  
  28. >>> tokenize('<(GO BEARS)>')
  29. Traceback (most recent call last):
  30. ...
  31. ValueError: invalid token GO
  32. """
  33. # Surround all brackets by spaces so that they are separated by split.
  34. for b in all_brackets:
  35. line = line.replace(b, ' ' + b + ' ')
  36.  
  37. # Convert numerals to numbers and raise ValueErrors for invalid tokens.
  38. tokens = []
  39. for t in line.split():
  40. if t in all_brackets:
  41. tokens.append(t)
  42. elif coerce_to_number(t) != None:
  43. tokens.append(coerce_to_number(t))
  44. else:
  45. raise ValueError('invalid token %s' % (t) )
  46. return tokens
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement