Advertisement
Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- #!/usr/bin/env python
- new_token = lambda: { 'type': None, 'value': None, 'from': None, 'to': None, }
- operators = set(['+', '*', '-'])
- values = set([1, 2, 3, 4, 5, 6, 8, 9, 0])
- def tokenize(expression):
- print "Tokenizing (%r)" % expression
- print values
- tokens, pos = [], 0
- for item in expression.split(" "):
- tokens.append(new_token())
- if item in operators:
- tokens[-1]['type'] = 'operator'
- elif int(item) in values:
- tokens[-1]['type'] = 'value'
- else:
- raise Exception("ERROR! (%r)" % item)
- tokens[-1]['value'] = item
- tokens[-1]['from'] = pos
- tokens[-1]['to'] = pos + len(item)
- pos += len(item) + 1 # space
- return tokens
- def tokenize_test(expression, tokens):
- for token in tokens:
- assert(' ' not in expression[token['from']:token['to']])
- if __name__ == '__main__':
- import sys
- expression = sys.argv[1]
- tokens = tokenize(expression)
- for token in tokens:
- print token
- tokenize_test(expression, tokens)
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement