daily pastebin goal
52%
SHARE
TWEET

Untitled

a guest Jul 1st, 2010 631 Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
  1. #!/usr/bin/env python
  2.  
  3. new_token = lambda: { 'type': None, 'value': None, 'from': None, 'to': None, }
  4.  
  5. operators = set(['+', '*', '-'])
  6. values = set([1, 2, 3, 4, 5, 6, 8, 9, 0])
  7.  
  8. def tokenize(expression):
  9.     print "Tokenizing (%r)" % expression
  10.     print values
  11.     tokens, pos = [], 0
  12.     for item in expression.split(" "):
  13.         tokens.append(new_token())
  14.         if item in operators:
  15.             tokens[-1]['type'] = 'operator'
  16.         elif int(item) in values:
  17.             tokens[-1]['type'] = 'value'
  18.         else:
  19.             raise Exception("ERROR! (%r)" % item)
  20.  
  21.         tokens[-1]['value'] = item
  22.         tokens[-1]['from'] = pos
  23.         tokens[-1]['to'] = pos + len(item)
  24.         pos += len(item) + 1    # space
  25.  
  26.     return tokens
  27.  
  28. def tokenize_test(expression, tokens):
  29.     for token in tokens:
  30.         assert(' ' not in expression[token['from']:token['to']])
  31.  
  32. if __name__ == '__main__':
  33.     import sys
  34.     expression = sys.argv[1]
  35.     tokens = tokenize(expression)
  36.     for token in tokens:
  37.         print token
  38.     tokenize_test(expression, tokens)
RAW Paste Data
We use cookies for various purposes including analytics. By continuing to use Pastebin, you agree to our use of cookies as described in the Cookies Policy. OK, I Understand
 
Top