Untitled

#!/usr/bin/env python

new_token = lambda: { 'type': None, 'value': None, 'from': None, 'to': None, }

operators = set(['+', '*', '-'])
values = set([1, 2, 3, 4, 5, 6, 8, 9, 0])

def tokenize(expression):
    print "Tokenizing (%r)" % expression
    print values
    tokens, pos = [], 0
    for item in expression.split(" "):
        tokens.append(new_token())
        if item in operators:
            tokens[-1]['type'] = 'operator'
        elif int(item) in values:
            tokens[-1]['type'] = 'value'
        else:
            raise Exception("ERROR! (%r)" % item)

        tokens[-1]['value'] = item
        tokens[-1]['from'] = pos
        tokens[-1]['to'] = pos + len(item)
        pos += len(item) + 1    # space

    return tokens

def tokenize_test(expression, tokens):
    for token in tokens:
        assert(' ' not in expression[token['from']:token['to']])

if __name__ == '__main__':
    import sys
    expression = sys.argv[1]
    tokens = tokenize(expression)
    for token in tokens:
        print token
    tokenize_test(expression, tokens)