scaner

import sys
import ply.lex as lex

reserved = {
    'zeros' : 'ZEROS',
    'ones' : 'ONES',
    'eye' : 'EYE',
    'while' :'WHILE',
    'if' : 'IF',
    'else' : 'ELSE',
    'for' : 'FOR',
    'continue' : 'CONTINUE',
    'return' : 'RETURN',
    'break' : 'BREAK',
    'print' : 'PRINT'
}

literals = [ '(',')',';','{','}','[',']',':',',']


tokens = [
    'PLUS',
    'MINUS',
    'TIMES',
    'DIVIDE',
    'ID',
    'INT',
    'FLOAT',
    'ASSIGN',
    'LESS',
    'MORE',
    'EQUAL',
    'IN_EQUAL',
    'MORE_EQUAL',
    'LESS_EQUAL',
    'DOTPLUS',
    'DOTTIMES',
    'DOTMINUS',
    'DOTDIVIDE',
    'TRANS',
    'DIVASSIGN',
    'PLUSASSIGN',
    'MULASSIGN',
    'MINUSASSIGN',
    'ASSIGN',
    'QUAT_MARKS'
]

tokens += reserved.values()


t_PLUS = r'\+'
t_MINUS = r'-'
t_TIMES = r'\*'
t_DIVIDE = r'/'
t_TRANS = r'\''
t_EQUAL = r'=='
t_LESS = r'<'
t_MORE = r'>'
t_DOTPLUS = r'\.\+'
t_DOTMINUS = r'\.-'
t_DOTTIMES = r'\.\*'
t_DOTDIVIDE = r'\./'
t_LESS_EQUAL = r'<='
t_MORE_EQUAL = r'>='
t_IN_EQUAL = r'!='
t_DIVASSIGN = r'/='
t_PLUSASSIGN = r'\+='
t_MINUSASSIGN = r'\-='
t_MULASSIGN = r'\*='
t_ASSIGN = r'='
t_QUAT_MARKS = r'"'


t_ignore = ' \t'
t_ignore_COMMENT = r'\#.*'

def t_ID(t):
    r'[a-zA-Z_]\w*'
    if t.value in reserved:
        t.type = reserved[t.value]
    return t


def t_FLOAT( t):
    r'\d*\.\d+'
    t.value = float(t.value)
    return t

def t_INT( t):
    r'\d+'
    t.value = int(t.value)
    return t


def t_error(t):
    print("line %d: illegal character '%s'" %(t.lineno, t.value[0]) )
    t.lexer.skip(1)


def t_newline(t):
    r'\n+'
    t.lexer.lineno += len(t.value)

try:
    filename = sys.argv[1] if len(sys.argv) > 1 else "example.txt"
    file = open(filename, "r")
except IOError:
    print("Cannot open {0} file".format(filename))
    sys.exit(0)

lexer = lex.lex()
text = file.read()
lexer.input(text)  # Give the lexer some input

    # Tokenize
while True:
   tok = lexer.token()
   if not tok:
        break  # No more input
   print("(%d) : %s(%s)" % (tok.lineno, tok.type, tok.value))