class
	{
	New(fileName)
		{

		.file = File(fileName,'r')

		.currentLine = 0
		.wordsInLine = #[]
		.positionInLine = 0
		.currentWord = ""
		.token = ""

		.tokensTypes = #["KEYWORD","SYMBOL","IDENTIFIER",
						"INT_CONST","STRING_CONST","ERROR"]
		.comments = "(^//.*)|\*"
		.symbol = "(.*)(\\{|\\}|\\(|\\)|\\[|\\]|\\.|\\,|;|\\+|-|\\*|/|&|\\||<|>|=|~)(.*)"
		.keyword = "^(class|constructor|function|method|field|static|var|int|
			char|boolean|void|true|false|null|this|let|do|if|else|while|return)$"
		.idetifier = "^[^\\d\\W]\\w*\\Z"
		.int = "^[\d]+$"
		.str = "^[\"][\w]"
		.current = #["",""]
		.tokens = Object()
		}
	Token()
		{
		return .token
		}
	TypeToEnum(str)
		{
		for	(i = 0; i < .tokensTypes.Size(); i++)
			if .tokensTypes[i] is str
				return i
		return -1
		}

	SetCurrentLine()
		{
		do{
			.currentLine = .file.Readline()
			.currentLine = .currentLine.LeftTrim()
			}while .currentLine.Blank?() or .isComment()
		if .currentLine =~ "\""
			.replaceSpaces()
		if .currentLine =~ .symbol
			.addSpaceToSymbol()
		Print(.currentLine)
		.wordsInLine = .currentLine.Trim().Tr(" \t", " ").Split(" ")
		}

	replaceSpaces()
		{
		inString = false
		for (i = 0; i < .currentLine.Size(); ++i)
			{
			if .currentLine[i] is "\""
				inString = not inString
			if .currentLine[i] is " " and inString
				.currentLine = .currentLine.ReplaceSubstr(i,1,"_")
			}
		}

	addSpaceToSymbol()
		{
			symbols = #["[","{","}","(",")","[","]",".",",",";","+",
						"-","*","&","|","<",">","=","~","]"]
			for sym in symbols
				if .currentLine.Has?(sym)
					{
					if .special(sym)
						.currentLine = .currentLine.Replace("\\" $ sym," "$ "\\" $ sym $ " ")
					else
						.currentLine = .currentLine.Replace(sym," "$ sym $ " ")
					}
		}

	special(sym)
		{
		specials = #["[","]","(",")","."]
		for s in specials
			{
			if sym is s
				return true
			}
		return false
		}

	isComment()
		{
			return .currentLine =~ .comments
		}
	CommentsRemover(code)
		{
		lines = code.Lines()
		result = ""
		for (i = 0; i < lines.Size(); ++i){
			if(lines[i] is "")
				{
				continue;
				}
			line = lines[i].Split(" ")
			line = line.LeftTrim()
			if line[0] is '/' and line[1] is '/'
				{
				continue;
				}
			result $= lines[i]
		}
		return result
	  }

	hasMoreTokens()
		{
		x = .file.Tell()
		i = .file.Readline()
		if (i is false and .positionInLine >= .wordsInLine.Size()){
			.file.Close()
			return false
			}
		.file.Seek(x,"set")
		return true
		}
	Advance()
		{
		if (not .hasMoreTokens()) { return; }

		if (.positionInLine >= .wordsInLine.Size()) {
		   .SetCurrentLine()
		   .positionInLine = 0
	   }
	   .token = .wordsInLine[.positionInLine++];
		}

	TokenType()
		{
		if (.token =~ .keyword) { return .TypeToEnum("KEYWORD") }
		else if (.token =~ .symbol) { return .TypeToEnum("SYMBOL") }
		else if (.token =~ .idetifier) { return .TypeToEnum("IDENTIFIER") }
		else if (.token =~ .int) { return .TypeToEnum("INT_CONST") }
		else if (.token =~ .str) { return .TypeToEnum("STRING_CONST") }
		else {
			Print(.token $ " ERR")
			 return .TypeToEnum("ERROR") }
		}
	KeyWord()
		{
		return .current[0].Upper()
		}
	Symbol()
		{
		 if (.token is "<") { return "&lt;" }
		else if (.token is ">") { return "&gt;";}
		else if (.token is "&") { return "&amp;" }
		else return .token
		}


	StringVal()
		{
		.token = .token.Replace("_"," ")
		return .token
		}
	Xml()
		{
		result = "<tokens>\n"
		while (.hasMoreTokens()) {
			.Advance();
			type = .TokenType()
			switch type {
				case .TypeToEnum("KEYWORD"):
					result $= Xml("keyword"," " $ .token $ " ") $ "\n"
				case .TypeToEnum("SYMBOL"):
					result $= Xml("symbol"," " $ .Symbol() $ " ") $ "\n"
				case .TypeToEnum("IDENTIFIER"):
					result $= Xml("identifier", " " $ .token $ " ") $ "\n"
				case .TypeToEnum("INT_CONST"):
					result $= Xml("integerConstant", " " $ .token $ " ") $ "\n"
				case .TypeToEnum("STRING_CONST"):
					result $= Xml("stringConstant", " " $ .StringVal() $ " ") $ "\n"
				default:
					 result $= Xml("error"," ERROR ") $ "\n"
				}
			}
		return result $ "</tokens>"
		}
	}