Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- declare
- [File]={Module.link ['File.ozf']}
- local
- fun {Tokenize Source}
- try
- {TokenizeHelper {Sanitize Source}}
- catch E then
- {String.toAtom {Append E " must start with a lowercase letter."}}
- end
- end
- fun {TokenizeHelper Source}
- case Source
- of nil then nil
- [] H|T then
- if {String.isInt H} then {String.toInt H}|{TokenizeHelper T}
- elseif {String.isFloat H} then {String.toFloat H}|{TokenizeHelper T}
- elseif {Char.isUpper H.1} then raise H end
- elseif {Char.isDigit H.1} then raise H end
- else {String.toAtom H}|{TokenizeHelper T}
- end
- end
- end
- fun {Sanitize Source}
- {String.tokens {RemoveExtraSpaces {SpaceEverything Source} 0} & }
- end
- fun {SpaceEverything Source}
- case Source
- of nil then nil
- [] &\r|T then {SpaceEverything T}
- [] &\n|T then & |{SpaceEverything T}
- [] &\t|T then & |{SpaceEverything T}
- [] &,|T then & |&,|& |{SpaceEverything T}
- [] &SEMI>|T then & |&SEMI>|& |{SpaceEverything T}
- [] &{|T then & |&{|& |{SpaceEverything T}
- [] &}|T then & |&}|& |{SpaceEverything T}
- [] &(|T then & |&(|& |{SpaceEverything T}
- [] &)|T then & |&)|& |{SpaceEverything T}
- [] &*|T then & |&*|& |{SpaceEverything T}
- [] &/|T then & |&/|& |{SpaceEverything T}
- [] &+|T then & |&+|& |{SpaceEverything T}
- [] &-|T then & |&-|& |{SpaceEverything T}
- [] &!|&=|T then & |&!|&=|& |{SpaceEverything T}
- [] &<|&=|T then & |&<|&=|& |{SpaceEverything T}
- [] &=|&=|T then & |&=|&=|& |{SpaceEverything T}
- [] &>|&=|T then & |&>|&=|& |{SpaceEverything T}
- [] &=|T then & |&=|& |{SpaceEverything T}
- [] &<|T then & |&<|& |{SpaceEverything T}
- [] &>|T then & |&>|& |{SpaceEverything T}
- [] H|T then H|{SpaceEverything T}
- end
- end
- fun {RemoveExtraSpaces Source SpacesMode}
- case Source
- of nil then nil
- [] & |T then
- if SpacesMode == 0 then & |{RemoveExtraSpaces T 1}
- else {RemoveExtraSpaces T 1}
- end
- [] H|T then H|{RemoveExtraSpaces T 0}
- end
- end
- in
- fun {Tokenizer Source}
- {Tokenize Source}
- end
- end
- Content = {File.readList "foo.txt"}
- {Browse {Tokenizer Content}}
Add Comment
Please, Sign In to add comment