Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- declare
- [File]={Module.link ['File.ozf']}
- local
- fun {Sanitize2 Source}
- case Source
- of nil then nil
- [] & |& |T then & |{Sanitize2 T}
- [] H|T then H|{Sanitize2 T}
- end
- end
- fun {Tokenizer Source}
- {Tokenize1 {String.tokens {Sanitize2 {Sanitize2 {Sanitize1 Source}}} & }}
- end
- fun {Tokenize1 Source}
- case Source
- of nil then nil
- [] H|T then
- if {String.isInt H} then {String.toInt H}|{Tokenize1 T}
- elseif {String.isFloat H} then {String.toFloat H}|{Tokenize1 T}
- else {String.toAtom H}|{Tokenize1 T}
- end
- end
- end
- fun {Sanitize1 Source}
- case Source
- of nil then nil
- [] &\r|T then {Sanitize1 T}
- [] &\n|T then & |{Sanitize1 T}
- [] &,|T then & |&,|& |{Sanitize1 T}
- [] &SEMI>|T then & |&SEMI>|& |{Sanitize1 T}
- [] &{|T then & |&{|& |{Sanitize1 T}
- [] &}|T then & |&}|& |{Sanitize1 T}
- [] &(|T then & |&(|& |{Sanitize1 T}
- [] &)|T then & |&)|& |{Sanitize1 T}
- [] &*|T then & |&*|& |{Sanitize1 T}
- [] &/|T then & |&/|& |{Sanitize1 T}
- [] &+|T then & |&+|& |{Sanitize1 T}
- [] &-|T then & |&-|& |{Sanitize1 T}
- [] &!|&=|T then & |&!|&=|& |{Sanitize1 T}
- [] &<|&=|T then & |&<|&=|& |{Sanitize1 T}
- [] &=|&=|T then & |&=|&=|& |{Sanitize1 T}
- [] &>|&=|T then & |&>|&=|& |{Sanitize1 T}
- [] &=|T then & |&=|& |{Sanitize1 T}
- [] &<|T then & |&<|& |{Sanitize1 T}
- [] &>|T then & |&>|& |{Sanitize1 T}
- [] H|T then H|{Sanitize1 T}
- end
- end
- in
- fun {Tokenize Source}
- {Tokenizer Source}
- end
- end
- Content = {File.readList "foo.txt"}
- {Browse {Tokenize Content}}
Add Comment
Please, Sign In to add comment