Guest User

Untitled

a guest
Mar 10th, 2018
141
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
Oz 1.66 KB | None | 0 0
  1. declare
  2. [File]={Module.link ['File.ozf']}
  3. local
  4.    fun {Sanitize2 Source}
  5.       case Source
  6.       of nil then nil
  7.       [] & |& |T then & |{Sanitize2 T}
  8.       [] H|T then H|{Sanitize2 T}
  9.       end
  10.    end
  11.  
  12.    fun {Tokenizer Source}
  13.       {Tokenize1 {String.tokens {Sanitize2 {Sanitize2 {Sanitize1 Source}}} & }}
  14.    end
  15.  
  16.    fun {Tokenize1 Source}
  17.       case Source
  18.       of nil then nil
  19.       [] H|T then
  20.      if {String.isInt H} then {String.toInt H}|{Tokenize1 T}
  21.      elseif {String.isFloat H} then {String.toFloat H}|{Tokenize1 T}
  22.      else {String.toAtom H}|{Tokenize1 T}
  23.      end
  24.       end
  25.    end
  26.  
  27.    fun {Sanitize1 Source}
  28.       case Source
  29.       of nil then nil
  30.       [] &\r|T then {Sanitize1 T}
  31.       [] &\n|T then & |{Sanitize1 T}
  32.       [] &,|T then & |&,|& |{Sanitize1 T}
  33.       [] &SEMI>|T then & |&SEMI>|& |{Sanitize1 T}
  34.       [] &{|T then & |&{|& |{Sanitize1 T}
  35.       [] &}|T then & |&}|& |{Sanitize1 T}
  36.       [] &(|T then & |&(|& |{Sanitize1 T}
  37.       [] &)|T then & |&)|& |{Sanitize1 T}
  38.       [] &*|T then & |&*|& |{Sanitize1 T}
  39.       [] &/|T then & |&/|& |{Sanitize1 T}
  40.       [] &+|T then & |&+|& |{Sanitize1 T}
  41.       [] &-|T then & |&-|& |{Sanitize1 T}
  42.       [] &!|&=|T then & |&!|&=|& |{Sanitize1 T}
  43.       [] &<|&=|T then & |&<|&=|& |{Sanitize1 T}
  44.       [] &=|&=|T then & |&=|&=|& |{Sanitize1 T}
  45.       [] &>|&=|T then & |&>|&=|& |{Sanitize1 T}
  46.       [] &=|T then & |&=|& |{Sanitize1 T}
  47.       [] &<|T then & |&<|& |{Sanitize1 T}
  48.       [] &>|T then & |&>|& |{Sanitize1 T}
  49.       [] H|T then H|{Sanitize1 T}
  50.       end
  51.    end
  52. in
  53.    fun {Tokenize Source}
  54.       {Tokenizer Source}
  55.    end
  56. end
  57. Content = {File.readList "foo.txt"}
  58. {Browse {Tokenize Content}}
Add Comment
Please, Sign In to add comment