Guest User

Untitled

a guest
Mar 10th, 2018
136
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
Oz 2.21 KB | None | 0 0
  1. declare
  2. [File]={Module.link ['File.ozf']}
  3. local
  4.    fun {Tokenize Source}
  5.       try
  6.      {TokenizeHelper {Sanitize Source}}
  7.       catch E then
  8.      {String.toAtom {Append E " must start with a lowercase letter."}}
  9.       end
  10.    end
  11.  
  12.    fun {TokenizeHelper Source}
  13.       case Source
  14.       of nil then nil
  15.       [] H|T then
  16.      if {String.isInt H} then {String.toInt H}|{TokenizeHelper T}
  17.      elseif {String.isFloat H} then {String.toFloat H}|{TokenizeHelper T}
  18.      elseif {Char.isUpper H.1} then raise H end
  19.      elseif {Char.isDigit H.1} then raise H end
  20.      else {String.toAtom H}|{TokenizeHelper T}
  21.      end
  22.       end
  23.    end
  24.  
  25.    fun {Sanitize Source}
  26.       {String.tokens {RemoveExtraSpaces {SpaceEverything Source} 0} & }
  27.    end
  28.  
  29.    fun {SpaceEverything Source}
  30.       case Source
  31.       of nil then nil
  32.       [] &\r|T then {SpaceEverything T}
  33.       [] &\n|T then & |{SpaceEverything T}
  34.       [] &\t|T then & |{SpaceEverything T}
  35.       [] &,|T then & |&,|& |{SpaceEverything T}
  36.       [] &SEMI>|T then & |&SEMI>|& |{SpaceEverything T}
  37.       [] &{|T then & |&{|& |{SpaceEverything T}
  38.       [] &}|T then & |&}|& |{SpaceEverything T}
  39.       [] &(|T then & |&(|& |{SpaceEverything T}
  40.       [] &)|T then & |&)|& |{SpaceEverything T}
  41.       [] &*|T then & |&*|& |{SpaceEverything T}
  42.       [] &/|T then & |&/|& |{SpaceEverything T}
  43.       [] &+|T then & |&+|& |{SpaceEverything T}
  44.       [] &-|T then & |&-|& |{SpaceEverything T}
  45.       [] &!|&=|T then & |&!|&=|& |{SpaceEverything T}
  46.       [] &<|&=|T then & |&<|&=|& |{SpaceEverything T}
  47.       [] &=|&=|T then & |&=|&=|& |{SpaceEverything T}
  48.       [] &>|&=|T then & |&>|&=|& |{SpaceEverything T}
  49.       [] &=|T then & |&=|& |{SpaceEverything T}
  50.       [] &<|T then & |&<|& |{SpaceEverything T}
  51.       [] &>|T then & |&>|& |{SpaceEverything T}
  52.       [] H|T then H|{SpaceEverything T}
  53.       end
  54.    end
  55.  
  56.    fun {RemoveExtraSpaces Source SpacesMode}
  57.       case Source
  58.       of nil then nil
  59.       [] & |T then
  60.      if SpacesMode == 0 then & |{RemoveExtraSpaces T 1}
  61.      else {RemoveExtraSpaces T 1}
  62.      end
  63.       [] H|T then H|{RemoveExtraSpaces T 0}
  64.       end
  65.    end
  66. in
  67.    fun {Tokenizer Source}
  68.       {Tokenize Source}
  69.    end
  70. end
  71. Content = {File.readList "foo.txt"}
  72. {Browse {Tokenizer Content}}
Add Comment
Please, Sign In to add comment