Advertisement
osipyonok

Untitled

Jan 16th, 2017
184
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 4.50 KB | None | 0 0
  1. import java.io.File;
  2. import java.io.FileInputStream;
  3. import java.io.FileNotFoundException;
  4. import java.io.IOException;
  5. import java.util.*;
  6. import java.util.regex.*;
  7.  
  8. enum Tokens {
  9. Keyword,
  10. Atom,
  11. Int,
  12. Hex,
  13. Double,
  14. Comment,
  15. String,
  16. Char,
  17. Directive,
  18. Operator,
  19. Punctuation
  20. }
  21. /////////////////////////// Main.java
  22.  
  23.  
  24. public class Main {
  25. static Map<Tokens , Pattern> regular = new HashMap<Tokens , Pattern>();
  26. static Map<Tokens , Integer> priority = new HashMap<Tokens , Integer>();
  27.  
  28. static void InitRegex(){
  29. regular.put(Tokens.Atom , Pattern.compile("^(_|[a-z])([a-z]|[0-9]|-)*"));
  30. regular.put(Tokens.Keyword , Pattern.compile("^(eq|car|cdr|defun|defvar|setq|and|not|or|xor|atom|equal|nil|t|defmacro|cond|if|when|case|loop|do|return|let|list|do|length|setf|format|terpri|dotimes|write-char)"));
  31. regular.put(Tokens.Int , Pattern.compile("^(0|[1-9]\\d*)"));
  32. regular.put(Tokens.String, Pattern.compile("^((\"\"([^\\\"\"\\n\\r\\\\]*|\\\\[\\\\0abfnrtuUxv]|\\\\x[\\da-fA-F]{1,4}|\\\\u[\\da-fA-F]{4}|\\\\U[\\da-fA-F]{8})*\"\")|(@\"\"([^\\\"\"]|\"\"\"\")*\"\"))"));
  33. regular.put(Tokens.Comment, Pattern.compile("^(;+[^\n\r]*)"));
  34. regular.put(Tokens.Punctuation, Pattern.compile("^[():]"));
  35. regular.put(Tokens.Hex, Pattern.compile("^0[xX]((0|[1-9a-fA-F][\\da-fA-F]*))"));
  36. regular.put(Tokens.Operator, Pattern.compile("^\\+ |\\- |\\* |mod |rem |incf |decf |= |> |< |>= |<= |max |min "));
  37. regular.put(Tokens.Double, Pattern.compile("^(((0|[1-9]\\d*)?\\.\\d+([eE][+-]?\\d+)?[FfDdMm]?)|((0|[1-9]\\d*)([eE][+-]?\\d+)[FfDdMm]?)|((0|[1-9]\\d*)[FfDdMm]))"));
  38. regular.put(Tokens.Directive, Pattern.compile("^mp:.*"));
  39. regular.put(Tokens.Char , Pattern.compile("^'[a-z]*"));
  40. }
  41.  
  42. static void InitPriority(){
  43. priority.put(Tokens.Operator, 1);
  44. priority.put(Tokens.Punctuation, 2);
  45. priority.put(Tokens.Double, 3);
  46. priority.put(Tokens.Int, 4);
  47. priority.put(Tokens.Hex, 4);
  48. priority.put(Tokens.Atom, 5);
  49. priority.put(Tokens.Keyword, 6);
  50. priority.put(Tokens.String, 7);
  51. priority.put(Tokens.Char, 7);
  52. priority.put(Tokens.Directive, 8);
  53. priority.put(Tokens.Comment, 9);
  54. }
  55.  
  56. static void Tokenize(String text){
  57. String code = text;
  58. while(code.length() > 0 && (code.charAt(0) == ' ' || code.charAt(0) == '\n' || code.charAt(0) == '\t'))code = code.substring(1);
  59. while(code.length() > 0){
  60. Tokens tokType = Tokens.Comment;
  61. int len = -1;
  62. for(Tokens tok : regular.keySet()){
  63. Pattern trg = regular.get(tok);
  64. int t = 1 , tmax = 0;
  65. Matcher tmat = trg.matcher(code.substring(0, t));
  66. while(t <= code.length()){
  67. Matcher m = trg.matcher(code.substring(0, t));
  68. if(m.matches() == false){
  69. if(t > 20)break;
  70. ++t;
  71. continue;
  72. }
  73. tmat = m;
  74. tmax = t;
  75. ++t;
  76.  
  77. if(t == code.length()){
  78. break;
  79. }
  80. }
  81. if(tmat.matches() == false)continue;
  82. int tlen = tmax;
  83. if(tlen > len || (tlen == len && priority.get(tokType) < priority.get(tok))){
  84. tokType = tok;
  85. len = tlen;
  86. }
  87. }
  88. if(len <= 0){
  89. System.out.println("Unknown lexeme " + code);
  90. return;
  91. }
  92. System.out.println(code.substring(0 , len) + " " + tokType.toString());
  93. code = code.substring(len);
  94. code = code.trim();
  95. }
  96. }
  97.  
  98. public static void main(String[] args) {
  99. InitRegex();
  100. InitPriority();
  101. File file = new File("input.txt");
  102. try {
  103. FileInputStream fis = new FileInputStream(file);
  104. byte[] data = new byte[(int) file.length()];
  105. fis.read(data);
  106. fis.close();
  107. fis.close();
  108. String str = new String(data, "UTF-8");
  109. Tokenize(str);
  110. }catch(FileNotFoundException e){
  111. System.out.print("File not found");
  112. }catch(IOException e) {
  113. e.printStackTrace();
  114. }
  115. }
  116. }
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement