Advertisement
Guest User

Untitled

a guest
Apr 24th, 2017
74
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 3.43 KB | None | 0 0
  1. using System;
  2. using System.Collections.Generic;
  3. using System.Text;
  4. using System.Text.RegularExpressions;
  5.  
  6. namespace MyLexer
  7. {
  8. public class TokenDefinition
  9. {
  10. public string Name { get; set; }
  11.  
  12. public Regex Regex { get; set; }
  13.  
  14. public bool Ignore { get; set; }
  15.  
  16. public TokenDefinition(string name, Regex regex, bool ignore = false)
  17. {
  18. Name = name;
  19. Regex = regex;
  20. Ignore = ignore;
  21. }
  22.  
  23. public Token CreateToken(int line, int column, int index, string matchedText)
  24. {
  25. return new Token
  26. {
  27. Name = Name,
  28. Position = new TokenPosition(line, column, index),
  29. MatchedText = matchedText
  30. };
  31. }
  32. }
  33.  
  34. public class Token
  35. {
  36. public string Name { get; set; }
  37.  
  38. public TokenPosition Position { get; set; }
  39.  
  40. public string MatchedText { get; set; }
  41. }
  42.  
  43. public class TokenPosition
  44. {
  45. public int Line { get; set; }
  46.  
  47. public int Column { get; set; }
  48.  
  49. public int Index { get; set; }
  50.  
  51. public TokenPosition(int line, int column, int index)
  52. {
  53. Line = line;
  54. Column = column;
  55. Index = index;
  56. }
  57.  
  58. public override string ToString()
  59. {
  60. return $"({Index}: {Line},{Column})";
  61. }
  62. }
  63.  
  64. public class Lexer
  65. {
  66. public static Regex s_EofRegex = new Regex(@"\r\n|\r|\n", RegexOptions.Compiled);
  67. public List<TokenDefinition> _tokenDefinitions = new List<TokenDefinition>();
  68.  
  69. public void AddDefinition(TokenDefinition tokenDefinition)
  70. {
  71. _tokenDefinitions.Add(tokenDefinition);
  72. }
  73.  
  74. public IEnumerable<Token> GetTokens(string text)
  75. {
  76. int line = 1, column = 1, currentIndex = 0;
  77.  
  78. while (currentIndex < text.Length)
  79. {
  80. TokenDefinition matchedTokenDefinition = null;
  81. string matchedText = null;
  82. foreach (var tokenDefinition in _tokenDefinitions)
  83. {
  84. var tokenMatch = tokenDefinition.Regex.Match(text, currentIndex);
  85. if (tokenMatch.Success && tokenMatch.Index == currentIndex)
  86. {
  87. matchedText = tokenMatch.Value;
  88. matchedTokenDefinition = tokenDefinition;
  89. }
  90. }
  91.  
  92. if (matchedTokenDefinition == null)
  93. {
  94. Console.WriteLine($"Failed to match any definition at '{text[currentIndex]}' in {new TokenPosition(line, column, currentIndex)}.");
  95. continue;
  96. }
  97.  
  98. if (!matchedTokenDefinition.Ignore)
  99. yield return matchedTokenDefinition.CreateToken(line, column, currentIndex, matchedText);
  100.  
  101. var eofMatch = s_EofRegex.Match(matchedText);
  102. if (eofMatch.Success)
  103. {
  104. while (eofMatch.Success)
  105. {
  106. line += 1;
  107. column = matchedText.Length - (eofMatch.Index + eofMatch.Length) + 1;
  108. eofMatch = eofMatch.NextMatch();
  109. }
  110. }
  111. else
  112. {
  113. column += matchedText.Length;
  114. }
  115.  
  116. currentIndex += matchedText.Length;
  117. }
  118. }
  119. }
  120. }
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement