Advertisement
Guest User

Untitled

a guest
Apr 6th, 2020
209
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
Python 3.18 KB | None | 0 0
  1. import re
  2. import dis
  3. from collections import deque
  4.  
  5. TOKEN_TYPE_NUMBER = 0
  6. TOKEN_TYPE_ADD = 1
  7. TOKEN_TYPE_SUB = 2
  8. TOKEN_TYPE_MUL = 3
  9. TOKEN_TYPE_DIV = 4
  10. TOKEN_TYPE_ECHO = 5
  11. TOKEN_TYPE_EOL = 6
  12. TOKEN_TYPE_EOF = 7
  13.  
  14. TOKEN_NAMES = ["num", "add", "sub", "mul", "div", "echo", "eol", "eof"]
  15.  
  16. PATTERN = r"(\d+)|(\+)|(\-)|(\*)|(\/)|(echo)|(;|\n)"
  17.  
  18. class Token:
  19.     def __init__(self, t, v, l, s):
  20.         self.type = t
  21.         self.value = v
  22.         self.symbol = s
  23.         self.line = l
  24.  
  25.     def __str__(self):
  26.         v = self.value if self.value != "\n" else ""
  27.         return f"{TOKEN_NAMES[self.type]} : {v}"
  28.  
  29. def firstindex(l, cond):
  30.     return next(i for i, v in enumerate(l) if cond(v))
  31.  
  32. def lastindex(l, cond):
  33.     filtered = [i for i, v in enumerate(l) if cond(v)]
  34.     if len(filtered) == 0: return -1
  35.     *_, last = filtered
  36.     return last
  37.  
  38. def match_tokens(pattern, text):
  39.     lines = list(re.finditer(r"\n", text))
  40.     for match in re.finditer(pattern, text):
  41.         groups = match.groups()
  42.         nonempty = firstindex(groups, lambda x: x is not None)
  43.         t = nonempty
  44.         v = groups[nonempty]
  45.         l = lastindex(lines, lambda x: x.start() < match.start())
  46.         s = match.start() - (lines[l].start() if l != -1 else 0)
  47.  
  48.         yield Token(t, v, l + 1, s)
  49.  
  50. def tokenize(text):
  51.     tokens = list(match_tokens(PATTERN, text))
  52.     if tokens[-1].type != TOKEN_TYPE_EOL:
  53.         tokens.append(Token(TOKEN_TYPE_EOL, "\n", tokens[-1].line, tokens[-1].symbol + len(tokens[-1].value)))
  54.     tokens.append(Token(TOKEN_TYPE_EOF, "", tokens[-1].line + 1, 0))
  55.  
  56.     return tokens
  57.  
  58. def parse(tokens):
  59.     i = 0
  60.  
  61.     def match(t):
  62.         nonlocal i
  63.  
  64.         if tokens[i].type != t: return False
  65.  
  66.         i += 1
  67.         return True
  68.  
  69.     def consume(t):
  70.         if not match(t):
  71.             raise Exception(
  72.                 "expected " + TOKEN_NAMES[t] + \
  73.                 " got " + TOKEN_NAMES[tokens[i].type] + \
  74.                 f" at {tokens[i].line}:{tokens[i].symbol}"
  75.             )
  76.  
  77.         return True
  78.  
  79.     def expr():
  80.         additive()
  81.  
  82.     def additive():
  83.         multiplicative()
  84.  
  85.         while True:
  86.             if match(TOKEN_TYPE_ADD):
  87.                 multiplicative()
  88.                 bytecode.append("add")
  89.                 continue
  90.             elif match(TOKEN_TYPE_SUB):
  91.                 multiplicative()
  92.                 bytecode.append("sub")
  93.                 continue
  94.             break
  95.  
  96.     def multiplicative():
  97.         unary()
  98.  
  99.         while True:
  100.             if match(TOKEN_TYPE_MUL):
  101.                 unary()
  102.                 bytecode.append("mul")
  103.                 continue
  104.             elif match(TOKEN_TYPE_DIV):
  105.                 unary()
  106.                 bytecode.append("div")
  107.                 continue
  108.             break
  109.  
  110.     def unary():
  111.         if match(TOKEN_TYPE_ADD):
  112.             unary()
  113.             bytecode.append("unpos")
  114.             return
  115.         elif match(TOKEN_TYPE_SUB):
  116.             unary()
  117.             bytecode.append("unneg")
  118.             return
  119.  
  120.         primary()
  121.  
  122.     def primary():
  123.         if match(TOKEN_TYPE_NUMBER):
  124.             bytecode.append(f"push {tokens[i - 1].value}")
  125.             return
  126.         elif match(TOKEN_TYPE_ECHO):
  127.             expr()
  128.  
  129.             bytecode.append("echo")
  130.             return
  131.  
  132.         raise Exception("unexpected token " + TOKEN_NAMES[tokens[i].type] + f" at {tokens[i].line}:{tokens[i].symbol}")
  133.  
  134.     bytecode = []
  135.  
  136.     while not match(TOKEN_TYPE_EOF):
  137.         expr()
  138.         consume(TOKEN_TYPE_EOL)
  139.  
  140.     return bytecode
  141.  
  142. def test():
  143.     text = """\
  144. -1 / 2 * 2; echo 1 + 2
  145. echo +-1 * 7 / 3\
  146.     """
  147.  
  148.     print(text)
  149.     print()
  150.     tokens = tokenize(text)
  151.     for tok in tokens:
  152.         print(tok)
  153.     print()
  154.     print("\n".join(parse(tokens)))
  155.  
  156.  
  157. if __name__ == '__main__':
  158.     test()
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement