Advertisement
charles-esterbrook

Lark IndexError

Nov 12th, 2020 (edited)
45
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
Python 4.65 KB | None | 0 0
  1. """
  2. IndexError_ChildFilterLALR_NoPlaceholders_2020-11.py
  3.  
  4. https://github.com/lark-parser/lark/issues/754
  5.  
  6. Works in Lark 0.9, but not 0.10.1 or github master on 2020-11-12
  7. Python 3.8.5 64-bit on Win 10 Pro
  8.  
  9. python IndexError_ChildFilterLALR_NoPlaceholders_2020-11.py
  10.  
  11. Expected output is:
  12.    code: ...
  13.    parser: ...
  14.    tree: ...
  15.    Done.
  16.  
  17. Works on Lark 0.9.0
  18. On Lark 0.10.1 or latest master branch, raises IndexError:
  19.  
  20. Traceback (most recent call last):
  21.  File "IndexError_ChildFilterLALR_NoPlaceholders_2020-11.py", line 104, in <module>
  22.    tree = p.parse(code)
  23.  File "...\lark-github\lark\lark.py", line 494, in parse
  24.    return self.parser.parse(text, start=start)
  25.  File "...\lark-github\lark\parser_frontends.py", line 138, in parse
  26.    return self._parse(start, self.make_lexer(text))
  27.  File "...\lark-github\lark\parser_frontends.py", line 73, in _parse
  28.    return self.parser.parse(input, start, *args)
  29.  File "...\lark-github\lark\parsers\earley.py", line 320, in parse
  30.    return transformer.transform(solutions[0])
  31.  File "...\lark-github\lark\parsers\earley_forest.py", line 353, in transform
  32.    self.visit(root)
  33.  File "...\lark-github\lark\parsers\earley_forest.py", line 295, in visit
  34.    vpno(current)
  35.  File "...\lark-github\lark\parsers\earley_forest.py", line 587, in visit_packed_node_out
  36.    super(ForestToParseTree, self).visit_packed_node_out(node)
  37.  File "...\lark-github\lark\parsers\earley_forest.py", line 417, in visit_packed_node_out
  38.    transformed = self.transform_packed_node(node, self.data[id(node)])
  39.  File "...\lark-github\lark\parsers\earley_forest.py", line 570, in transform_packed_node
  40.    return self._call_rule_func(node, children)
  41.  File "...\lark-github\lark\parsers\earley_forest.py", line 524, in _call_rule_func
  42.    return self.callbacks[node.rule](data)
  43.  File "...\lark-github\lark\parse_tree_builder.py", line 29, in __call__
  44.    res = self.node_builder(children)
  45.  File "...\lark-github\lark\parse_tree_builder.py", line 129, in __call__
  46.    filtered.append(children[i])
  47. IndexError: list index out of range
  48. """
  49. import lark
  50. import lark.indenter
  51. code = '''\
  52. if a is b
  53.    print(a)
  54. '''
  55. # improper subset of one of the Python grammar examples:
  56. grammar = r"""
  57. file_input: (_NEWLINE | stmt)*
  58.  
  59. ?stmt: simple_stmt | compound_stmt
  60. ?simple_stmt: small_stmt (";" small_stmt)* [";"] _NEWLINE
  61. ?small_stmt: (NAME test)
  62.  
  63. ?compound_stmt: if_stmt
  64. if_stmt: "if" test ":"? suite
  65. suite: (","? simple_stmt) | (_NEWLINE _INDENT stmt+ _DEDENT)
  66.  
  67. ?test: binary_bool_test
  68. ?binary_bool_test: not_test (("and" | "or") not_test)*
  69. ?not_test: "not" not_test -> not | comparison
  70. ?comparison: expr (_comp_op expr)*
  71. star_expr: "*" expr
  72. ?expr: shift_expr (("bor" | "band" | "bxor") shift_expr)*
  73. ?shift_expr: arith_expr (_shift_op arith_expr)*
  74. ?arith_expr: term (_add_op term)*
  75. ?term: factor (_mul_op factor)*
  76. ?factor: _factor_op factor | atom_expr
  77.  
  78. _factor_op: "+" | "-"
  79. _add_op:    "+" | "-"
  80. _shift_op:  "<<" | ">>"
  81. _mul_op:    "*" | "/" | "%" | "//"
  82. _comp_op:   "<" | ">" | "==" | ">=" | "<=" | "<>" | "in" | "is"
  83.  
  84. ?atom_expr: atom_expr "(" [arguments] ")" -> func_call
  85.          | atom
  86.  
  87. ?atom: "(" test ")"
  88.     | name
  89.     | number
  90.  
  91. name: NAME
  92. ?number: INT_LITERAL_DEC
  93.  
  94. ?testlist_comp: (test|star_expr) [("," (test|star_expr))+ [","] | ","]
  95. exprlist: (expr|star_expr) ("," (expr|star_expr))* [","]
  96. testlist: test ("," test)* [","]
  97.  
  98. arguments: argvalue ("," argvalue)*  ("," [starargs | kwargs])?
  99.         | starargs
  100.         | kwargs
  101.  
  102. starargs: "*" test ("," "*" test)* ("," argvalue)* ["," kwargs]
  103. kwargs: "**" test
  104.  
  105. ?argvalue: test ("=" test)?
  106.  
  107. NAME: /[a-zA-Z_]\w*/
  108. _NEWLINE: ( /\r?\n[\t ]*/  )+
  109. INT_LITERAL_DEC:   /0|[1-9]\d*/i
  110.  
  111. %ignore /[\t \f]+/  // WS
  112. %ignore /\\[\t \f]*\r?\n/   // LINE_CONT
  113. %declare _INDENT _DEDENT
  114. """
  115.  
  116. # for Pythonic indentation:
  117. class CustomIndenter(lark.indenter.Indenter):
  118.     NL_type = '_NEWLINE'
  119.     OPEN_PAREN_types = ['LPAR', 'LSQB', 'LBRACE']
  120.     CLOSE_PAREN_types = ['RPAR', 'RSQB', 'RBRACE']
  121.     INDENT_type = '_INDENT'
  122.     DEDENT_type = '_DEDENT'
  123.     tab_len = 8
  124.  
  125. def parser():
  126.     return lark.Lark(
  127.         grammar,
  128.         debug=True,
  129.         parser='earley',      # lalr, earley
  130.         lexer='standard',     # in ('standard', 'contextual', 'dynamic', 'dynamic_complete') or issubclass(lexer, Lexer)
  131.         postlex=CustomIndenter(),
  132.         start='file_input',
  133.         keep_all_tokens=True,
  134.         maybe_placeholders=True,
  135.         propagate_positions=True,
  136.         ambiguity='resolve')  # in ('resolve', 'explicit', 'auto')
  137.  
  138. print('code:')
  139. print(code)
  140.  
  141. p = parser()
  142. print('parser:', p)
  143. tree = p.parse(code)
  144. print('tree:', tree)
  145.  
  146. print('Done.')
  147.  
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement