h2o-bindings/bin/pyparser.py
def _parse1(self):
        """
        First stage of parsing the code (stored as a raw stream of tokens).

        This method will do the initial pass of the ``self._tokens`` list of tokens, and mark different section as
        belonging to one of the categories: comment, whitespace, docstring, import, code, decorator, def, class, end.
        These sections will be returned as a list of tuples ``(fragment_type, start, end)``, where ``start`` and ``end``
        are indices in the list of raw tokens.
        """
        fragments = []
        tokens = self._tokens

        def advance_after_newline(i0):
            """Return the index of the first token after the end of the current (logical) line."""
            for i in range(i0, len(tokens)):
                if tokens[i].op == NEWLINE:
                    break
            return i + 1

        i = 0
        while i < len(tokens):
            # Assume that we always start at the beginning of a new block
            i0 = i
            tok = tokens[i]
            fragment_type = "???"  # to be determined in the switch clause below

            if tok.op == ENDMARKER:
                fragment_type = "end"
                i += 1
                assert i == len(tokens), "ENDMARKER token encountered before the end of the stream"

            elif tok.op == NL:
                fragment_type = "whitespace"
                # If there are multiple whitespaces, gobble them all
                while tokens[i].op == NL:
                    i += 1

            elif tok.op == COMMENT:
                fragment_type = "comment"
                # Collapse multiple comment lines into a single comment fragment; but only if they are at the same
                # level of indentation.
                is_banner = False
                while i < len(tokens) and tokens[i].op == COMMENT and tokens[i].start_col == tok.start_col:
                    assert tokens[i + 1].op == NL, "Unexpected token after a comment: %r" % tokens[i + 1]
                    s = tokens[i].str
                    if re.match(r"^#\s?[#*=-]{10,}$", s) or re.match(r"^#\s?[#*=-]{4,}.*?[#*=-]{4,}$", s):
                        is_banner = True
                    i += 2
                if is_banner:
                    fragment_type = "banner-comment"

            elif (tok.op == STRING and tokens[i + 1].op == NEWLINE and
                  all(frag[0] == "whitespace" or frag[0] == "comment" for frag in fragments)):
                i += 2
                fragment_type = "docstring"

            elif tok.op == OP and tok.str == "@" and tokens[i + 1].op == NAME:
                while tokens[i].op == OP and tokens[i].str == "@" and tokens[i + 1].op == NAME:
                    i = advance_after_newline(i)
                fragment_type = "decorator"

            elif tok.op == NAME and tok.str in {"from", "import"}:
                while tokens[i].op == NAME and tokens[i].str in {"from", "import"}:
                    i = advance_after_newline(i)
                fragment_type = "import"

            elif tok.op in {INDENT, DEDENT, NEWLINE}:
                assert False, "Unexpected token %d: %r" % (i, tok)

            else:
                i = advance_after_newline(i)
                if i < len(tokens) and tokens[i].op == INDENT:
                    level = 1
                    while level > 0:
                        i += 1
                        level += tokens[i].indent()
                    assert tokens[i].op == DEDENT
                    i += 1  # consume the last DEDENT
                while i < len(tokens) and tokens[i].op == COMMENT and tokens[i].start_col > tok.start_col:
                    assert tokens[i + 1].op == NL
                    i += 2
                if tok.op == NAME and tok.str in {"def", "class"}:
                    fragment_type = tok.str
                else:
                    fragment_type = "code"
            assert i > i0, "Stuck at i = %d" % i
            fragments.append((fragment_type, i0, i))
        return fragments
def _parse1(self):
        """
        First stage of parsing the code (stored as a raw stream of tokens).

        This method will do the initial pass of the ``self._tokens`` list of tokens, and mark different section as
        belonging to one of the categories: comment, whitespace, docstring, import, code, decorator, def, class, end.
        These sections will be returned as a list of tuples ``(fragment_type, start, end)``, where ``start`` and ``end``
        are indices in the list of raw tokens.
        """
        fragments = []
        tokens = self._tokens

        def advance_after_newline(i0):
            """Return the index of the first token after the end of the current (logical) line."""
            for i in range(i0, len(tokens)):
                if tokens[i].op == NEWLINE:
                    break
            return i + 1

        i = 0
        while i < len(tokens):
            # Assume that we always start at the beginning of a new block
            i0 = i
            tok = tokens[i]
            fragment_type = "???"  # to be determined in the switch clause below

            if tok.op == ENDMARKER:
                fragment_type = "end"
                i += 1
                assert i == len(tokens), "ENDMARKER token encountered before the end of the stream"

            elif tok.op == NL:
                fragment_type = "whitespace"
                # If there are multiple whitespaces, gobble them all
                while tokens[i].op == NL:
                    i += 1

            elif tok.op == COMMENT:
                fragment_type = "comment"
                # Collapse multiple comment lines into a single comment fragment; but only if they are at the same
                # level of indentation.
                is_banner = False
                while i < len(tokens) and tokens[i].op == COMMENT and tokens[i].start_col == tok.start_col:
                    assert tokens[i + 1].op == NL, "Unexpected token after a comment: %r" % tokens[i + 1]
                    s = tokens[i].str
                    if re.match(r"^#\s?[#*=-]{10,}$", s) or re.match(r"^#\s?[#*=-]{4,}.*?[#*=-]{4,}$", s):
                        is_banner = True
                    i += 2
                if is_banner:
                    fragment_type = "banner-comment"

            elif (tok.op == STRING and tokens[i + 1].op == NEWLINE and
                  all(frag[0] == "whitespace" or frag[0] == "comment" for frag in fragments)):
                i += 2
                fragment_type = "docstring"

            elif tok.op == OP and tok.str == "@" and tokens[i + 1].op == NAME:
                while tokens[i].op == OP and tokens[i].str == "@" and tokens[i + 1].op == NAME:
                    i = advance_after_newline(i)
                fragment_type = "decorator"

            elif tok.op == NAME and tok.str in {"from", "import"}:
                while tokens[i].op == NAME and tokens[i].str in {"from", "import"}:
                    i = advance_after_newline(i)
                fragment_type = "import"

            elif tok.op in {INDENT, DEDENT, NEWLINE}:
                assert False, "Unexpected token %d: %r" % (i, tok)

            else:
                i = advance_after_newline(i)
                if i < len(tokens) and tokens[i].op == INDENT:
                    level = 1
                    while level > 0:
                        i += 1
                        level += tokens[i].indent()
                    assert tokens[i].op == DEDENT
                    i += 1  # consume the last DEDENT
                while i < len(tokens) and tokens[i].op == COMMENT and tokens[i].start_col > tok.start_col:
                    assert tokens[i + 1].op == NL
                    i += 2
                if tok.op == NAME and tok.str in {"def", "class"}:
                    fragment_type = tok.str
                else:
                    fragment_type = "code"
            assert i > i0, "Stuck at i = %d" % i
            fragments.append((fragment_type, i0, i))
        return fragments
First stage of parsing the code (stored as a raw stream of tokens).

        This method will do the initial pass of the ``self._tokens`` list of tokens, and mark different section as
        belonging to one of the categories: comment, whitespace, docstring, import, code, decorator, def, class, end.
        These sections will be returned as a list of tuples ``(fragment_type, start, end)``, where ``start`` and ``end``
        are indices in the list of raw tokens.