pylatexenc/latexwalker.py
def get_latex_nodes(self, pos=0, stop_upon_closing_brace=None, stop_upon_end_environment=None,
                        stop_upon_closing_mathmode=None):
        """
        Parses the latex content given to the constructor (and stored in `self.s`)
        into a list of nodes.

        Returns a tuple `(nodelist, pos, len)` where nodelist is a list of
        :py:class:`LatexNode`\ 's.
        
        If `stop_upon_closing_brace` is given and set to a character, then parsing stops
        once the given closing brace is encountered (but not inside a subgroup).  The
        brace is given as a character, ']' or '}'.  The returned `len` includes the
        closing brace, but the closing brace is not included in any of the nodes in the
        `nodelist`.

        If `stop_upon_end_environment` is provided, then parsing stops once the given
        environment was closed.  If there is an environment mismatch, then a
        `LatexWalkerParseError` is raised except in tolerant parsing mode (see
        py:meth:`parse_flags()`).  Again, the closing environment is included in the
        length count but not the nodes.

        If `stop_upon_closing_mathmode` is specified, then the parsing stops
        once the corresponding math mode (assumed already open) is closed.
        Currently, only inline math modes delimited by ``$`` are supported.
        I.e., currently, if set, only the value
        ``stop_upon_closing_mathmode='$'`` is valid.
        """

        nodelist = []
    
        brackets_are_chars = True
        if (stop_upon_closing_brace == ']'):
            brackets_are_chars = False

        origpos = pos

        class PosPointer:
            def __init__(self, pos=0, lastchars=''):
                self.pos = pos
                self.lastchars = lastchars

        p = PosPointer(pos)

        def do_read(nodelist, p):
            """
            Read a single token and process it, recursing into brace blocks and environments etc if
            needed, and appending stuff to nodelist.

            Return True whenever we should stop trying to read more. (e.g. upon reaching the a matched
            stop_upon_end_environment etc.)
            """

            try:
                tok = self.get_token(p.pos, brackets_are_chars=brackets_are_chars)
            except LatexWalkerEndOfStream:
                if self.tolerant_parsing:
                    return True
                raise # re-raise

            p.pos = tok.pos + tok.len

            # if it's a char, just append it to the stream of last characters.
            if (tok.tok == 'char'):
                p.lastchars += tok.pre_space + tok.arg
                return False

            # if it's not a char, push the last `p.lastchars` into the node list before anything else
            if len(p.lastchars):
                strnode = LatexCharsNode(chars=p.lastchars+tok.pre_space)
                nodelist.append(strnode)
                p.lastchars = ''
            elif len(tok.pre_space):
                # If we have pre_space, add a separate chars node that contains
                # the spaces.  We do this seperately, so that latex2text can
                # ignore these groups by default to avoid too much space on the
                # output.  This allows latex2text to implement the
                # `strict_latex_spaces=True` flag correctly.
                spacestrnode = LatexCharsNode(chars=tok.pre_space)
                nodelist.append(spacestrnode)

            # and see what the token is.

            if (tok.tok == 'brace_close'):
                # we've reached the end of the group. stop the parsing.
                if (tok.arg != stop_upon_closing_brace):
                    if (not self.tolerant_parsing):
                        raise LatexWalkerParseError(
                            s=self.s,
                            pos=tok.pos,
                            msg='Unexpected mismatching closing brace: `%s\'' %(tok.arg)
                        )
                    return False
                return True

            if (tok.tok == 'end_environment'):
                # we've reached the end of an environment.
                if (tok.arg != stop_upon_end_environment):
                    if (not self.tolerant_parsing):
                        raise LatexWalkerParseError(
                            s=self.s,
                            pos=tok.pos,
                            msg=('Unexpected mismatching closing environment: `%s\', '
                                 'expecting `%s\'' %(tok.arg, stop_upon_end_environment))
                        )
                    return False
                return True

            if (tok.tok == 'mathmode_inline'):
                # if we care about keeping math mode inlines verbatim, gulp all of the expression.
                if stop_upon_closing_mathmode is not None:
                    if stop_upon_closing_mathmode != '$':
                        raise LatexWalkerParseError(
                            s=self.s,
                            pos=tok.pos,
                            msg='Unexpected mismatching closing math mode: `$\''
                        )
                    return True

                # we have encountered a new math inline, so gulp all of the math expression
                (mathinline_nodelist, mpos, mlen) = self.get_latex_nodes(p.pos, stop_upon_closing_mathmode='$')
                p.pos = mpos + mlen

                nodelist.append(LatexMathNode(displaytype='inline', nodelist=mathinline_nodelist))
                return

            if (tok.tok == 'comment'):
                commentnode = LatexCommentNode(comment=tok.arg, comment_post_space=tok.post_space)
                nodelist.append(commentnode)
                return

            if (tok.tok == 'brace_open'):
                # another braced group to read.
                (groupnode, bpos, blen) = self.get_latex_braced_group(tok.pos)
                p.pos = bpos + blen
                nodelist.append(groupnode)
                return

            if (tok.tok == 'begin_environment'):
                # an environment to read.
                (envnode, epos, elen) = self.get_latex_environment(tok.pos, environmentname=tok.arg)
                p.pos = epos + elen
                # add node and continue.
                nodelist.append(envnode)
                return

            if (tok.tok == 'macro'):
                # read a macro. see if it has arguments.
                nodeoptarg = None
                nodeargs = []
                macname = tok.arg.rstrip('*') # for lookup in macro_dict
                if macname in self.macro_dict:
                    mac = self.macro_dict[macname]

                    def getoptarg(pos):
                        """
                        Gets a possibly optional argument. returns (argnode, new-pos) where argnode
                        might be `None` if the argument was not specified.
                        """
                        optarginfotuple = self.get_latex_maybe_optional_arg(pos)
                        if optarginfotuple is not None:
                            (nodeoptarg, optargpos, optarglen) = optarginfotuple
                            return (nodeoptarg, optargpos+optarglen)
                        return (None, pos)

                    def getarg(pos):
                        """
                        Gets a mandatory argument. returns (argnode, new-pos)
                        """
                        (nodearg, npos, nlen) = self.get_latex_expression(pos, strict_braces=False)
                        return (nodearg, npos + nlen)

                    if mac.optarg:
                        (nodeoptarg, p.pos) = getoptarg(p.pos)

                    if isinstance(mac.numargs, _basestring):
                        # specific argument specification
                        for arg in mac.numargs:
                            if arg == '{':
                                (node, p.pos) = getarg(p.pos)
                                nodeargs.append(node)
                            elif arg == '[':
                                (node, p.pos) = getoptarg(p.pos)
                                nodeargs.append(node)
                            else:
                                raise LatexWalkerError(
                                    "Unknown macro argument kind for macro %s: %s"
                                    % (mac.macroname, arg)
                                )
                    else:
                        for n in range(mac.numargs):
                            (nodearg, p.pos) = getarg(p.pos)
                            nodeargs.append(nodearg)

                nodelist.append(LatexMacroNode(macroname=tok.arg,
                                               nodeoptarg=nodeoptarg,
                                               nodeargs=nodeargs,
                                               macro_post_space=tok.post_space))
                return None

            raise LatexWalkerParseError(s=self.s, pos=p.pos, msg="Unknown token: %r" %(tok))



        while True:
            try:
                r_endnow = do_read(nodelist, p)
            except LatexWalkerEndOfStream:
                if stop_upon_closing_brace or stop_upon_end_environment:
                    # unexpected eof
                    if (not self.tolerant_parsing):
                        raise LatexWalkerError("Unexpected end of stream!")
                    else:
                        r_endnow = False
                else:
                    r_endnow = True

            if (r_endnow):
                # add last chars
                if (p.lastchars):
                    strnode = LatexCharsNode(chars=p.lastchars)
                    nodelist.append(strnode)
                return (nodelist, origpos, p.pos - origpos)

        raise LatexWalkerError(                # lgtm [py/unreachable-statement]
            "CONGRATULATIONS !! "
            "You are the first human to telepathically break an infinite loop !!!!!!!"
        )
def get_latex_nodes(self, pos=0, stop_upon_closing_brace=None, stop_upon_end_environment=None,
                        stop_upon_closing_mathmode=None):
        """
        Parses the latex content given to the constructor (and stored in `self.s`)
        into a list of nodes.

        Returns a tuple `(nodelist, pos, len)` where nodelist is a list of
        :py:class:`LatexNode`\ 's.
        
        If `stop_upon_closing_brace` is given and set to a character, then parsing stops
        once the given closing brace is encountered (but not inside a subgroup).  The
        brace is given as a character, ']' or '}'.  The returned `len` includes the
        closing brace, but the closing brace is not included in any of the nodes in the
        `nodelist`.

        If `stop_upon_end_environment` is provided, then parsing stops once the given
        environment was closed.  If there is an environment mismatch, then a
        `LatexWalkerParseError` is raised except in tolerant parsing mode (see
        py:meth:`parse_flags()`).  Again, the closing environment is included in the
        length count but not the nodes.

        If `stop_upon_closing_mathmode` is specified, then the parsing stops
        once the corresponding math mode (assumed already open) is closed.
        Currently, only inline math modes delimited by ``$`` are supported.
        I.e., currently, if set, only the value
        ``stop_upon_closing_mathmode='$'`` is valid.
        """

        nodelist = []
    
        brackets_are_chars = True
        if (stop_upon_closing_brace == ']'):
            brackets_are_chars = False

        origpos = pos

        class PosPointer:
            def __init__(self, pos=0, lastchars=''):
                self.pos = pos
                self.lastchars = lastchars

        p = PosPointer(pos)

        def do_read(nodelist, p):
            """
            Read a single token and process it, recursing into brace blocks and environments etc if
            needed, and appending stuff to nodelist.

            Return True whenever we should stop trying to read more. (e.g. upon reaching the a matched
            stop_upon_end_environment etc.)
            """

            try:
                tok = self.get_token(p.pos, brackets_are_chars=brackets_are_chars)
            except LatexWalkerEndOfStream:
                if self.tolerant_parsing:
                    return True
                raise # re-raise

            p.pos = tok.pos + tok.len

            # if it's a char, just append it to the stream of last characters.
            if (tok.tok == 'char'):
                p.lastchars += tok.pre_space + tok.arg
                return False

            # if it's not a char, push the last `p.lastchars` into the node list before anything else
            if len(p.lastchars):
                strnode = LatexCharsNode(chars=p.lastchars+tok.pre_space)
                nodelist.append(strnode)
                p.lastchars = ''
            elif len(tok.pre_space):
                # If we have pre_space, add a separate chars node that contains
                # the spaces.  We do this seperately, so that latex2text can
                # ignore these groups by default to avoid too much space on the
                # output.  This allows latex2text to implement the
                # `strict_latex_spaces=True` flag correctly.
                spacestrnode = LatexCharsNode(chars=tok.pre_space)
                nodelist.append(spacestrnode)

            # and see what the token is.

            if (tok.tok == 'brace_close'):
                # we've reached the end of the group. stop the parsing.
                if (tok.arg != stop_upon_closing_brace):
                    if (not self.tolerant_parsing):
                        raise LatexWalkerParseError(
                            s=self.s,
                            pos=tok.pos,
                            msg='Unexpected mismatching closing brace: `%s\'' %(tok.arg)
                        )
                    return False
                return True

            if (tok.tok == 'end_environment'):
                # we've reached the end of an environment.
                if (tok.arg != stop_upon_end_environment):
                    if (not self.tolerant_parsing):
                        raise LatexWalkerParseError(
                            s=self.s,
                            pos=tok.pos,
                            msg=('Unexpected mismatching closing environment: `%s\', '
                                 'expecting `%s\'' %(tok.arg, stop_upon_end_environment))
                        )
                    return False
                return True

            if (tok.tok == 'mathmode_inline'):
                # if we care about keeping math mode inlines verbatim, gulp all of the expression.
                if stop_upon_closing_mathmode is not None:
                    if stop_upon_closing_mathmode != '$':
                        raise LatexWalkerParseError(
                            s=self.s,
                            pos=tok.pos,
                            msg='Unexpected mismatching closing math mode: `$\''
                        )
                    return True

                # we have encountered a new math inline, so gulp all of the math expression
                (mathinline_nodelist, mpos, mlen) = self.get_latex_nodes(p.pos, stop_upon_closing_mathmode='$')
                p.pos = mpos + mlen

                nodelist.append(LatexMathNode(displaytype='inline', nodelist=mathinline_nodelist))
                return

            if (tok.tok == 'comment'):
                commentnode = LatexCommentNode(comment=tok.arg, comment_post_space=tok.post_space)
                nodelist.append(commentnode)
                return

            if (tok.tok == 'brace_open'):
                # another braced group to read.
                (groupnode, bpos, blen) = self.get_latex_braced_group(tok.pos)
                p.pos = bpos + blen
                nodelist.append(groupnode)
                return

            if (tok.tok == 'begin_environment'):
                # an environment to read.
                (envnode, epos, elen) = self.get_latex_environment(tok.pos, environmentname=tok.arg)
                p.pos = epos + elen
                # add node and continue.
                nodelist.append(envnode)
                return

            if (tok.tok == 'macro'):
                # read a macro. see if it has arguments.
                nodeoptarg = None
                nodeargs = []
                macname = tok.arg.rstrip('*') # for lookup in macro_dict
                if macname in self.macro_dict:
                    mac = self.macro_dict[macname]

                    def getoptarg(pos):
                        """
                        Gets a possibly optional argument. returns (argnode, new-pos) where argnode
                        might be `None` if the argument was not specified.
                        """
                        optarginfotuple = self.get_latex_maybe_optional_arg(pos)
                        if optarginfotuple is not None:
                            (nodeoptarg, optargpos, optarglen) = optarginfotuple
                            return (nodeoptarg, optargpos+optarglen)
                        return (None, pos)

                    def getarg(pos):
                        """
                        Gets a mandatory argument. returns (argnode, new-pos)
                        """
                        (nodearg, npos, nlen) = self.get_latex_expression(pos, strict_braces=False)
                        return (nodearg, npos + nlen)

                    if mac.optarg:
                        (nodeoptarg, p.pos) = getoptarg(p.pos)

                    if isinstance(mac.numargs, _basestring):
                        # specific argument specification
                        for arg in mac.numargs:
                            if arg == '{':
                                (node, p.pos) = getarg(p.pos)
                                nodeargs.append(node)
                            elif arg == '[':
                                (node, p.pos) = getoptarg(p.pos)
                                nodeargs.append(node)
                            else:
                                raise LatexWalkerError(
                                    "Unknown macro argument kind for macro %s: %s"
                                    % (mac.macroname, arg)
                                )
                    else:
                        for n in range(mac.numargs):
                            (nodearg, p.pos) = getarg(p.pos)
                            nodeargs.append(nodearg)

                nodelist.append(LatexMacroNode(macroname=tok.arg,
                                               nodeoptarg=nodeoptarg,
                                               nodeargs=nodeargs,
                                               macro_post_space=tok.post_space))
                return None

            raise LatexWalkerParseError(s=self.s, pos=p.pos, msg="Unknown token: %r" %(tok))



        while True:
            try:
                r_endnow = do_read(nodelist, p)
            except LatexWalkerEndOfStream:
                if stop_upon_closing_brace or stop_upon_end_environment:
                    # unexpected eof
                    if (not self.tolerant_parsing):
                        raise LatexWalkerError("Unexpected end of stream!")
                    else:
                        r_endnow = False
                else:
                    r_endnow = True

            if (r_endnow):
                # add last chars
                if (p.lastchars):
                    strnode = LatexCharsNode(chars=p.lastchars)
                    nodelist.append(strnode)
                return (nodelist, origpos, p.pos - origpos)

        raise LatexWalkerError(                # lgtm [py/unreachable-statement]
            "CONGRATULATIONS !! "
            "You are the first human to telepathically break an infinite loop !!!!!!!"
        )
Parses the latex content given to the constructor (and stored in `self.s`)
        into a list of nodes.

        Returns a tuple `(nodelist, pos, len)` where nodelist is a list of
        :py:class:`LatexNode`\ 's.
        
        If `stop_upon_closing_brace` is given and set to a character, then parsing stops
        once the given closing brace is encountered (but not inside a subgroup).  The
        brace is given as a character, ']' or '}'.  The returned `len` includes the
        closing brace, but the closing brace is not included in any of the nodes in the
        `nodelist`.

        If `stop_upon_end_environment` is provided, then parsing stops once the given
        environment was closed.  If there is an environment mismatch, then a
        `LatexWalkerParseError` is raised except in tolerant parsing mode (see
        py:meth:`parse_flags()`).  Again, the closing environment is included in the
        length count but not the nodes.

        If `stop_upon_closing_mathmode` is specified, then the parsing stops
        once the corresponding math mode (assumed already open) is closed.
        Currently, only inline math modes delimited by ``$`` are supported.
        I.e., currently, if set, only the value
        ``stop_upon_closing_mathmode='$'`` is valid.