pylatexenc/latexwalker.py
def get_token(self, pos, brackets_are_chars=True, environments=True, keep_inline_math=None):
        """
        Parses the latex content given to the constructor (and stored in `self.s`),
        starting at position `pos`, to parse a single "token", as defined by
        :py:class:`LatexToken`.

        Parse the token in the stream pointed to at position `pos`.

        Returns a :py:class:`LatexToken`. Raises :py:exc:`LatexWalkerEndOfStream` if end
        of stream reached.

        If `brackets_are_chars=False`, then square bracket characters count as
        'brace_open' and 'brace_close' token types (see :py:class:`LatexToken`); otherwise
        (the default) they are considered just like other normal characters.

        If `environments=False`, then '\\begin' and '\\end' tokens count as regular
        'macro' tokens (see :py:class:`LatexToken`); otherwise (the default) they are
        considered as the token types 'begin_environment' and 'end_environment'.

        If `keep_inline_math` is not `None`, then that value overrides that of
        `self.keep_inline_math` for the duration of this method call.
        """

        s = self.s # shorthand

        with _PushPropOverride(self, 'keep_inline_math', keep_inline_math):

            space = ''
            while (pos < len(s) and s[pos].isspace()):
                space += s[pos]
                pos += 1
                if (space.endswith('\n\n')):  # two \n's indicate new paragraph.
                    # Adding pre-space is overkill here I think.
                    return LatexToken(tok='char', arg='\n\n', pos=pos-2, len=2, pre_space='')


            if (pos >= len(s)):
                raise LatexWalkerEndOfStream()

            if (s[pos] == '\\'):
                # escape sequence
                i = 2
                macro = s[pos+1] # next char is necessarily part of macro
                # following chars part of macro only if all are alphabetical
                isalphamacro = False
                if (s[pos+1].isalpha()):
                    isalphamacro = True
                    while pos+i<len(s) and s[pos+i].isalpha():
                        macro += s[pos+i]
                        i += 1
                # possibly followed by a star
                if (pos+i<len(s) and s[pos+i] == '*'):
                    macro += '*'
                    i += 1

                # see if we have a begin/end environment
                if (environments and (macro == 'begin' or macro == 'end')):
                    # \begin{environment} or \end{environment}
                    envmatch = re.match(r'^\s*\{([\w*]+)\}', s[pos+i:])
                    if (envmatch is None):
                        raise LatexWalkerParseError(
                            s=s,
                            pos=pos,
                            msg="Bad \\%s macro: expected {environment}" %(macro)
                        )

                    return LatexToken(
                        tok=('begin_environment' if macro == 'begin' else  'end_environment'),
                        arg=envmatch.group(1),
                        pos=pos,
                        len=i+envmatch.end(), # !!envmatch.end() counts from pos+i
                        pre_space=space
                        )

                # get the following whitespace, and store it in the macro's post_space
                post_space = ''
                if isalphamacro:
                    # important, LaTeX does not consume space after non-alpha macros, like \&
                    while pos+i<len(s) and s[pos+i].isspace():
                        post_space += s[pos+i]
                        i += 1

                return LatexToken(tok='macro', arg=macro, pos=pos, len=i,
                                  pre_space=space, post_space=post_space)

            if (s[pos] == '%'):
                # latex comment
                m = re.search(r'(\n|\r|\n\r)\s*', s[pos:])
                mlen = None
                if m is not None:
                    arglen = m.start() # relative to pos already
                    mlen = m.end() # relative to pos already
                    mspace = m.group()
                else:
                    arglen = len(s)-pos# [  ==len(s[pos:])  ]
                    mlen = arglen
                    mspace = ''
                return LatexToken(tok='comment', arg=s[pos+1:pos+arglen], pos=pos, len=mlen,
                                  pre_space=space, post_space=mspace)

            openbracechars = '{'
            closebracechars = '}'
            if not brackets_are_chars:
                openbracechars += '['
                closebracechars += ']'

            if s[pos] in openbracechars:
                return LatexToken(tok='brace_open', arg=s[pos], pos=pos, len=1, pre_space=space)

            if s[pos] in closebracechars:
                return LatexToken(tok='brace_close', arg=s[pos], pos=pos, len=1, pre_space=space)

            # check if it is an inline math char, if we care about inline math.
            if (s[pos] == '$' and self.keep_inline_math):
                # check that we don't have double-$$, which would be a display environment.
                if not (pos+1 < len(s) and s[pos+1] == '$'):
                    return LatexToken(tok='mathmode_inline', arg=s[pos], pos=pos, len=1, pre_space=space)
                # otherwise, proceed to 'char' type.

            return LatexToken(tok='char', arg=s[pos], pos=pos, len=1, pre_space=space)
def get_token(self, pos, brackets_are_chars=True, environments=True, keep_inline_math=None):
        """
        Parses the latex content given to the constructor (and stored in `self.s`),
        starting at position `pos`, to parse a single "token", as defined by
        :py:class:`LatexToken`.

        Parse the token in the stream pointed to at position `pos`.

        Returns a :py:class:`LatexToken`. Raises :py:exc:`LatexWalkerEndOfStream` if end
        of stream reached.

        If `brackets_are_chars=False`, then square bracket characters count as
        'brace_open' and 'brace_close' token types (see :py:class:`LatexToken`); otherwise
        (the default) they are considered just like other normal characters.

        If `environments=False`, then '\\begin' and '\\end' tokens count as regular
        'macro' tokens (see :py:class:`LatexToken`); otherwise (the default) they are
        considered as the token types 'begin_environment' and 'end_environment'.

        If `keep_inline_math` is not `None`, then that value overrides that of
        `self.keep_inline_math` for the duration of this method call.
        """

        s = self.s # shorthand

        with _PushPropOverride(self, 'keep_inline_math', keep_inline_math):

            space = ''
            while (pos < len(s) and s[pos].isspace()):
                space += s[pos]
                pos += 1
                if (space.endswith('\n\n')):  # two \n's indicate new paragraph.
                    # Adding pre-space is overkill here I think.
                    return LatexToken(tok='char', arg='\n\n', pos=pos-2, len=2, pre_space='')


            if (pos >= len(s)):
                raise LatexWalkerEndOfStream()

            if (s[pos] == '\\'):
                # escape sequence
                i = 2
                macro = s[pos+1] # next char is necessarily part of macro
                # following chars part of macro only if all are alphabetical
                isalphamacro = False
                if (s[pos+1].isalpha()):
                    isalphamacro = True
                    while pos+i<len(s) and s[pos+i].isalpha():
                        macro += s[pos+i]
                        i += 1
                # possibly followed by a star
                if (pos+i<len(s) and s[pos+i] == '*'):
                    macro += '*'
                    i += 1

                # see if we have a begin/end environment
                if (environments and (macro == 'begin' or macro == 'end')):
                    # \begin{environment} or \end{environment}
                    envmatch = re.match(r'^\s*\{([\w*]+)\}', s[pos+i:])
                    if (envmatch is None):
                        raise LatexWalkerParseError(
                            s=s,
                            pos=pos,
                            msg="Bad \\%s macro: expected {environment}" %(macro)
                        )

                    return LatexToken(
                        tok=('begin_environment' if macro == 'begin' else  'end_environment'),
                        arg=envmatch.group(1),
                        pos=pos,
                        len=i+envmatch.end(), # !!envmatch.end() counts from pos+i
                        pre_space=space
                        )

                # get the following whitespace, and store it in the macro's post_space
                post_space = ''
                if isalphamacro:
                    # important, LaTeX does not consume space after non-alpha macros, like \&
                    while pos+i<len(s) and s[pos+i].isspace():
                        post_space += s[pos+i]
                        i += 1

                return LatexToken(tok='macro', arg=macro, pos=pos, len=i,
                                  pre_space=space, post_space=post_space)

            if (s[pos] == '%'):
                # latex comment
                m = re.search(r'(\n|\r|\n\r)\s*', s[pos:])
                mlen = None
                if m is not None:
                    arglen = m.start() # relative to pos already
                    mlen = m.end() # relative to pos already
                    mspace = m.group()
                else:
                    arglen = len(s)-pos# [  ==len(s[pos:])  ]
                    mlen = arglen
                    mspace = ''
                return LatexToken(tok='comment', arg=s[pos+1:pos+arglen], pos=pos, len=mlen,
                                  pre_space=space, post_space=mspace)

            openbracechars = '{'
            closebracechars = '}'
            if not brackets_are_chars:
                openbracechars += '['
                closebracechars += ']'

            if s[pos] in openbracechars:
                return LatexToken(tok='brace_open', arg=s[pos], pos=pos, len=1, pre_space=space)

            if s[pos] in closebracechars:
                return LatexToken(tok='brace_close', arg=s[pos], pos=pos, len=1, pre_space=space)

            # check if it is an inline math char, if we care about inline math.
            if (s[pos] == '$' and self.keep_inline_math):
                # check that we don't have double-$$, which would be a display environment.
                if not (pos+1 < len(s) and s[pos+1] == '$'):
                    return LatexToken(tok='mathmode_inline', arg=s[pos], pos=pos, len=1, pre_space=space)
                # otherwise, proceed to 'char' type.

            return LatexToken(tok='char', arg=s[pos], pos=pos, len=1, pre_space=space)
Parses the latex content given to the constructor (and stored in `self.s`),
        starting at position `pos`, to parse a single "token", as defined by
        :py:class:`LatexToken`.

        Parse the token in the stream pointed to at position `pos`.

        Returns a :py:class:`LatexToken`. Raises :py:exc:`LatexWalkerEndOfStream` if end
        of stream reached.

        If `brackets_are_chars=False`, then square bracket characters count as
        'brace_open' and 'brace_close' token types (see :py:class:`LatexToken`); otherwise
        (the default) they are considered just like other normal characters.

        If `environments=False`, then '\\begin' and '\\end' tokens count as regular
        'macro' tokens (see :py:class:`LatexToken`); otherwise (the default) they are
        considered as the token types 'begin_environment' and 'end_environment'.

        If `keep_inline_math` is not `None`, then that value overrides that of
        `self.keep_inline_math` for the duration of this method call.