def next
      while @pos < @input.length
        if self.class.ignores_list[@input[@pos]]
          ignore_symbol
          next
        end

        m = self.class.token_regexps.match(@input[@pos..-1])

        if m && ! m[0].empty?
          val = nil
          type = nil
          resolved_type = nil
          m.names.each do |n|
            if m[n]
              type = n.to_sym
              resolved_type = (n.start_with?('__anonymous_') ? nil : type)
              val = m[n]
              break
            end
          end

          if type
            tok = build_token(resolved_type, val)
            @pos += m.end(0)
            tok = self.class.callables[type].call(tok) if self.class.callables[type]

            if tok && tok.type
              return tok
            else
              next
            end
          end
        end
        
        if self.class.literals_list[@input[@pos]]
          tok = build_token(@input[@pos], @input[@pos])
          matched = true
          @pos += 1
          return tok
        end

        if self.class.error_hander
          pos = @pos
          tok = build_token(:error, @input[@pos])
          tok = self.class.error_hander.call(tok)
          if pos == @pos
            raise LexError.new("Illegal character '#{@input[@pos]}' at index #{@pos}")
          else
            return tok if tok && tok.type
          end
        else
          raise LexError.new("Illegal character '#{@input[@pos]}' at index #{@pos}")
        end

      end
      return nil
    end
def next
      while @pos < @input.length
        if self.class.ignores_list[@input[@pos]]
          ignore_symbol
          next
        end

        m = self.class.token_regexps.match(@input[@pos..-1])

        if m && ! m[0].empty?
          val = nil
          type = nil
          resolved_type = nil
          m.names.each do |n|
            if m[n]
              type = n.to_sym
              resolved_type = (n.start_with?('__anonymous_') ? nil : type)
              val = m[n]
              break
            end
          end

          if type
            tok = build_token(resolved_type, val)
            @pos += m.end(0)
            tok = self.class.callables[type].call(tok) if self.class.callables[type]

            if tok && tok.type
              return tok
            else
              next
            end
          end
        end
        
        if self.class.literals_list[@input[@pos]]
          tok = build_token(@input[@pos], @input[@pos])
          matched = true
          @pos += 1
          return tok
        end

        if self.class.error_hander
          pos = @pos
          tok = build_token(:error, @input[@pos])
          tok = self.class.error_hander.call(tok)
          if pos == @pos
            raise LexError.new("Illegal character '#{@input[@pos]}' at index #{@pos}")
          else
            return tok if tok && tok.type
          end
        else
          raise LexError.new("Illegal character '#{@input[@pos]}' at index #{@pos}")
        end

      end
      return nil
    end
Processes the next token in input

 This is the main interface to lexer. It returns next available token or **nil**
 if there are no more tokens available in the input string.

 {#each} Raises {LexError} if the input cannot be processed. This happens if
 there were no matches by 'token' rules and no matches by 'literals' rule.
 If the {.on_error} handler is not set, the exception will be raised immediately,
 however, if the handler is set, the eception will be raised only if the {#pos}
 after returning from error handler is still unchanged.

 @api public
 @raise [LexError] if the input cannot be processed
 @return [LexToken] if the next chunk of input was processed successfully
 @return [nil] if there are no more tokens available in input

 @example
   lex = MyLexer.new("hello WORLD")

   t = lex.next
   puts "#{tok.type} -> #{tok.value}" #=> "LOWERS -> hello"
   t = lex.next
   puts "#{tok.type} -> #{tok.value}" #=> "UPPERS -> WORLD"
   t = lex.next # => nil