def consume
      return nil if @s.eos?

      @s.mark

      # Consume comments.
      if comment_token = consume_comments
        if @options[:preserve_comments]
          return comment_token
        else
          return consume
        end
      end

      # Consume whitespace.
      return create_token(:whitespace) if @s.scan(RE_WHITESPACE)

      char = @s.consume

      case char.to_sym
      when :'"'
        consume_string

      when :'#'
        if @s.peek =~ RE_NAME || valid_escape?(@s.peek(2))
          create_token(:hash,
            :type  => start_identifier?(@s.peek(3)) ? :id : :unrestricted,
            :value => consume_name)
        else
          create_token(:delim, :value => char)
        end

      when :'$'
        if @s.peek == '='
          @s.consume
          create_token(:suffix_match)
        else
          create_token(:delim, :value => char)
        end

      when :"'"
        consume_string

      when :'('
        create_token(:'(')

      when :')'
        create_token(:')')

      when :*
        if @s.peek == '='
          @s.consume
          create_token(:substring_match)

        # Non-standard: Preserve the IE * hack.
        elsif @options[:preserve_hacks] && @s.peek =~ RE_NAME_START
          @s.reconsume
          consume_ident

        else
          create_token(:delim, :value => char)
        end

      when :+
        if start_number?
          @s.reconsume
          consume_numeric
        else
          create_token(:delim, :value => char)
        end

      when :','
        create_token(:comma)

      when :-
        nextTwoChars   = @s.peek(2)
        nextThreeChars = char + nextTwoChars

        if start_number?(nextThreeChars)
          @s.reconsume
          consume_numeric
        elsif nextTwoChars == '->'
          @s.consume
          @s.consume
          create_token(:cdc)
        elsif start_identifier?(nextThreeChars)
          @s.reconsume
          consume_ident
        else
          create_token(:delim, :value => char)
        end

      when :'.'
        if start_number?
          @s.reconsume
          consume_numeric
        else
          create_token(:delim, :value => char)
        end

      when :':'
        create_token(:colon)

      when :';'
        create_token(:semicolon)

      when :<
        if @s.peek(3) == '!--'
          @s.consume
          @s.consume
          @s.consume

          create_token(:cdo)
        else
          create_token(:delim, :value => char)
        end

      when :'@'
        if start_identifier?(@s.peek(3))
          create_token(:at_keyword, :value => consume_name)
        else
          create_token(:delim, :value => char)
        end

      when :'['
        create_token(:'[')

      when :'\\'
        if valid_escape?
          @s.reconsume
          consume_ident
        else
          # Parse error.
          create_token(:delim,
            :error => true,
            :value => char)
        end

      when :']'
        create_token(:']')

      when :'^'
        if @s.peek == '='
          @s.consume
          create_token(:prefix_match)
        else
          create_token(:delim, :value => char)
        end

      when :'{'
        create_token(:'{')

      when :'}'
        create_token(:'}')

      when :U, :u
        if @s.peek(2) =~ RE_UNICODE_RANGE_START
          @s.consume
          consume_unicode_range
        else
          @s.reconsume
          consume_ident
        end

      when :|
        case @s.peek
        when '='
          @s.consume
          create_token(:dash_match)

        when '|'
          @s.consume
          create_token(:column)

        else
          create_token(:delim, :value => char)
        end

      when :~
        if @s.peek == '='
          @s.consume
          create_token(:include_match)
        else
          create_token(:delim, :value => char)
        end

      else
        case char
        when RE_DIGIT
          @s.reconsume
          consume_numeric

        when RE_NAME_START
          @s.reconsume
          consume_ident

        else
          create_token(:delim, :value => char)
        end
      end
    end
def consume
      return nil if @s.eos?

      @s.mark

      # Consume comments.
      if comment_token = consume_comments
        if @options[:preserve_comments]
          return comment_token
        else
          return consume
        end
      end

      # Consume whitespace.
      return create_token(:whitespace) if @s.scan(RE_WHITESPACE)

      char = @s.consume

      case char.to_sym
      when :'"'
        consume_string

      when :'#'
        if @s.peek =~ RE_NAME || valid_escape?(@s.peek(2))
          create_token(:hash,
            :type  => start_identifier?(@s.peek(3)) ? :id : :unrestricted,
            :value => consume_name)
        else
          create_token(:delim, :value => char)
        end

      when :'$'
        if @s.peek == '='
          @s.consume
          create_token(:suffix_match)
        else
          create_token(:delim, :value => char)
        end

      when :"'"
        consume_string

      when :'('
        create_token(:'(')

      when :')'
        create_token(:')')

      when :*
        if @s.peek == '='
          @s.consume
          create_token(:substring_match)

        # Non-standard: Preserve the IE * hack.
        elsif @options[:preserve_hacks] && @s.peek =~ RE_NAME_START
          @s.reconsume
          consume_ident

        else
          create_token(:delim, :value => char)
        end

      when :+
        if start_number?
          @s.reconsume
          consume_numeric
        else
          create_token(:delim, :value => char)
        end

      when :','
        create_token(:comma)

      when :-
        nextTwoChars   = @s.peek(2)
        nextThreeChars = char + nextTwoChars

        if start_number?(nextThreeChars)
          @s.reconsume
          consume_numeric
        elsif nextTwoChars == '->'
          @s.consume
          @s.consume
          create_token(:cdc)
        elsif start_identifier?(nextThreeChars)
          @s.reconsume
          consume_ident
        else
          create_token(:delim, :value => char)
        end

      when :'.'
        if start_number?
          @s.reconsume
          consume_numeric
        else
          create_token(:delim, :value => char)
        end

      when :':'
        create_token(:colon)

      when :';'
        create_token(:semicolon)

      when :<
        if @s.peek(3) == '!--'
          @s.consume
          @s.consume
          @s.consume

          create_token(:cdo)
        else
          create_token(:delim, :value => char)
        end

      when :'@'
        if start_identifier?(@s.peek(3))
          create_token(:at_keyword, :value => consume_name)
        else
          create_token(:delim, :value => char)
        end

      when :'['
        create_token(:'[')

      when :'\\'
        if valid_escape?
          @s.reconsume
          consume_ident
        else
          # Parse error.
          create_token(:delim,
            :error => true,
            :value => char)
        end

      when :']'
        create_token(:']')

      when :'^'
        if @s.peek == '='
          @s.consume
          create_token(:prefix_match)
        else
          create_token(:delim, :value => char)
        end

      when :'{'
        create_token(:'{')

      when :'}'
        create_token(:'}')

      when :U, :u
        if @s.peek(2) =~ RE_UNICODE_RANGE_START
          @s.consume
          consume_unicode_range
        else
          @s.reconsume
          consume_ident
        end

      when :|
        case @s.peek
        when '='
          @s.consume
          create_token(:dash_match)

        when '|'
          @s.consume
          create_token(:column)

        else
          create_token(:delim, :value => char)
        end

      when :~
        if @s.peek == '='
          @s.consume
          create_token(:include_match)
        else
          create_token(:delim, :value => char)
        end

      else
        case char
        when RE_DIGIT
          @s.reconsume
          consume_numeric

        when RE_NAME_START
          @s.reconsume
          consume_ident

        else
          create_token(:delim, :value => char)
        end
      end
    end
-- Instance Methods ------------------------------------------------------
 Initializes a new Tokenizer.

 Options:

   * **:preserve_comments** - If `true`, comments will be preserved as
     `:comment` tokens.

   * **:preserve_hacks** - If `true`, certain non-standard browser hacks
     such as the IE "*" hack will be preserved even though they violate
     CSS 3 syntax rules.

 Consumes a token and returns the token that was consumed.

 4.3.1. http://dev.w3.org/csswg/css-syntax/#consume-a-token