def extract_filter_tokens(expression = nil)   #:nodoc:
    return [] unless expression

    tokens = []
    str = expression
    while str =~ /"(?:[^"]|"")*"|\S+/
      tokens << $&
      str = $~.post_match
    end

    # Remove leading and trailing quotes and unescape other quotes
    tokens.map! do |token|
      token.sub!(/^"/, '')
      token.sub!(/"$/, '')
      token.gsub!(/""/, '"')

      # if token is number, convert to numeric.
      if token =~ /^([+-]?)(?=\d|\.\d)\d*(\.\d*)?([Ee]([+-]?\d+))?$/
        token.to_f == token.to_i ? token.to_i : token.to_f
      else
        token
      end
    end

    tokens
  end
def extract_filter_tokens(expression = nil)   #:nodoc:
    return [] unless expression

    tokens = []
    str = expression
    while str =~ /"(?:[^"]|"")*"|\S+/
      tokens << $&
      str = $~.post_match
    end

    # Remove leading and trailing quotes and unescape other quotes
    tokens.map! do |token|
      token.sub!(/^"/, '')
      token.sub!(/"$/, '')
      token.gsub!(/""/, '"')

      # if token is number, convert to numeric.
      if token =~ /^([+-]?)(?=\d|\.\d)\d*(\.\d*)?([Ee]([+-]?\d+))?$/
        token.to_f == token.to_i ? token.to_i : token.to_f
      else
        token
      end
    end

    tokens
  end
Extract the tokens from the filter expression. The tokens are mainly non-
 whitespace groups. The only tricky part is to extract string tokens that
 contain whitespace and/or quoted double quotes (Excel's escaped quotes).

 Examples: 'x <  2000'
           'x >  2000 and x <  5000'
           'x = "foo"'
           'x = "foo bar"'
           'x = "foo "" bar"'