def get_response(url, referer = nil)
      full_path = url.query.nil? ? url.path : "#{url.path}?#{url.query}"

      opts = {}
      opts['User-Agent'] = user_agent if user_agent
      opts['Referer'] = referer.to_s if referer
      opts['Cookie']  = ::HTTP::Cookie.cookie_value(cookie_jar.cookies(url)) if accept_cookies?
      opts['Accept-Encoding'] = 'gzip,deflate'

      retries = 0
      begin
        start = Time.now
        # format request
        req = Net::HTTP::Get.new(full_path, opts)
        # HTTP Basic authentication
        req.basic_auth url.user, url.password if url.user
        if @opts[:http_user]
          req.basic_auth @opts[:http_user], @opts[:http_password]
        end
        # urls auth schema has higher priority
        req.basic_auth url.user, url.password if url.user
        response = connection(url).request(req)
        finish = Time.now
        response_time = ((finish - start) * 1000).round
        cookie_jar.parse(response['Set-Cookie'], url) if accept_cookies? && response['Set-Cookie']
        return response, response_time
      rescue *RESCUABLE_ERRORS => e
        puts e.inspect if verbose?
        refresh_connection(url)
        retries += 1
        if retries < 3
          retry
        else
          raise e
        end
      end
    end
def get_response(url, referer = nil)
      full_path = url.query.nil? ? url.path : "#{url.path}?#{url.query}"

      opts = {}
      opts['User-Agent'] = user_agent if user_agent
      opts['Referer'] = referer.to_s if referer
      opts['Cookie']  = ::HTTP::Cookie.cookie_value(cookie_jar.cookies(url)) if accept_cookies?
      opts['Accept-Encoding'] = 'gzip,deflate'

      retries = 0
      begin
        start = Time.now
        # format request
        req = Net::HTTP::Get.new(full_path, opts)
        # HTTP Basic authentication
        req.basic_auth url.user, url.password if url.user
        if @opts[:http_user]
          req.basic_auth @opts[:http_user], @opts[:http_password]
        end
        # urls auth schema has higher priority
        req.basic_auth url.user, url.password if url.user
        response = connection(url).request(req)
        finish = Time.now
        response_time = ((finish - start) * 1000).round
        cookie_jar.parse(response['Set-Cookie'], url) if accept_cookies? && response['Set-Cookie']
        return response, response_time
      rescue *RESCUABLE_ERRORS => e
        puts e.inspect if verbose?
        refresh_connection(url)
        retries += 1
        if retries < 3
          retry
        else
          raise e
        end
      end
    end
Get an HTTPResponse for *url*, sending the appropriate User-Agent string