def get(url, referer = nil)
      limit = redirect_limit
      loc = url
      loop do
        # if redirected to a relative url, merge it with the host of the original
        # request url
        loc = url.merge(loc) if loc.relative?

        response, response_time = get_response(loc, referer)
        code = Integer(response.code)
        redirect_to =
          begin
            response.is_a?(Net::HTTPRedirection) ? URI(response['location']).normalize : nil
          rescue URI::InvalidURIError => e
            @opts[:logger].debug { "Request #{url} got #{e}" } if @opts[:logger]
            nil
          end
        yield response, code, loc, redirect_to, response_time
        limit -= 1
        break unless (loc = redirect_to) && allowed?(redirect_to, url) && limit > 0
      end
    end
def get(url, referer = nil)
      limit = redirect_limit
      loc = url
      loop do
        # if redirected to a relative url, merge it with the host of the original
        # request url
        loc = url.merge(loc) if loc.relative?

        response, response_time = get_response(loc, referer)
        code = Integer(response.code)
        redirect_to =
          begin
            response.is_a?(Net::HTTPRedirection) ? URI(response['location']).normalize : nil
          rescue URI::InvalidURIError => e
            @opts[:logger].debug { "Request #{url} got #{e}" } if @opts[:logger]
            nil
          end
        yield response, code, loc, redirect_to, response_time
        limit -= 1
        break unless (loc = redirect_to) && allowed?(redirect_to, url) && limit > 0
      end
    end
Retrieve HTTP responses for *url*, including redirects.
 Yields the response object, response code, and URI location
 for each response.