def fetch
      puts "fetching #{@uri.to_s}".green.on_black
      start_time = Time.now
      begin
        request = Net::HTTP::Get.new(@uri.request_uri)
        request["User-Agent"] = UserAgents.random
        response = @http.request(request) 
        end_time = Time.now
        case response
        when Net::HTTPSuccess then
          page = Page.new(@uri, response_code: response.code.to_i,
                                response_head: response.instance_variable_get("@header"),
                                response_body: response.body,
                                response_time: (end_time-start_time).to_f,
                                crawled_time: (Time.now.to_f*1000).to_i)
        when Net::HTTPRedirection then
          page = Page.new(@uri, response_code: response.code.to_i,
                                response_head: response.instance_variable_get("@header"),
                                response_body: response.body,
                                response_time: (end_time-start_time).to_f,
                                redirect_url:  response['location'])
        when Net::HTTPNotFound then
          page = Page.new(@uri, response_code: response.code.to_i,
                                response_time: (end_time-start_time).to_f)
        end
      rescue Exception => e
        puts e.inspect
        puts e.backtrace
      end
    end
def fetch
      puts "fetching #{@uri.to_s}".green.on_black
      start_time = Time.now
      begin
        request = Net::HTTP::Get.new(@uri.request_uri)
        request["User-Agent"] = UserAgents.random
        response = @http.request(request) 
        end_time = Time.now
        case response
        when Net::HTTPSuccess then
          page = Page.new(@uri, response_code: response.code.to_i,
                                response_head: response.instance_variable_get("@header"),
                                response_body: response.body,
                                response_time: (end_time-start_time).to_f,
                                crawled_time: (Time.now.to_f*1000).to_i)
        when Net::HTTPRedirection then
          page = Page.new(@uri, response_code: response.code.to_i,
                                response_head: response.instance_variable_get("@header"),
                                response_body: response.body,
                                response_time: (end_time-start_time).to_f,
                                redirect_url:  response['location'])
        when Net::HTTPNotFound then
          page = Page.new(@uri, response_code: response.code.to_i,
                                response_time: (end_time-start_time).to_f)
        end
      rescue Exception => e
        puts e.inspect
        puts e.backtrace
      end
    end
Fetch a page from the given url using net/http