def crawl(options = {})
      @errors = []
      i = 0
      each_url do |url, handler, default_data|
        break if options[:max_urls] && i >= options[:max_urls]
        begin
          page = agent.get(url)
          Spidey.logger.info "Handling #{url.inspect}"
          send handler, page, default_data
        rescue => ex
          add_error url: url, handler: handler, error: ex
        end
        sleep request_interval if request_interval > 0
        i += 1
      end
    end
def crawl(options = {})
      @errors = []
      i = 0
      each_url do |url, handler, default_data|
        break if options[:max_urls] && i >= options[:max_urls]
        begin
          page = agent.get(url)
          Spidey.logger.info "Handling #{url.inspect}"
          send handler, page, default_data
        rescue => ex
          add_error url: url, handler: handler, error: ex
        end
        sleep request_interval if request_interval > 0
        i += 1
      end
    end
Accepts:
   request_interval: number of seconds to wait between requests (default: 3)
 Iterates through URLs queued for handling, including any that are added in the course of crawling. Accepts:
   max_urls: maximum number of URLs to crawl before returning (optional)