def collect_urls
      @fetch_queue << @crawl_url.resolved_base_url
      until @fetch_queue.empty? || @processed.length >= @options[:max_requests]
        url = @fetch_queue.pop
        yield(url)
        page_urls_for(url)
      end
      result = @processed + @fetch_queue
      Logger.log "Crawling finished:"
      Logger.log "Processed links: #{@processed.length}"
      Logger.log "Found links:     #{result.length}"
      result.to_a
    rescue Interrupt, IRB::Abort
      Logger.err_log 'Crawl interrupted.'
      @fetch_queue.to_a
    end
def collect_urls
      @fetch_queue << @crawl_url.resolved_base_url
      until @fetch_queue.empty? || @processed.length >= @options[:max_requests]
        url = @fetch_queue.pop
        yield(url)
        page_urls_for(url)
      end
      result = @processed + @fetch_queue
      Logger.log "Crawling finished:"
      Logger.log "Processed links: #{@processed.length}"
      Logger.log "Found links:     #{result.length}"
      result.to_a
    rescue Interrupt, IRB::Abort
      Logger.err_log 'Crawl interrupted.'
      @fetch_queue.to_a
    end
Collects all links on domain for domain.
 @return [Array] with links.
 @example URLs for example.com
   crawler = Crawler.new('example.com')
   crawler.collect_urls
 @example URLs for example.com with block (executes in its own thread)
   crawler = Crawler.new('example.com')
   crawler.collect_urls do |new_url|
     puts "New URL found: #{new_url}"
   end