def each_url(&_block)
      index = 0
      while index < urls.count # urls grows dynamically, don't use &:each
        url = urls[index]
        next unless url
        yield url, handlers[url].first, handlers[url].last
        index += 1
      end
    end
def each_url(&_block)
      index = 0
      while index < urls.count # urls grows dynamically, don't use &:each
        url = urls[index]
        next unless url
        yield url, handlers[url].first, handlers[url].last
        index += 1
      end
    end
Override this for custom storage or prioritization of crawled URLs.
 Iterates through URL queue, yielding the URL, handler, and default data.