def archivable?(document)
      # check if the document is likely to be a html page
      archivable_document = document.url.valid

      # check of the document hasn't already been evaluated and discarded
      has_been_marked_as_invalid = self.invalid_urls.include?(document.url.raw_url)

      # Add the dpcument to the list of needed
      if !archivable_document && !has_been_marked_as_invalid
        emitter.warning("> Invalid URL: #{document.url.raw_url}")
        invalid_urls << document.url.raw_url
      end
      archivable_document
    end
def archivable?(document)
      # check if the document is likely to be a html page
      archivable_document = document.url.valid

      # check of the document hasn't already been evaluated and discarded
      has_been_marked_as_invalid = self.invalid_urls.include?(document.url.raw_url)

      # Add the dpcument to the list of needed
      if !archivable_document && !has_been_marked_as_invalid
        emitter.warning("> Invalid URL: #{document.url.raw_url}")
        invalid_urls << document.url.raw_url
      end
      archivable_document
    end
Determine if a document is actually the type of document
 that can be archived