def should_be_visited?(url, with_tracker = true)
      case
      # robots.txt
      when !allowed_by_robot?(url)
        false
      # Check against whitelist pattern matching
      when !@follow_links_like.empty? && @follow_links_like.none? { |p| url.path =~ p }
        false
      # Check against blacklist pattern matching
      when @skip_links_like.any? { |p| url.path =~ p }
        false
      # Page is marked as expired
      when page_expired?(Page.new(url))
        true
      # Check against url tracker
      when with_tracker && url_tracker.visited?(@options[:include_query_string_in_saved_page] ? url.to_s : url.to_s.gsub(/\?.*$/, ''))
        false
      else
        true
      end
    end
def should_be_visited?(url, with_tracker = true)
      case
      # robots.txt
      when !allowed_by_robot?(url)
        false
      # Check against whitelist pattern matching
      when !@follow_links_like.empty? && @follow_links_like.none? { |p| url.path =~ p }
        false
      # Check against blacklist pattern matching
      when @skip_links_like.any? { |p| url.path =~ p }
        false
      # Page is marked as expired
      when page_expired?(Page.new(url))
        true
      # Check against url tracker
      when with_tracker && url_tracker.visited?(@options[:include_query_string_in_saved_page] ? url.to_s : url.to_s.gsub(/\?.*$/, ''))
        false
      else
        true
      end
    end
URLs enqueue policy