def strike(link, cache_key = nil, yield_exception = false)
      request = Typhoeus::Request.new(link, :followlocation => true, headers: {"User-Agent"=>"Mozilla/5.0"})
      if cache_key
        # look for cached version
        response = @cache[cache_key]
        unless response.nil?
          @logger.debug "Cache hit: #{cache_key}"
          yield request, response
          return
        end
      end
      request.on_complete do |response|
        if response.code == 0
          # Could not get an http response, something's wrong.
          err = "ERROR: Unknown error (#{response}) while requesting #{link}"
          @logger.error err
          yield request, Exception.new(err) if yield_exception
        elsif response.timed_out?
          # aw hell no
          err = "ERROR: Timed out while requesting #{link}"
          @logger.error err
          yield request, Exception.new(err) if yield_exception
        elsif response.success? || response.code - 200 < 100
          # in the middle of such dead slow network/processing, I am optimizing a compare and an AND! #funny 
          begin
            @cache[cache_key] = response.body if cache_key
            yield request, response.body
          rescue => e
            @logger.warn "WARNING: Exception while processing response for #{link}"
            @logger.warn e
          end
        else
          # Received a non-successful http response.
          err = "ERROR: HTTP request failed: #{response.code.to_s} while requesting #{link}"
          @logger.error err
          yield request, Exception.new(err) if yield_exception
        end
        @done_requests += 1
        @pending_requests -= 1
        check_killed
      end
      @pending_requests += 1
      @hydra.queue(request)
      @logger.debug "++++ Hydra has #{@hydra.queued_requests.length} queued requests"
      # prevent queue from growing too big, thus delaying hydra.run too much
      @hydra.run if @hydra.queued_requests.length > @max_hydra_queue_length
    end
def strike(link, cache_key = nil, yield_exception = false)
      request = Typhoeus::Request.new(link, :followlocation => true, headers: {"User-Agent"=>"Mozilla/5.0"})
      if cache_key
        # look for cached version
        response = @cache[cache_key]
        unless response.nil?
          @logger.debug "Cache hit: #{cache_key}"
          yield request, response
          return
        end
      end
      request.on_complete do |response|
        if response.code == 0
          # Could not get an http response, something's wrong.
          err = "ERROR: Unknown error (#{response}) while requesting #{link}"
          @logger.error err
          yield request, Exception.new(err) if yield_exception
        elsif response.timed_out?
          # aw hell no
          err = "ERROR: Timed out while requesting #{link}"
          @logger.error err
          yield request, Exception.new(err) if yield_exception
        elsif response.success? || response.code - 200 < 100
          # in the middle of such dead slow network/processing, I am optimizing a compare and an AND! #funny 
          begin
            @cache[cache_key] = response.body if cache_key
            yield request, response.body
          rescue => e
            @logger.warn "WARNING: Exception while processing response for #{link}"
            @logger.warn e
          end
        else
          # Received a non-successful http response.
          err = "ERROR: HTTP request failed: #{response.code.to_s} while requesting #{link}"
          @logger.error err
          yield request, Exception.new(err) if yield_exception
        end
        @done_requests += 1
        @pending_requests -= 1
        check_killed
      end
      @pending_requests += 1
      @hydra.queue(request)
      @logger.debug "++++ Hydra has #{@hydra.queued_requests.length} queued requests"
      # prevent queue from growing too big, thus delaying hydra.run too much
      @hydra.run if @hydra.queued_requests.length > @max_hydra_queue_length
    end
hydra_queue