def amqp_send_retry(packet, parent_token, count = 0, multiplier = 1, elapsed = 0, broker_ids = nil)
      check_broker_ids = amqp_send_once(packet, broker_ids)

      if @retry_interval && @retry_timeout && parent_token
        interval = [(@retry_interval * multiplier) + (@request_stats.avg_duration || 0), @retry_timeout - elapsed].min
        EM.add_timer(interval) do
          begin
            if @pending_requests[parent_token]
              count += 1
              elapsed += interval
              if elapsed < @retry_timeout && (packet.expires_at <= 0 || Time.now.to_i < packet.expires_at)
                packet.tries << packet.token
                packet.token = RightSupport::Data::UUID.generate
                @pending_requests[parent_token].retry_parent_token = parent_token if count == 1
                @pending_requests[packet.token] = @pending_requests[parent_token]
                broker_ids ||= @agent.client.all
                amqp_send_retry(packet, parent_token, count, multiplier * RETRY_BACKOFF_FACTOR, elapsed,
                                broker_ids.push(broker_ids.shift))
                @retry_stats.update(packet.type.split('/').last)
              else
                Log.warning("RE-SEND TIMEOUT after #{elapsed.to_i} seconds for #{packet.trace} #{packet.type}")
                result = OperationResult.non_delivery(OperationResult::RETRY_TIMEOUT)
                handle_response(Result.new(packet.token, @identity, result, @identity))
              end
              @connectivity_checker.check(check_broker_ids.first) if check_broker_ids.any? && count == 1
            end
          rescue TemporarilyOffline => e
            ErrorTracker.log(self, "Failed retry for #{packet.trace} #{packet.type} because temporarily offline")
          rescue SendFailure => e
            ErrorTracker.log(self, "Failed retry for #{packet.trace} #{packet.type} because of send failure")
          rescue Exception => e
            # Not sending a response here because something more basic is broken in the retry
            # mechanism and don't want an error response to preempt a delayed actual response
            ErrorTracker.log(self, "Failed retry for #{packet.trace} #{packet.type} without responding", e, packet)
          end
        end
      end
      true
    end
def amqp_send_retry(packet, parent_token, count = 0, multiplier = 1, elapsed = 0, broker_ids = nil)
      check_broker_ids = amqp_send_once(packet, broker_ids)

      if @retry_interval && @retry_timeout && parent_token
        interval = [(@retry_interval * multiplier) + (@request_stats.avg_duration || 0), @retry_timeout - elapsed].min
        EM.add_timer(interval) do
          begin
            if @pending_requests[parent_token]
              count += 1
              elapsed += interval
              if elapsed < @retry_timeout && (packet.expires_at <= 0 || Time.now.to_i < packet.expires_at)
                packet.tries << packet.token
                packet.token = RightSupport::Data::UUID.generate
                @pending_requests[parent_token].retry_parent_token = parent_token if count == 1
                @pending_requests[packet.token] = @pending_requests[parent_token]
                broker_ids ||= @agent.client.all
                amqp_send_retry(packet, parent_token, count, multiplier * RETRY_BACKOFF_FACTOR, elapsed,
                                broker_ids.push(broker_ids.shift))
                @retry_stats.update(packet.type.split('/').last)
              else
                Log.warning("RE-SEND TIMEOUT after #{elapsed.to_i} seconds for #{packet.trace} #{packet.type}")
                result = OperationResult.non_delivery(OperationResult::RETRY_TIMEOUT)
                handle_response(Result.new(packet.token, @identity, result, @identity))
              end
              @connectivity_checker.check(check_broker_ids.first) if check_broker_ids.any? && count == 1
            end
          rescue TemporarilyOffline => e
            ErrorTracker.log(self, "Failed retry for #{packet.trace} #{packet.type} because temporarily offline")
          rescue SendFailure => e
            ErrorTracker.log(self, "Failed retry for #{packet.trace} #{packet.type} because of send failure")
          rescue Exception => e
            # Not sending a response here because something more basic is broken in the retry
            # mechanism and don't want an error response to preempt a delayed actual response
            ErrorTracker.log(self, "Failed retry for #{packet.trace} #{packet.type} without responding", e, packet)
          end
        end
      end
      true
    end
Send request via AMQP with one or more retries if do not receive a response in time
 Send timeout result if reach configured retry timeout limit
 Use exponential backoff with RETRY_BACKOFF_FACTOR for retry spacing
 Adjust retry interval by average response time to avoid adding to system load
 when system gets slow
 Rotate through brokers on retries
 Check connectivity after first retry timeout

 === Parameters
 packet(Request):: Request packet to send
 parent_token(String):: Token for original request
 count(Integer):: Number of retries so far
 multiplier(Integer):: Multiplier for retry interval for exponential backoff
 elapsed(Integer):: Elapsed time in seconds since this request was first attempted
 broker_ids(Array):: Identity of brokers to be used in priority order

 === Return
 true:: Always return true