def run_with_retry(&block)
      Log.debug "Running job '#{@job_id}' with status '#{@status}'"
      # If we are not the first job in this group, wait @delay
      ChimpDaemon.instance.semaphore.synchronize do
        if @group.started >= @concurrency && @delay.nonzero?
          Log.info "[#{@job_uuid}] Sleeping #{@delay} seconds between tasks"
          sleep @delay
        end
        @group.started += 1
      end

      @status = STATUS_RUNNING
      @time_start = Time.now

      Log.info self.describe_work_start unless @quiet

      #
      # The inner level of exception handling here tries to catch anything
      # that can be easily retired or failed-- normal exceptions.
      #
      # The outer level of exception handling handles weird stuff; for example,
      # sometimes rest_connection raises RuntimeError exceptions...
      #
      # This fixes acu75562.
      #
      begin
        begin
          yield if not @dry_run

          if @owner != nil
            @status = STATUS_DONE
            @group.job_completed
          else
            Log.warn "[#{@job_uuid}][#{@job_id}] Ownership of job_id #{job_id} lost. User cancelled operation?"
          end

        rescue SystemExit, Interrupt => ex
          $stderr.puts 'Exiting!'
          raise ex

        rescue Interrupt => ex
          name = @array['name'] if @array
          name = @server['name'] || @server['nickname'] if @server
          Log.error self.describe_work_error

          if @retry_count > 0
            @status = STATUS_RETRYING
            Log.error "[#{@job_uuid}][#{@job_id}] Error executing on \"#{name}\". Retrying in #{@retry_sleep} seconds..."
            @retry_count -= 1
            sleep @retry_sleep
            retry
          end

          @status = STATUS_ERROR
          @error = ex
          Log.error "[#{@job_uuid}][#{@job_id}] Error executing on \"#{name}\": #{ex}"

        ensure
          @time_end = Time.now
          Log.info self.describe_work_done unless @quiet
        end

      rescue RuntimeError => ex
        err = ex.message + "IP: #{@server.params["ip_address"]}\n" if @server.params['ip_address']
        err += " Group: #{@group.group_id}\n" if @group.group_id
        err += " Notes: #{@job_notes}\n" if @job_notes
        err += " Notes: #{@job_notes}\n" if @job_notes
        Log.error "[#{@job_uuid}][#{@job_id}] Caught RuntimeError: #{err} Job failed.\n"
        @status = STATUS_ERROR
        @error = ex
      end
    end
def run_with_retry(&block)
      Log.debug "Running job '#{@job_id}' with status '#{@status}'"
      # If we are not the first job in this group, wait @delay
      ChimpDaemon.instance.semaphore.synchronize do
        if @group.started >= @concurrency && @delay.nonzero?
          Log.info "[#{@job_uuid}] Sleeping #{@delay} seconds between tasks"
          sleep @delay
        end
        @group.started += 1
      end

      @status = STATUS_RUNNING
      @time_start = Time.now

      Log.info self.describe_work_start unless @quiet

      #
      # The inner level of exception handling here tries to catch anything
      # that can be easily retired or failed-- normal exceptions.
      #
      # The outer level of exception handling handles weird stuff; for example,
      # sometimes rest_connection raises RuntimeError exceptions...
      #
      # This fixes acu75562.
      #
      begin
        begin
          yield if not @dry_run

          if @owner != nil
            @status = STATUS_DONE
            @group.job_completed
          else
            Log.warn "[#{@job_uuid}][#{@job_id}] Ownership of job_id #{job_id} lost. User cancelled operation?"
          end

        rescue SystemExit, Interrupt => ex
          $stderr.puts 'Exiting!'
          raise ex

        rescue Interrupt => ex
          name = @array['name'] if @array
          name = @server['name'] || @server['nickname'] if @server
          Log.error self.describe_work_error

          if @retry_count > 0
            @status = STATUS_RETRYING
            Log.error "[#{@job_uuid}][#{@job_id}] Error executing on \"#{name}\". Retrying in #{@retry_sleep} seconds..."
            @retry_count -= 1
            sleep @retry_sleep
            retry
          end

          @status = STATUS_ERROR
          @error = ex
          Log.error "[#{@job_uuid}][#{@job_id}] Error executing on \"#{name}\": #{ex}"

        ensure
          @time_end = Time.now
          Log.info self.describe_work_done unless @quiet
        end

      rescue RuntimeError => ex
        err = ex.message + "IP: #{@server.params["ip_address"]}\n" if @server.params['ip_address']
        err += " Group: #{@group.group_id}\n" if @group.group_id
        err += " Notes: #{@job_notes}\n" if @job_notes
        err += " Notes: #{@job_notes}\n" if @job_notes
        Log.error "[#{@job_uuid}][#{@job_id}] Caught RuntimeError: #{err} Job failed.\n"
        @status = STATUS_ERROR
        @error = ex
      end
    end
Run a unit of work with retries
 This is called from the subclass with a code block to yield to