def execute_job
      # Ensure that a killed worker cleans up correctly
      Signal.trap('TERM') do |s|
        Signal.trap('TERM') {} # Prevent retry signals
        @log.info "Caught TERM signal"
        @log.info "Terminating script, if running"
        @script.terminate if @script
        @log.info "Post-execution cleanup"
        signal_safe_post_script(@job, @file_system, @script)

        # Upload results
        @file_system.finalise_results_directory
        upload_files(@job, @file_system.results_path, @file_system.logs_path)
        set_job_state_to :completed
        @job.error('Worker killed')
        @log.info "Worker terminated"
        exit
      end

      @log.info('Job starting')
      @job.prepare(@hive_mind.id)
      exception = nil
      begin
        @log.info "Setting job paths"
        @file_system = Hive::FileSystem.new(@job.job_id, Hive.config.logging.home, @log)
        set_job_state_to :preparing

        if ! @job.repository.to_s.empty?
          @log.info "Checking out the repository"
          @log.debug "  #{@job.repository}"
          @log.debug "  #{@file_system.testbed_path}"
          checkout_code(@job.repository, @file_system.testbed_path, @job.branch)
        end

        @log.info "Initialising execution script"
        @script = Hive::ExecutionScript.new(
          file_system: @file_system,
          log: @log,
          keep_running: ->() { self.keep_script_running? }
        )
        @script.append_bash_cmd "mkdir -p #{@file_system.testbed_path}/#{@job.execution_directory}"
        @script.append_bash_cmd "cd #{@file_system.testbed_path}/#{@job.execution_directory}"

        @log.info "Setting the execution variables in the environment"
        @script.set_env 'HIVE_RESULTS', @file_system.results_path
        @script.set_env 'HIVE_SCRIPT_ERRORS', @file_system.script_errors_file
        @job.execution_variables.to_h.each_pair do |var, val|
          @script.set_env "HIVE_#{var.to_s}".upcase, val if ! val.kind_of?(Array)
        end
        if @job.execution_variables.retry_urns && !@job.execution_variables.retry_urns.empty?
          @script.set_env "RETRY_URNS", @job.execution_variables.retry_urns
        end
        if @job.execution_variables.tests && @job.execution_variables.tests != [""]
          @script.set_env "TEST_NAMES", @job.execution_variables.tests
        end
        

        @log.info "Appending test script to execution script"
        @script.append_bash_cmd @job.command

        set_job_state_to :running

        @log.info "Pre-execution setup"
        pre_script(@job, @file_system, @script)

        @job.start
        @log.info "Running execution script"
        exit_value = @script.run
        @job.end(exit_value)
      rescue => e
        exception = e
      end

      begin
        @log.info "Post-execution cleanup"
        set_job_state_to :uploading
        post_script(@job, @file_system, @script)

        # Upload results
        @file_system.finalise_results_directory
        upload_results(@job, "#{@file_system.testbed_path}/#{@job.execution_directory}", @file_system.results_path)
      rescue => e
        @log.error( "Post execution failed: " + e.message)
        @log.error("  : #{e.backtrace.join("\n  : ")}")
      end

      if exception or File.size(@file_system.script_errors_file) > 0
        set_job_state_to :completed
        begin
          after_error(@job, @file_system, @script)
          upload_files(@job, @file_system.results_path, @file_system.logs_path)
        rescue => e
          @log.error("Exception while uploading files: #{e.backtrace.join("\n  : ")}")
        end
        if exception
          @job.error( exception.message )
          raise exception
        else
          @job.error( 'Errors raised by execution script' )
          raise 'See errors file for errors reported in test.'
        end
      else
        @job.complete
        begin
          upload_files(@job, @file_system.results_path, @file_system.logs_path)
        rescue => e
          @log.error("Exception while uploading files: #{e.backtrace.join("\n  : ")}")
        end
      end

      Signal.trap('TERM') do
        @log.info("Worker terminated")
        exit
      end

      set_job_state_to :completed
      exit_value == 0
    end
def execute_job
      # Ensure that a killed worker cleans up correctly
      Signal.trap('TERM') do |s|
        Signal.trap('TERM') {} # Prevent retry signals
        @log.info "Caught TERM signal"
        @log.info "Terminating script, if running"
        @script.terminate if @script
        @log.info "Post-execution cleanup"
        signal_safe_post_script(@job, @file_system, @script)

        # Upload results
        @file_system.finalise_results_directory
        upload_files(@job, @file_system.results_path, @file_system.logs_path)
        set_job_state_to :completed
        @job.error('Worker killed')
        @log.info "Worker terminated"
        exit
      end

      @log.info('Job starting')
      @job.prepare(@hive_mind.id)
      exception = nil
      begin
        @log.info "Setting job paths"
        @file_system = Hive::FileSystem.new(@job.job_id, Hive.config.logging.home, @log)
        set_job_state_to :preparing

        if ! @job.repository.to_s.empty?
          @log.info "Checking out the repository"
          @log.debug "  #{@job.repository}"
          @log.debug "  #{@file_system.testbed_path}"
          checkout_code(@job.repository, @file_system.testbed_path, @job.branch)
        end

        @log.info "Initialising execution script"
        @script = Hive::ExecutionScript.new(
          file_system: @file_system,
          log: @log,
          keep_running: ->() { self.keep_script_running? }
        )
        @script.append_bash_cmd "mkdir -p #{@file_system.testbed_path}/#{@job.execution_directory}"
        @script.append_bash_cmd "cd #{@file_system.testbed_path}/#{@job.execution_directory}"

        @log.info "Setting the execution variables in the environment"
        @script.set_env 'HIVE_RESULTS', @file_system.results_path
        @script.set_env 'HIVE_SCRIPT_ERRORS', @file_system.script_errors_file
        @job.execution_variables.to_h.each_pair do |var, val|
          @script.set_env "HIVE_#{var.to_s}".upcase, val if ! val.kind_of?(Array)
        end
        if @job.execution_variables.retry_urns && !@job.execution_variables.retry_urns.empty?
          @script.set_env "RETRY_URNS", @job.execution_variables.retry_urns
        end
        if @job.execution_variables.tests && @job.execution_variables.tests != [""]
          @script.set_env "TEST_NAMES", @job.execution_variables.tests
        end
        

        @log.info "Appending test script to execution script"
        @script.append_bash_cmd @job.command

        set_job_state_to :running

        @log.info "Pre-execution setup"
        pre_script(@job, @file_system, @script)

        @job.start
        @log.info "Running execution script"
        exit_value = @script.run
        @job.end(exit_value)
      rescue => e
        exception = e
      end

      begin
        @log.info "Post-execution cleanup"
        set_job_state_to :uploading
        post_script(@job, @file_system, @script)

        # Upload results
        @file_system.finalise_results_directory
        upload_results(@job, "#{@file_system.testbed_path}/#{@job.execution_directory}", @file_system.results_path)
      rescue => e
        @log.error( "Post execution failed: " + e.message)
        @log.error("  : #{e.backtrace.join("\n  : ")}")
      end

      if exception or File.size(@file_system.script_errors_file) > 0
        set_job_state_to :completed
        begin
          after_error(@job, @file_system, @script)
          upload_files(@job, @file_system.results_path, @file_system.logs_path)
        rescue => e
          @log.error("Exception while uploading files: #{e.backtrace.join("\n  : ")}")
        end
        if exception
          @job.error( exception.message )
          raise exception
        else
          @job.error( 'Errors raised by execution script' )
          raise 'See errors file for errors reported in test.'
        end
      else
        @job.complete
        begin
          upload_files(@job, @file_system.results_path, @file_system.logs_path)
        rescue => e
          @log.error("Exception while uploading files: #{e.backtrace.join("\n  : ")}")
        end
      end

      Signal.trap('TERM') do
        @log.info("Worker terminated")
        exit
      end

      set_job_state_to :completed
      exit_value == 0
    end
Execute a job