#include <chrono>
#include <iostream>
#include <cuda_runtime.h>
#define CUDA_CHECK(ans) \
  { gpuAssert((ans), __FILE__, __LINE__); }

inline void gpuAssert(cudaError_t code,
                      const char *file,
                      int line,
                      bool abort = true) {
  if(code != cudaSuccess) {
    std::cerr <<"GPUassert: " << cudaGetErrorString(code) << " " << file << " "
                              << line << std::endl;
    exit(1);
  }
}


//Copy from other GPUs to this device
double normalMemCpyOther2This(float ** gpu_ptrs, int num_gpus, int bytes_to_transfer_each, int curr_gpu) {
    std::chrono::high_resolution_clock::time_point start = std::chrono::high_resolution_clock::now();
    for (int i = 0; i < num_gpus; i++) {
        if (i != curr_gpu) {
            CUDA_CHECK(cudaMemcpy(gpu_ptrs[curr_gpu] + (bytes_to_transfer_each/4/num_gpus)*i,
             gpu_ptrs[i] + (bytes_to_transfer_each/4/num_gpus)*i,
             bytes_to_transfer_each/num_gpus,
                cudaMemcpyDefault));
        }
        cudaSetDevice(i);
        cudaDeviceSynchronize();
        cudaSetDevice(curr_gpu);
        cudaDeviceSynchronize();

    }
    std::chrono::high_resolution_clock::time_point end = std::chrono::high_resolution_clock::now();
    std::chrono::duration<double> secs = end-start;
    return secs.count();
}

//From this device to other GPUS, async
double normalMemCpyThis2Other(float ** gpu_ptrs, int num_gpus, int bytes_to_transfer_each, int curr_gpu) {
    std::chrono::high_resolution_clock::time_point start = std::chrono::high_resolution_clock::now();
    for (int i = 0; i < num_gpus; i++) {
        if (i != curr_gpu) {
            CUDA_CHECK(cudaMemcpy(gpu_ptrs[i] + (bytes_to_transfer_each/4/num_gpus)*i,
             gpu_ptrs[curr_gpu] + (bytes_to_transfer_each/4/num_gpus)*i,
             bytes_to_transfer_each/num_gpus,
                cudaMemcpyDefault));
        }
        cudaSetDevice(i);
        cudaDeviceSynchronize();
        cudaSetDevice(curr_gpu);
        cudaDeviceSynchronize();

    }
    std::chrono::high_resolution_clock::time_point end = std::chrono::high_resolution_clock::now();
    std::chrono::duration<double> secs = end-start;
    return secs.count();
}

//Copy from other GPUs to this device, async
double normalMemCpyOther2ThisAsync(float ** gpu_ptrs, int num_gpus, int bytes_to_transfer_each, int curr_gpu) {
    cudaStream_t streams[num_gpus];
    for (int i = 0; i < num_gpus; i++) {
        CUDA_CHECK(cudaStreamCreate(&streams[i]));
    }

    std::chrono::high_resolution_clock::time_point start = std::chrono::high_resolution_clock::now();
    for (int i = 0; i < num_gpus; i++) {
        if (i != curr_gpu) {
            CUDA_CHECK(cudaMemcpyAsync(gpu_ptrs[curr_gpu] + (bytes_to_transfer_each/4/num_gpus)*i,
             gpu_ptrs[i] + (bytes_to_transfer_each/4/num_gpus)*i,
             bytes_to_transfer_each/num_gpus,
                cudaMemcpyDefault, streams[i]));
        }
        cudaSetDevice(i);
        cudaDeviceSynchronize();
        cudaSetDevice(curr_gpu);
        cudaDeviceSynchronize();
    }
    
    for (int i = 0; i < num_gpus; i++) {
        CUDA_CHECK(cudaStreamSynchronize(streams[i]));
    }
    
    std::chrono::high_resolution_clock::time_point end = std::chrono::high_resolution_clock::now();
    std::chrono::duration<double> secs = end-start;
    return secs.count();
}

//From this device to other GPUS, async
double normalMemCpyThis2OtherAsync(float ** gpu_ptrs, int num_gpus, int bytes_to_transfer_each, int curr_gpu) {
    cudaStream_t streams[num_gpus];
    for (int i = 0; i < num_gpus; i++) {
        CUDA_CHECK(cudaStreamCreate(&streams[i]));
    }

    std::chrono::high_resolution_clock::time_point start = std::chrono::high_resolution_clock::now();
    for (int i = 0; i < num_gpus; i++) {
        if (i != curr_gpu) {
            CUDA_CHECK(cudaMemcpyAsync(gpu_ptrs[i] + (bytes_to_transfer_each/4/num_gpus)*i,
             gpu_ptrs[curr_gpu] + (bytes_to_transfer_each/4/num_gpus)*i,
             bytes_to_transfer_each/num_gpus,
                cudaMemcpyDefault, streams[i]));
        }
        cudaSetDevice(i);
        cudaDeviceSynchronize();
        cudaSetDevice(curr_gpu);
        cudaDeviceSynchronize();

    }

    for (int i = 0; i < num_gpus; i++) {
        CUDA_CHECK(cudaStreamSynchronize(streams[i]));
    }

    std::chrono::high_resolution_clock::time_point end = std::chrono::high_resolution_clock::now();
    std::chrono::duration<double> secs = end-start;
    return secs.count();
}

int main(int argc, char * argv[]) {

    if (argc != 3) {
        std::cerr << "Usage: " << argv[0] << " num gpus total_model_memory" << std::endl;
        exit(1);
    }

    int num_gpus = atoi(argv[1]);

    int bytes_to_transfer_each = atoi(argv[2])*1024*1024;

    float ** gpu_ptrs = new float*[num_gpus];

    //Initiate with random memory, we don't care what it is. Also init the streams
    for (int i = 0; i < num_gpus; i++) {
        cudaSetDevice(i);
        CUDA_CHECK(cudaMalloc(&gpu_ptrs[i], bytes_to_transfer_each));
    }

    auto other_this = normalMemCpyOther2This(gpu_ptrs, num_gpus, bytes_to_transfer_each, 0);
    auto this_other = normalMemCpyThis2Other(gpu_ptrs, num_gpus, bytes_to_transfer_each, 0);
    auto other_thisAsync = normalMemCpyOther2ThisAsync(gpu_ptrs, num_gpus, bytes_to_transfer_each, 0);
    auto this_otherAsync = normalMemCpyThis2OtherAsync(gpu_ptrs, num_gpus, bytes_to_transfer_each, 0);
    
    for (int i = 1; i < num_gpus; i++) {
      other_this += normalMemCpyOther2This(gpu_ptrs, num_gpus, bytes_to_transfer_each, i);
      this_other += normalMemCpyThis2Other(gpu_ptrs, num_gpus, bytes_to_transfer_each, i);
      other_thisAsync += normalMemCpyOther2ThisAsync(gpu_ptrs, num_gpus, bytes_to_transfer_each, i);
      this_otherAsync += normalMemCpyThis2OtherAsync(gpu_ptrs, num_gpus, bytes_to_transfer_each, i);
    }

    std::cout << std::fixed << "Other to this took: " << other_this << " seconds." << std::endl;
    std::cout << std::fixed << "This to other took: " << this_other << " seconds." << std::endl;
    std::cout << std::fixed << "Other to this Async took: " << other_this << " seconds." << std::endl;
    std::cout << std::fixed << "This to other Async took: " << this_other << " seconds." << std::endl;

    //Attempt to enable peer access
    for (int i = 0; i<num_gpus; i++) {
        for (int j = 0; j<num_gpus; j++) {
            if (i != j) {
                int result;
                CUDA_CHECK(cudaDeviceCanAccessPeer(&result, i, j));
                if (result) {
                    cudaSetDevice(i);
                    cudaDeviceEnablePeerAccess (j, 0);
                } else {
                    std::cout << std::fixed << "Peer access unavailable between devices: " << i << " and " << j << std::endl;
                }
            }
        }
    }

    //Redo the benchmarks, see if it is any different

    other_this = normalMemCpyOther2This(gpu_ptrs, num_gpus, bytes_to_transfer_each, 0);
    this_other = normalMemCpyThis2Other(gpu_ptrs, num_gpus, bytes_to_transfer_each, 0);
    other_thisAsync = normalMemCpyOther2ThisAsync(gpu_ptrs, num_gpus, bytes_to_transfer_each, 0);
    this_otherAsync = normalMemCpyThis2OtherAsync(gpu_ptrs, num_gpus, bytes_to_transfer_each, 0);
    
    for (int i = 1; i < num_gpus; i++) {
      other_this += normalMemCpyOther2This(gpu_ptrs, num_gpus, bytes_to_transfer_each, i);
      this_other += normalMemCpyThis2Other(gpu_ptrs, num_gpus, bytes_to_transfer_each, i);
      other_thisAsync += normalMemCpyOther2ThisAsync(gpu_ptrs, num_gpus, bytes_to_transfer_each, i);
      this_otherAsync += normalMemCpyThis2OtherAsync(gpu_ptrs, num_gpus, bytes_to_transfer_each, i);
    }

    std::cout << std::fixed << "Peer other to this took: " << other_this << " seconds." << std::endl;
    std::cout << std::fixed << "Peer this to other took: " << this_other << " seconds." << std::endl;
    std::cout << std::fixed << "Peer other to this Async took: " << other_this << " seconds." << std::endl;
    std::cout << std::fixed << "Peer this to other Async took: " << this_other << " seconds." << std::endl;

    return 0;

}

	.file	"tmpxft_003b7180_00000000-6_memory_tester.cudafe1.cpp"
	.text
#APP
	.globl _ZSt21ios_base_library_initv

#NO_APP
	.section	.text._ZSt5fixedRSt8ios_base,"axG",@progbits,_ZSt5fixedRSt8ios_base,comdat
	.weak	_ZSt5fixedRSt8ios_base
	.type	_ZSt5fixedRSt8ios_base, @function
_ZSt5fixedRSt8ios_base:
.LFB3384:
	.cfi_startproc
	endbr64
	movl	24(%rdi), %edx
	movq	%rdi, %rax
	andl	$-261, %edx
	orl	$4, %edx
	movl	%edx, 24(%rdi)
	ret
	.cfi_endproc
.LFE3384:
	.size	_ZSt5fixedRSt8ios_base, .-_ZSt5fixedRSt8ios_base
	.text
	.type	_ZL26__cudaUnregisterBinaryUtilv, @function
_ZL26__cudaUnregisterBinaryUtilv:
.LFB3746:
	.cfi_startproc
	endbr64
	movq	_ZL20__cudaFatCubinHandle(%rip), %rdi
	jmp	__cudaUnregisterFatBinary@PLT
	.cfi_endproc
.LFE3746:
	.size	_ZL26__cudaUnregisterBinaryUtilv, .-_ZL26__cudaUnregisterBinaryUtilv
	.align 2
	.type	_ZNSolsEPFRSoS_E.isra.0, @function
_ZNSolsEPFRSoS_E.isra.0:
.LFB4453:
	.cfi_startproc
	jmp	*%rsi
	.cfi_endproc
.LFE4453:
	.size	_ZNSolsEPFRSoS_E.isra.0, .-_ZNSolsEPFRSoS_E.isra.0
	.section	.rodata.str1.1,"aMS",@progbits,1
.LC0:
	.string	"GPUassert: "
.LC1:
	.string	" "
	.text
	.type	_Z9gpuAssert9cudaErrorPKcib.isra.0, @function
_Z9gpuAssert9cudaErrorPKcib.isra.0:
.LFB4454:
	.cfi_startproc
	testl	%edi, %edi
	je	.L4
	pushq	%r13
	.cfi_def_cfa_offset 16
	.cfi_offset 13, -16
	pushq	%r12
	.cfi_def_cfa_offset 24
	.cfi_offset 12, -24
	movq	%rsi, %r12
	leaq	.LC0(%rip), %rsi
	pushq	%rbp
	.cfi_def_cfa_offset 32
	.cfi_offset 6, -32
	movl	%edx, %ebp
	pushq	%rbx
	.cfi_def_cfa_offset 40
	.cfi_offset 3, -40
	movl	%edi, %ebx
	leaq	_ZSt4cerr(%rip), %rdi
	pushq	%rax
	.cfi_def_cfa_offset 48
	call	_ZStlsISt11char_traitsIcEERSt13basic_ostreamIcT_ES5_PKc@PLT
	movl	%ebx, %edi
	leaq	.LC1(%rip), %rbx
	movq	%rax, %r13
	call	cudaGetErrorString@PLT
	movq	%r13, %rdi
	movq	%rax, %rsi
	call	_ZStlsISt11char_traitsIcEERSt13basic_ostreamIcT_ES5_PKc@PLT
	movq	%rbx, %rsi
	movq	%rax, %rdi
	call	_ZStlsISt11char_traitsIcEERSt13basic_ostreamIcT_ES5_PKc@PLT
	movq	%r12, %rsi
	movq	%rax, %rdi
	call	_ZStlsISt11char_traitsIcEERSt13basic_ostreamIcT_ES5_PKc@PLT
	movq	%rbx, %rsi
	movq	%rax, %rdi
	call	_ZStlsISt11char_traitsIcEERSt13basic_ostreamIcT_ES5_PKc@PLT
	movl	%ebp, %esi
	movq	%rax, %rdi
	call	_ZNSolsEi@PLT
	movq	_ZSt4endlIcSt11char_traitsIcEERSt13basic_ostreamIT_T0_ES6_@GOTPCREL(%rip), %rsi
	movq	%rax, %rdi
	call	_ZNSolsEPFRSoS_E.isra.0
	movl	$1, %edi
	call	exit@PLT
.L4:
	.cfi_def_cfa_offset 8
	.cfi_restore 3
	.cfi_restore 6
	.cfi_restore 12
	.cfi_restore 13
	ret
	.cfi_endproc
.LFE4454:
	.size	_Z9gpuAssert9cudaErrorPKcib.isra.0, .-_Z9gpuAssert9cudaErrorPKcib.isra.0
	.section	.text.startup,"ax",@progbits
	.type	_ZL24__sti____cudaRegisterAllv, @function
_ZL24__sti____cudaRegisterAllv:
.LFB3769:
	.cfi_startproc
	endbr64
	pushq	%rax
	.cfi_def_cfa_offset 16
	leaq	_ZL15__fatDeviceText(%rip), %rdi
	call	__cudaRegisterFatBinary@PLT
	movq	%rax, %rdi
	movq	%rax, _ZL20__cudaFatCubinHandle(%rip)
	call	__cudaRegisterFatBinaryEnd@PLT
	leaq	_ZL26__cudaUnregisterBinaryUtilv(%rip), %rdi
	popq	%rdx
	.cfi_def_cfa_offset 8
	jmp	atexit@PLT
	.cfi_endproc
.LFE3769:
	.size	_ZL24__sti____cudaRegisterAllv, .-_ZL24__sti____cudaRegisterAllv
	.section	.init_array,"aw"
	.align 8
	.quad	_ZL24__sti____cudaRegisterAllv
	.section	.text._ZNSt6chrono8durationIdSt5ratioILl1ELl1EEEC2IlS1_ILl1ELl1000000000EEvEERKNS0_IT_T0_EE,"axG",@progbits,_ZNSt6chrono8durationIdSt5ratioILl1ELl1EEEC5IlS1_ILl1ELl1000000000EEvEERKNS0_IT_T0_EE,comdat
	.align 2
	.weak	_ZNSt6chrono8durationIdSt5ratioILl1ELl1EEEC2IlS1_ILl1ELl1000000000EEvEERKNS0_IT_T0_EE
	.type	_ZNSt6chrono8durationIdSt5ratioILl1ELl1EEEC2IlS1_ILl1ELl1000000000EEvEERKNS0_IT_T0_EE, @function
_ZNSt6chrono8durationIdSt5ratioILl1ELl1EEEC2IlS1_ILl1ELl1000000000EEvEERKNS0_IT_T0_EE:
.LFB4100:
	.cfi_startproc
	endbr64
	cvtsi2sdq	(%rsi), %xmm0
	divsd	.LC2(%rip), %xmm0
	movsd	%xmm0, (%rdi)
	ret
	.cfi_endproc
.LFE4100:
	.size	_ZNSt6chrono8durationIdSt5ratioILl1ELl1EEEC2IlS1_ILl1ELl1000000000EEvEERKNS0_IT_T0_EE, .-_ZNSt6chrono8durationIdSt5ratioILl1ELl1EEEC2IlS1_ILl1ELl1000000000EEvEERKNS0_IT_T0_EE
	.weak	_ZNSt6chrono8durationIdSt5ratioILl1ELl1EEEC1IlS1_ILl1ELl1000000000EEvEERKNS0_IT_T0_EE
	.set	_ZNSt6chrono8durationIdSt5ratioILl1ELl1EEEC1IlS1_ILl1ELl1000000000EEvEERKNS0_IT_T0_EE,_ZNSt6chrono8durationIdSt5ratioILl1ELl1EEEC2IlS1_ILl1ELl1000000000EEvEERKNS0_IT_T0_EE
	.section	.rodata.str1.1
.LC3:
	.string	"/home/ubuntu/Datasets/stackv2/train-structured/XapaJIaMnu/gpuPeerAccessTester/master/memory_tester.cu"
	.text
	.globl	_Z22normalMemCpyOther2ThisPPfiii
	.type	_Z22normalMemCpyOther2ThisPPfiii, @function
_Z22normalMemCpyOther2ThisPPfiii:
.LFB3735:
	.cfi_startproc
	endbr64
	pushq	%r15
	.cfi_def_cfa_offset 16
	.cfi_offset 15, -16
	xorl	%r15d, %r15d
	pushq	%r14
	.cfi_def_cfa_offset 24
	.cfi_offset 14, -24
	pushq	%r13
	.cfi_def_cfa_offset 32
	.cfi_offset 13, -32
	pushq	%r12
	.cfi_def_cfa_offset 40
	.cfi_offset 12, -40
	movq	%rdi, %r12
	pushq	%rbp
	.cfi_def_cfa_offset 48
	.cfi_offset 6, -48
	movl	%ecx, %ebp
	pushq	%rbx
	.cfi_def_cfa_offset 56
	.cfi_offset 3, -56
	movl	%esi, %ebx
	subq	$56, %rsp
	.cfi_def_cfa_offset 112
	movl	%edx, 4(%rsp)
	movq	%fs:40, %rax
	movq	%rax, 40(%rsp)
	xorl	%eax, %eax
	call	_ZNSt6chrono3_V212system_clock3nowEv@PLT
	movl	$4, %ecx
	movq	%rax, %r13
	movl	4(%rsp), %eax
	cltd
	idivl	%ecx
	movl	%eax, 8(%rsp)
	movslq	%ebp, %rax
	leaq	(%r12,%rax,8), %r14
.L14:
	movl	%r15d, %r9d
	cmpl	%r15d, %ebx
	jle	.L19
	cmpl	%r15d, %ebp
	je	.L15
	movl	8(%rsp), %eax
	movq	(%r12,%r15,8), %rsi
	movl	$4, %ecx
	movl	%r9d, 12(%rsp)
	cltd
	idivl	%ebx
	imull	%r9d, %eax
	movslq	%eax, %rdi
	movl	4(%rsp), %eax
	salq	$2, %rdi
	cltd
	addq	%rdi, %rsi
	addq	(%r14), %rdi
	idivl	%ebx
	movslq	%eax, %rdx
	call	cudaMemcpy@PLT
	movl	$24, %edx
	leaq	.LC3(%rip), %rsi
	movl	%eax, %edi
	call	_Z9gpuAssert9cudaErrorPKcib.isra.0
	movl	12(%rsp), %r9d
.L15:
	movl	%r9d, %edi
	incq	%r15
	call	cudaSetDevice@PLT
	call	cudaDeviceSynchronize@PLT
	movl	%ebp, %edi
	call	cudaSetDevice@PLT
	call	cudaDeviceSynchronize@PLT
	jmp	.L14
.L19:
	call	_ZNSt6chrono3_V212system_clock3nowEv@PLT
	leaq	32(%rsp), %rsi
	leaq	24(%rsp), %rdi
	subq	%r13, %rax
	movq	%rax, 32(%rsp)
	call	_ZNSt6chrono8durationIdSt5ratioILl1ELl1EEEC1IlS1_ILl1ELl1000000000EEvEERKNS0_IT_T0_EE
	movsd	24(%rsp), %xmm0
	movq	40(%rsp), %rax
	subq	%fs:40, %rax
	je	.L17
	call	__stack_chk_fail@PLT
.L17:
	addq	$56, %rsp
	.cfi_def_cfa_offset 56
	popq	%rbx
	.cfi_def_cfa_offset 48
	popq	%rbp
	.cfi_def_cfa_offset 40
	popq	%r12
	.cfi_def_cfa_offset 32
	popq	%r13
	.cfi_def_cfa_offset 24
	popq	%r14
	.cfi_def_cfa_offset 16
	popq	%r15
	.cfi_def_cfa_offset 8
	ret
	.cfi_endproc
.LFE3735:
	.size	_Z22normalMemCpyOther2ThisPPfiii, .-_Z22normalMemCpyOther2ThisPPfiii
	.globl	_Z22normalMemCpyThis2OtherPPfiii
	.type	_Z22normalMemCpyThis2OtherPPfiii, @function
_Z22normalMemCpyThis2OtherPPfiii:
.LFB3740:
	.cfi_startproc
	endbr64
	pushq	%r15
	.cfi_def_cfa_offset 16
	.cfi_offset 15, -16
	xorl	%r15d, %r15d
	pushq	%r14
	.cfi_def_cfa_offset 24
	.cfi_offset 14, -24
	pushq	%r13
	.cfi_def_cfa_offset 32
	.cfi_offset 13, -32
	pushq	%r12
	.cfi_def_cfa_offset 40
	.cfi_offset 12, -40
	movq	%rdi, %r12
	pushq	%rbp
	.cfi_def_cfa_offset 48
	.cfi_offset 6, -48
	movl	%ecx, %ebp
	pushq	%rbx
	.cfi_def_cfa_offset 56
	.cfi_offset 3, -56
	movl	%esi, %ebx
	subq	$56, %rsp
	.cfi_def_cfa_offset 112
	movl	%edx, 4(%rsp)
	movq	%fs:40, %rax
	movq	%rax, 40(%rsp)
	xorl	%eax, %eax
	call	_ZNSt6chrono3_V212system_clock3nowEv@PLT
	movl	$4, %ecx
	movq	%rax, %r13
	movslq	%ebp, %rax
	leaq	(%r12,%rax,8), %r14
	movl	4(%rsp), %eax
	cltd
	idivl	%ecx
	movl	%eax, 8(%rsp)
.L21:
	movl	%r15d, %r9d
	cmpl	%r15d, %ebx
	jle	.L26
	cmpl	%r15d, %ebp
	je	.L22
	movl	8(%rsp), %eax
	movq	(%r14), %rsi
	movl	$4, %ecx
	movl	%r9d, 12(%rsp)
	cltd
	idivl	%ebx
	imull	%r9d, %eax
	movslq	%eax, %rdi
	movl	4(%rsp), %eax
	salq	$2, %rdi
	cltd
	addq	%rdi, %rsi
	addq	(%r12,%r15,8), %rdi
	idivl	%ebx
	movslq	%eax, %rdx
	call	cudaMemcpy@PLT
	movl	$45, %edx
	leaq	.LC3(%rip), %rsi
	movl	%eax, %edi
	call	_Z9gpuAssert9cudaErrorPKcib.isra.0
	movl	12(%rsp), %r9d
.L22:
	movl	%r9d, %edi
	incq	%r15
	call	cudaSetDevice@PLT
	call	cudaDeviceSynchronize@PLT
	movl	%ebp, %edi
	call	cudaSetDevice@PLT
	call	cudaDeviceSynchronize@PLT
	jmp	.L21
.L26:
	call	_ZNSt6chrono3_V212system_clock3nowEv@PLT
	leaq	32(%rsp), %rsi
	leaq	24(%rsp), %rdi
	subq	%r13, %rax
	movq	%rax, 32(%rsp)
	call	_ZNSt6chrono8durationIdSt5ratioILl1ELl1EEEC1IlS1_ILl1ELl1000000000EEvEERKNS0_IT_T0_EE
	movsd	24(%rsp), %xmm0
	movq	40(%rsp), %rax
	subq	%fs:40, %rax
	je	.L24
	call	__stack_chk_fail@PLT
.L24:
	addq	$56, %rsp
	.cfi_def_cfa_offset 56
	popq	%rbx
	.cfi_def_cfa_offset 48
	popq	%rbp
	.cfi_def_cfa_offset 40
	popq	%r12
	.cfi_def_cfa_offset 32
	popq	%r13
	.cfi_def_cfa_offset 24
	popq	%r14
	.cfi_def_cfa_offset 16
	popq	%r15
	.cfi_def_cfa_offset 8
	ret
	.cfi_endproc
.LFE3740:
	.size	_Z22normalMemCpyThis2OtherPPfiii, .-_Z22normalMemCpyThis2OtherPPfiii
	.globl	_Z27normalMemCpyOther2ThisAsyncPPfiii
	.type	_Z27normalMemCpyOther2ThisAsyncPPfiii, @function
_Z27normalMemCpyOther2ThisAsyncPPfiii:
.LFB3741:
	.cfi_startproc
	endbr64
	pushq	%rbp
	.cfi_def_cfa_offset 16
	.cfi_offset 6, -16
	movq	%rsp, %rbp
	.cfi_def_cfa_register 6
	pushq	%r15
	pushq	%r14
	pushq	%r13
	.cfi_offset 15, -24
	.cfi_offset 14, -32
	.cfi_offset 13, -40
	movq	%rdi, %r13
	pushq	%r12
	pushq	%rbx
	.cfi_offset 12, -48
	.cfi_offset 3, -56
	movl	%esi, %ebx
	subq	$72, %rsp
	movl	%edx, -88(%rbp)
	movl	%ecx, -84(%rbp)
	movq	%rsp, %rcx
	movq	%fs:40, %rax
	movq	%rax, -56(%rbp)
	xorl	%eax, %eax
	movslq	%esi, %rax
	leaq	15(,%rax,8), %rdx
	movq	%rdx, %rax
	andq	$-4096, %rdx
	andq	$-16, %rax
	subq	%rdx, %rcx
.L28:
	cmpq	%rcx, %rsp
	je	.L29
	subq	$4096, %rsp
	orq	$0, 4088(%rsp)
	jmp	.L28
.L29:
	andl	$4095, %eax
	subq	%rax, %rsp
	testq	%rax, %rax
	je	.L30
	orq	$0, -8(%rsp,%rax)
.L30:
	movq	%rsp, %r14
	xorl	%r12d, %r12d
	movq	%r14, %r15
.L31:
	cmpl	%ebx, %r12d
	jge	.L43
	movq	%r15, %rdi
	incl	%r12d
	addq	$8, %r15
	call	cudaStreamCreate@PLT
	movl	$65, %edx
	leaq	.LC3(%rip), %rsi
	movl	%eax, %edi
	call	_Z9gpuAssert9cudaErrorPKcib.isra.0
	jmp	.L31
.L43:
	call	_ZNSt6chrono3_V212system_clock3nowEv@PLT
	movl	$4, %ecx
	xorl	%r12d, %r12d
	movq	%rax, -96(%rbp)
	movl	-88(%rbp), %eax
	cltd
	idivl	%ecx
	movl	%eax, -100(%rbp)
	movslq	-84(%rbp), %rax
	leaq	0(%r13,%rax,8), %r15
.L33:
	movl	%r12d, %r10d
	cmpl	%r12d, %ebx
	jle	.L44
	cmpl	%r12d, -84(%rbp)
	je	.L34
	movl	-100(%rbp), %eax
	movq	(%r14,%r12,8), %r8
	movl	$4, %ecx
	movl	%r10d, -104(%rbp)
	movq	0(%r13,%r12,8), %rsi
	cltd
	idivl	%ebx
	imull	%r10d, %eax
	movslq	%eax, %rdi
	movl	-88(%rbp), %eax
	salq	$2, %rdi
	cltd
	addq	%rdi, %rsi
	addq	(%r15), %rdi
	idivl	%ebx
	movslq	%eax, %rdx
	call	cudaMemcpyAsync@PLT
	movl	$71, %edx
	leaq	.LC3(%rip), %rsi
	movl	%eax, %edi
	call	_Z9gpuAssert9cudaErrorPKcib.isra.0
	movl	-104(%rbp), %r10d
.L34:
	movl	%r10d, %edi
	incq	%r12
	call	cudaSetDevice@PLT
	call	cudaDeviceSynchronize@PLT
	movl	-84(%rbp), %edi
	call	cudaSetDevice@PLT
	call	cudaDeviceSynchronize@PLT
	jmp	.L33
.L44:
	xorl	%r12d, %r12d
	leaq	.LC3(%rip), %r13
.L36:
	cmpl	%r12d, %ebx
	jle	.L45
	movq	(%r14,%r12,8), %rdi
	incq	%r12
	call	cudaStreamSynchronize@PLT
	movl	$83, %edx
	movq	%r13, %rsi
	movl	%eax, %edi
	call	_Z9gpuAssert9cudaErrorPKcib.isra.0
	jmp	.L36
.L45:
	call	_ZNSt6chrono3_V212system_clock3nowEv@PLT
	movq	-96(%rbp), %rbx
	leaq	-64(%rbp), %rsi
	leaq	-72(%rbp), %rdi
	subq	%rbx, %rax
	movq	%rax, -64(%rbp)
	call	_ZNSt6chrono8durationIdSt5ratioILl1ELl1EEEC1IlS1_ILl1ELl1000000000EEvEERKNS0_IT_T0_EE
	movsd	-72(%rbp), %xmm0
	movq	-56(%rbp), %rax
	subq	%fs:40, %rax
	je	.L38
	call	__stack_chk_fail@PLT
.L38:
	leaq	-40(%rbp), %rsp
	popq	%rbx
	popq	%r12
	popq	%r13
	popq	%r14
	popq	%r15
	popq	%rbp
	.cfi_def_cfa 7, 8
	ret
	.cfi_endproc
.LFE3741:
	.size	_Z27normalMemCpyOther2ThisAsyncPPfiii, .-_Z27normalMemCpyOther2ThisAsyncPPfiii
	.globl	_Z27normalMemCpyThis2OtherAsyncPPfiii
	.type	_Z27normalMemCpyThis2OtherAsyncPPfiii, @function
_Z27normalMemCpyThis2OtherAsyncPPfiii:
.LFB3742:
	.cfi_startproc
	endbr64
	pushq	%rbp
	.cfi_def_cfa_offset 16
	.cfi_offset 6, -16
	movq	%rsp, %rbp
	.cfi_def_cfa_register 6
	pushq	%r15
	pushq	%r14
	pushq	%r13
	.cfi_offset 15, -24
	.cfi_offset 14, -32
	.cfi_offset 13, -40
	movq	%rdi, %r13
	pushq	%r12
	pushq	%rbx
	.cfi_offset 12, -48
	.cfi_offset 3, -56
	movl	%esi, %ebx
	subq	$72, %rsp
	movl	%edx, -88(%rbp)
	movl	%ecx, -84(%rbp)
	movq	%rsp, %rcx
	movq	%fs:40, %rax
	movq	%rax, -56(%rbp)
	xorl	%eax, %eax
	movslq	%esi, %rax
	leaq	15(,%rax,8), %rdx
	movq	%rdx, %rax
	andq	$-4096, %rdx
	andq	$-16, %rax
	subq	%rdx, %rcx
.L47:
	cmpq	%rcx, %rsp
	je	.L48
	subq	$4096, %rsp
	orq	$0, 4088(%rsp)
	jmp	.L47
.L48:
	andl	$4095, %eax
	subq	%rax, %rsp
	testq	%rax, %rax
	je	.L49
	orq	$0, -8(%rsp,%rax)
.L49:
	movq	%rsp, %r14
	xorl	%r12d, %r12d
	movq	%r14, %r15
.L50:
	cmpl	%ebx, %r12d
	jge	.L62
	movq	%r15, %rdi
	incl	%r12d
	addq	$8, %r15
	call	cudaStreamCreate@PLT
	movl	$95, %edx
	leaq	.LC3(%rip), %rsi
	movl	%eax, %edi
	call	_Z9gpuAssert9cudaErrorPKcib.isra.0
	jmp	.L50
.L62:
	call	_ZNSt6chrono3_V212system_clock3nowEv@PLT
	movl	$4, %ecx
	xorl	%r12d, %r12d
	movq	%rax, -96(%rbp)
	movslq	-84(%rbp), %rax
	leaq	0(%r13,%rax,8), %r15
	movl	-88(%rbp), %eax
	cltd
	idivl	%ecx
	movl	%eax, -100(%rbp)
.L52:
	movl	%r12d, %r10d
	cmpl	%r12d, %ebx
	jle	.L63
	cmpl	%r12d, -84(%rbp)
	je	.L53
	movl	-100(%rbp), %eax
	movq	(%r15), %rsi
	movl	$4, %ecx
	movl	%r10d, -104(%rbp)
	movq	(%r14,%r12,8), %r8
	cltd
	idivl	%ebx
	imull	%r10d, %eax
	movslq	%eax, %rdi
	movl	-88(%rbp), %eax
	salq	$2, %rdi
	cltd
	addq	%rdi, %rsi
	addq	0(%r13,%r12,8), %rdi
	idivl	%ebx
	movslq	%eax, %rdx
	call	cudaMemcpyAsync@PLT
	movl	$101, %edx
	leaq	.LC3(%rip), %rsi
	movl	%eax, %edi
	call	_Z9gpuAssert9cudaErrorPKcib.isra.0
	movl	-104(%rbp), %r10d
.L53:
	movl	%r10d, %edi
	incq	%r12
	call	cudaSetDevice@PLT
	call	cudaDeviceSynchronize@PLT
	movl	-84(%rbp), %edi
	call	cudaSetDevice@PLT
	call	cudaDeviceSynchronize@PLT
	jmp	.L52
.L63:
	xorl	%r12d, %r12d
	leaq	.LC3(%rip), %r13
.L55:
	cmpl	%r12d, %ebx
	jle	.L64
	movq	(%r14,%r12,8), %rdi
	incq	%r12
	call	cudaStreamSynchronize@PLT
	movl	$114, %edx
	movq	%r13, %rsi
	movl	%eax, %edi
	call	_Z9gpuAssert9cudaErrorPKcib.isra.0
	jmp	.L55
.L64:
	call	_ZNSt6chrono3_V212system_clock3nowEv@PLT
	movq	-96(%rbp), %rbx
	leaq	-64(%rbp), %rsi
	leaq	-72(%rbp), %rdi
	subq	%rbx, %rax
	movq	%rax, -64(%rbp)
	call	_ZNSt6chrono8durationIdSt5ratioILl1ELl1EEEC1IlS1_ILl1ELl1000000000EEvEERKNS0_IT_T0_EE
	movsd	-72(%rbp), %xmm0
	movq	-56(%rbp), %rax
	subq	%fs:40, %rax
	je	.L57
	call	__stack_chk_fail@PLT
.L57:
	leaq	-40(%rbp), %rsp
	popq	%rbx
	popq	%r12
	popq	%r13
	popq	%r14
	popq	%r15
	popq	%rbp
	.cfi_def_cfa 7, 8
	ret
	.cfi_endproc
.LFE3742:
	.size	_Z27normalMemCpyThis2OtherAsyncPPfiii, .-_Z27normalMemCpyThis2OtherAsyncPPfiii
	.section	.rodata.str1.1
.LC4:
	.string	"Usage: "
.LC5:
	.string	" num gpus total_model_memory"
.LC6:
	.string	"Other to this took: "
.LC7:
	.string	" seconds."
.LC8:
	.string	"This to other took: "
.LC9:
	.string	"Other to this Async took: "
.LC10:
	.string	"This to other Async took: "
.LC11:
	.string	"Peer access unavailable between devices: "
.LC12:
	.string	" and "
.LC13:
	.string	"Peer other to this took: "
.LC14:
	.string	"Peer this to other took: "
.LC15:
	.string	"Peer other to this Async took: "
.LC16:
	.string	"Peer this to other Async took: "
	.section	.text.startup
	.globl	main
	.type	main, @function
main:
.LFB3743:
	.cfi_startproc
	endbr64
	pushq	%r15
	.cfi_def_cfa_offset 16
	.cfi_offset 15, -16
	pushq	%r14
	.cfi_def_cfa_offset 24
	.cfi_offset 14, -24
	pushq	%r13
	.cfi_def_cfa_offset 32
	.cfi_offset 13, -32
	pushq	%r12
	.cfi_def_cfa_offset 40
	.cfi_offset 12, -40
	pushq	%rbp
	.cfi_def_cfa_offset 48
	.cfi_offset 6, -48
	movq	%rsi, %rbp
	pushq	%rbx
	.cfi_def_cfa_offset 56
	.cfi_offset 3, -56
	subq	$40, %rsp
	.cfi_def_cfa_offset 96
	movq	%fs:40, %rax
	movq	%rax, 24(%rsp)
	xorl	%eax, %eax
	cmpl	$3, %edi
	je	.L66
	leaq	.LC4(%rip), %rsi
	leaq	_ZSt4cerr(%rip), %rdi
	call	_ZStlsISt11char_traitsIcEERSt13basic_ostreamIcT_ES5_PKc@PLT
	movq	0(%rbp), %rsi
	movq	%rax, %rdi
	call	_ZStlsISt11char_traitsIcEERSt13basic_ostreamIcT_ES5_PKc@PLT
	leaq	.LC5(%rip), %rsi
	movq	%rax, %rdi
	call	_ZStlsISt11char_traitsIcEERSt13basic_ostreamIcT_ES5_PKc@PLT
	movq	_ZSt4endlIcSt11char_traitsIcEERSt13basic_ostreamIT_T0_ES6_@GOTPCREL(%rip), %rsi
	movq	%rax, %rdi
	call	_ZNSolsEPFRSoS_E.isra.0
	movl	$1, %edi
	call	exit@PLT
.L66:
	movq	8(%rsi), %rdi
	call	atoi@PLT
	movq	16(%rbp), %rdi
	movl	%eax, %ebx
	call	atoi@PLT
	movslq	%ebx, %rdi
	sall	$20, %eax
	movl	%eax, %ebp
	movabsq	$1152921504606846975, %rax
	cmpq	%rdi, %rax
	jb	.L67
	salq	$3, %rdi
	xorl	%r13d, %r13d
	movslq	%ebp, %r15
	call	_Znam@PLT
	movq	%rax, %r12
	movq	%rax, %r14
	jmp	.L68
.L67:
	movq	24(%rsp), %rax
	subq	%fs:40, %rax
	jne	.L84
	call	__cxa_throw_bad_array_new_length@PLT
.L68:
	cmpl	%ebx, %r13d
	jge	.L85
	movl	%r13d, %edi
	incl	%r13d
	call	cudaSetDevice@PLT
	movq	%r14, %rdi
	movq	%r15, %rsi
	addq	$8, %r14
	call	cudaMalloc@PLT
	movl	$138, %edx
	leaq	.LC3(%rip), %rsi
	movl	%eax, %edi
	call	_Z9gpuAssert9cudaErrorPKcib.isra.0
	jmp	.L68
.L85:
	xorl	%ecx, %ecx
	movl	%ebp, %edx
	movl	%ebx, %esi
	movq	%r12, %rdi
	call	_Z22normalMemCpyOther2ThisPPfiii
	xorl	%ecx, %ecx
	movl	%ebp, %edx
	movl	%ebx, %esi
	movq	%r12, %rdi
	movsd	%xmm0, (%rsp)
	movl	$1, %r13d
	call	_Z22normalMemCpyThis2OtherPPfiii
	xorl	%ecx, %ecx
	movl	%ebp, %edx
	movl	%ebx, %esi
	movq	%r12, %rdi
	movsd	%xmm0, 8(%rsp)
	call	_Z27normalMemCpyOther2ThisAsyncPPfiii
	xorl	%ecx, %ecx
	movl	%ebp, %edx
	movl	%ebx, %esi
	movq	%r12, %rdi
	call	_Z27normalMemCpyThis2OtherAsyncPPfiii
.L71:
	cmpl	%ebx, %r13d
	jge	.L86
	movl	%r13d, %ecx
	movl	%ebp, %edx
	movl	%ebx, %esi
	movq	%r12, %rdi
	call	_Z22normalMemCpyOther2ThisPPfiii
	addsd	(%rsp), %xmm0
	movl	%r13d, %ecx
	movl	%ebp, %edx
	movl	%ebx, %esi
	movq	%r12, %rdi
	movsd	%xmm0, (%rsp)
	call	_Z22normalMemCpyThis2OtherPPfiii
	addsd	8(%rsp), %xmm0
	movl	%r13d, %ecx
	movl	%ebp, %edx
	movl	%ebx, %esi
	movq	%r12, %rdi
	movsd	%xmm0, 8(%rsp)
	call	_Z27normalMemCpyOther2ThisAsyncPPfiii
	movl	%r13d, %ecx
	movl	%ebp, %edx
	movl	%ebx, %esi
	movq	%r12, %rdi
	incl	%r13d
	call	_Z27normalMemCpyThis2OtherAsyncPPfiii
	jmp	.L71
.L86:
	leaq	_ZSt5fixedRSt8ios_base(%rip), %rsi
	leaq	_ZSt4cout(%rip), %rdi
	xorl	%r13d, %r13d
	call	_ZNSolsEPFRSt8ios_baseS0_E@PLT
	leaq	.LC6(%rip), %rsi
	leaq	.LC3(%rip), %r15
	movq	%rax, %rdi
	call	_ZStlsISt11char_traitsIcEERSt13basic_ostreamIcT_ES5_PKc@PLT
	movsd	(%rsp), %xmm0
	movq	%rax, %rdi
	call	_ZNSo9_M_insertIdEERSoT_@PLT
	leaq	.LC7(%rip), %rsi
	movq	%rax, %rdi
	call	_ZStlsISt11char_traitsIcEERSt13basic_ostreamIcT_ES5_PKc@PLT
	movq	_ZSt4endlIcSt11char_traitsIcEERSt13basic_ostreamIT_T0_ES6_@GOTPCREL(%rip), %rsi
	movq	%rax, %rdi
	call	_ZNSolsEPFRSoS_E.isra.0
	leaq	_ZSt5fixedRSt8ios_base(%rip), %rsi
	leaq	_ZSt4cout(%rip), %rdi
	call	_ZNSolsEPFRSt8ios_baseS0_E@PLT
	leaq	.LC8(%rip), %rsi
	movq	%rax, %rdi
	call	_ZStlsISt11char_traitsIcEERSt13basic_ostreamIcT_ES5_PKc@PLT
	movsd	8(%rsp), %xmm0
	movq	%rax, %rdi
	call	_ZNSo9_M_insertIdEERSoT_@PLT
	leaq	.LC7(%rip), %rsi
	movq	%rax, %rdi
	call	_ZStlsISt11char_traitsIcEERSt13basic_ostreamIcT_ES5_PKc@PLT
	movq	_ZSt4endlIcSt11char_traitsIcEERSt13basic_ostreamIT_T0_ES6_@GOTPCREL(%rip), %rsi
	movq	%rax, %rdi
	call	_ZNSolsEPFRSoS_E.isra.0
	leaq	_ZSt5fixedRSt8ios_base(%rip), %rsi
	leaq	_ZSt4cout(%rip), %rdi
	call	_ZNSolsEPFRSt8ios_baseS0_E@PLT
	leaq	.LC9(%rip), %rsi
	movq	%rax, %rdi
	call	_ZStlsISt11char_traitsIcEERSt13basic_ostreamIcT_ES5_PKc@PLT
	movsd	(%rsp), %xmm0
	movq	%rax, %rdi
	call	_ZNSo9_M_insertIdEERSoT_@PLT
	leaq	.LC7(%rip), %rsi
	movq	%rax, %rdi
	call	_ZStlsISt11char_traitsIcEERSt13basic_ostreamIcT_ES5_PKc@PLT
	movq	_ZSt4endlIcSt11char_traitsIcEERSt13basic_ostreamIT_T0_ES6_@GOTPCREL(%rip), %rsi
	movq	%rax, %rdi
	call	_ZNSolsEPFRSoS_E.isra.0
	leaq	_ZSt5fixedRSt8ios_base(%rip), %rsi
	leaq	_ZSt4cout(%rip), %rdi
	call	_ZNSolsEPFRSt8ios_baseS0_E@PLT
	leaq	.LC10(%rip), %rsi
	movq	%rax, %rdi
	call	_ZStlsISt11char_traitsIcEERSt13basic_ostreamIcT_ES5_PKc@PLT
	movsd	8(%rsp), %xmm0
	movq	%rax, %rdi
	call	_ZNSo9_M_insertIdEERSoT_@PLT
	leaq	.LC7(%rip), %rsi
	movq	%rax, %rdi
	call	_ZStlsISt11char_traitsIcEERSt13basic_ostreamIcT_ES5_PKc@PLT
	movq	_ZSt4endlIcSt11char_traitsIcEERSt13basic_ostreamIT_T0_ES6_@GOTPCREL(%rip), %rsi
	movq	%rax, %rdi
	call	_ZNSolsEPFRSoS_E.isra.0
.L73:
	cmpl	%ebx, %r13d
	jge	.L87
	xorl	%r14d, %r14d
.L77:
	cmpl	%r14d, %r13d
	je	.L74
	leaq	20(%rsp), %rdi
	movl	%r14d, %edx
	movl	%r13d, %esi
	call	cudaDeviceCanAccessPeer@PLT
	movl	$163, %edx
	movq	%r15, %rsi
	movl	%eax, %edi
	call	_Z9gpuAssert9cudaErrorPKcib.isra.0
	cmpl	$0, 20(%rsp)
	je	.L75
	movl	%r13d, %edi
	call	cudaSetDevice@PLT
	xorl	%esi, %esi
	movl	%r14d, %edi
	call	cudaDeviceEnablePeerAccess@PLT
	jmp	.L74
.L75:
	leaq	_ZSt5fixedRSt8ios_base(%rip), %rsi
	leaq	_ZSt4cout(%rip), %rdi
	call	_ZNSolsEPFRSt8ios_baseS0_E@PLT
	leaq	.LC11(%rip), %rsi
	movq	%rax, %rdi
	call	_ZStlsISt11char_traitsIcEERSt13basic_ostreamIcT_ES5_PKc@PLT
	movl	%r13d, %esi
	movq	%rax, %rdi
	call	_ZNSolsEi@PLT
	leaq	.LC12(%rip), %rsi
	movq	%rax, %rdi
	call	_ZStlsISt11char_traitsIcEERSt13basic_ostreamIcT_ES5_PKc@PLT
	movl	%r14d, %esi
	movq	%rax, %rdi
	call	_ZNSolsEi@PLT
	movq	_ZSt4endlIcSt11char_traitsIcEERSt13basic_ostreamIT_T0_ES6_@GOTPCREL(%rip), %rsi
	movq	%rax, %rdi
	call	_ZNSolsEPFRSoS_E.isra.0
.L74:
	incl	%r14d
	cmpl	%r14d, %ebx
	jne	.L77
	incl	%r13d
	jmp	.L73
.L87:
	xorl	%ecx, %ecx
	movl	%ebp, %edx
	movl	%ebx, %esi
	movq	%r12, %rdi
	call	_Z22normalMemCpyOther2ThisPPfiii
	xorl	%ecx, %ecx
	movl	%ebp, %edx
	movl	%ebx, %esi
	movq	%r12, %rdi
	movsd	%xmm0, (%rsp)
	movl	$1, %r13d
	call	_Z22normalMemCpyThis2OtherPPfiii
	xorl	%ecx, %ecx
	movl	%ebp, %edx
	movl	%ebx, %esi
	movq	%r12, %rdi
	movsd	%xmm0, 8(%rsp)
	call	_Z27normalMemCpyOther2ThisAsyncPPfiii
	xorl	%ecx, %ecx
	movl	%ebp, %edx
	movl	%ebx, %esi
	movq	%r12, %rdi
	call	_Z27normalMemCpyThis2OtherAsyncPPfiii
.L78:
	cmpl	%ebx, %r13d
	jge	.L88
	movl	%r13d, %ecx
	movl	%ebp, %edx
	movl	%ebx, %esi
	movq	%r12, %rdi
	call	_Z22normalMemCpyOther2ThisPPfiii
	addsd	(%rsp), %xmm0
	movl	%r13d, %ecx
	movl	%ebp, %edx
	movl	%ebx, %esi
	movq	%r12, %rdi
	movsd	%xmm0, (%rsp)
	call	_Z22normalMemCpyThis2OtherPPfiii
	addsd	8(%rsp), %xmm0
	movl	%r13d, %ecx
	movl	%ebp, %edx
	movl	%ebx, %esi
	movq	%r12, %rdi
	movsd	%xmm0, 8(%rsp)
	call	_Z27normalMemCpyOther2ThisAsyncPPfiii
	movl	%r13d, %ecx
	movl	%ebp, %edx
	movl	%ebx, %esi
	movq	%r12, %rdi
	incl	%r13d
	call	_Z27normalMemCpyThis2OtherAsyncPPfiii
	jmp	.L78
.L88:
	leaq	_ZSt5fixedRSt8ios_base(%rip), %rsi
	leaq	_ZSt4cout(%rip), %rdi
	call	_ZNSolsEPFRSt8ios_baseS0_E@PLT
	leaq	.LC13(%rip), %rsi
	movq	%rax, %rdi
	call	_ZStlsISt11char_traitsIcEERSt13basic_ostreamIcT_ES5_PKc@PLT
	movsd	(%rsp), %xmm0
	movq	%rax, %rdi
	call	_ZNSo9_M_insertIdEERSoT_@PLT
	leaq	.LC7(%rip), %rsi
	movq	%rax, %rdi
	call	_ZStlsISt11char_traitsIcEERSt13basic_ostreamIcT_ES5_PKc@PLT
	movq	_ZSt4endlIcSt11char_traitsIcEERSt13basic_ostreamIT_T0_ES6_@GOTPCREL(%rip), %rsi
	movq	%rax, %rdi
	call	_ZNSolsEPFRSoS_E.isra.0
	leaq	_ZSt5fixedRSt8ios_base(%rip), %rsi
	leaq	_ZSt4cout(%rip), %rdi
	call	_ZNSolsEPFRSt8ios_baseS0_E@PLT
	leaq	.LC14(%rip), %rsi
	movq	%rax, %rdi
	call	_ZStlsISt11char_traitsIcEERSt13basic_ostreamIcT_ES5_PKc@PLT
	movsd	8(%rsp), %xmm0
	movq	%rax, %rdi
	call	_ZNSo9_M_insertIdEERSoT_@PLT
	leaq	.LC7(%rip), %rsi
	movq	%rax, %rdi
	call	_ZStlsISt11char_traitsIcEERSt13basic_ostreamIcT_ES5_PKc@PLT
	movq	_ZSt4endlIcSt11char_traitsIcEERSt13basic_ostreamIT_T0_ES6_@GOTPCREL(%rip), %rsi
	movq	%rax, %rdi
	call	_ZNSolsEPFRSoS_E.isra.0
	leaq	_ZSt5fixedRSt8ios_base(%rip), %rsi
	leaq	_ZSt4cout(%rip), %rdi
	call	_ZNSolsEPFRSt8ios_baseS0_E@PLT
	leaq	.LC15(%rip), %rsi
	movq	%rax, %rdi
	call	_ZStlsISt11char_traitsIcEERSt13basic_ostreamIcT_ES5_PKc@PLT
	movsd	(%rsp), %xmm0
	movq	%rax, %rdi
	call	_ZNSo9_M_insertIdEERSoT_@PLT
	leaq	.LC7(%rip), %rsi
	movq	%rax, %rdi
	call	_ZStlsISt11char_traitsIcEERSt13basic_ostreamIcT_ES5_PKc@PLT
	movq	_ZSt4endlIcSt11char_traitsIcEERSt13basic_ostreamIT_T0_ES6_@GOTPCREL(%rip), %rsi
	movq	%rax, %rdi
	call	_ZNSolsEPFRSoS_E.isra.0
	leaq	_ZSt5fixedRSt8ios_base(%rip), %rsi
	leaq	_ZSt4cout(%rip), %rdi
	call	_ZNSolsEPFRSt8ios_baseS0_E@PLT
	leaq	.LC16(%rip), %rsi
	movq	%rax, %rdi
	call	_ZStlsISt11char_traitsIcEERSt13basic_ostreamIcT_ES5_PKc@PLT
	movsd	8(%rsp), %xmm0
	movq	%rax, %rdi
	call	_ZNSo9_M_insertIdEERSoT_@PLT
	leaq	.LC7(%rip), %rsi
	movq	%rax, %rdi
	call	_ZStlsISt11char_traitsIcEERSt13basic_ostreamIcT_ES5_PKc@PLT
	movq	_ZSt4endlIcSt11char_traitsIcEERSt13basic_ostreamIT_T0_ES6_@GOTPCREL(%rip), %rsi
	movq	%rax, %rdi
	call	_ZNSolsEPFRSoS_E.isra.0
	movq	24(%rsp), %rax
	subq	%fs:40, %rax
	je	.L80
.L84:
	call	__stack_chk_fail@PLT
.L80:
	addq	$40, %rsp
	.cfi_def_cfa_offset 56
	xorl	%eax, %eax
	popq	%rbx
	.cfi_def_cfa_offset 48
	popq	%rbp
	.cfi_def_cfa_offset 40
	popq	%r12
	.cfi_def_cfa_offset 32
	popq	%r13
	.cfi_def_cfa_offset 24
	popq	%r14
	.cfi_def_cfa_offset 16
	popq	%r15
	.cfi_def_cfa_offset 8
	ret
	.cfi_endproc
.LFE3743:
	.size	main, .-main
	.section	.nvFatBinSegment,"aw"
	.align 8
	.type	_ZL15__fatDeviceText, @object
	.size	_ZL15__fatDeviceText, 24
_ZL15__fatDeviceText:
	.long	1180844977
	.long	1
	.quad	fatbinData
	.quad	0
	.local	_ZL20__cudaFatCubinHandle
	.comm	_ZL20__cudaFatCubinHandle,8,8
	.section	.rodata.cst8,"aM",@progbits,8
	.align 8
.LC2:
	.long	0
	.long	1104006501
	.ident	"GCC: (Ubuntu 13.3.0-6ubuntu2~24.04) 13.3.0"
	.section	.note.GNU-stack,"",@progbits
	.section	.note.gnu.property,"a"
	.align 8
	.long	1f - 0f
	.long	4f - 1f
	.long	5
0:
	.string	"GNU"
1:
	.align 8
	.long	0xc0000002
	.long	3f - 2f
2:
	.long	0x3
3:
	.align 8
4:

#include <chrono>
#include <iostream>
#include <hip/hip_runtime.h>
#define CUDA_CHECK(ans) \
  { gpuAssert((ans), __FILE__, __LINE__); }

inline void gpuAssert(hipError_t code,
                      const char *file,
                      int line,
                      bool abort = true) {
  if(code != hipSuccess) {
    std::cerr <<"GPUassert: " << hipGetErrorString(code) << " " << file << " "
                              << line << std::endl;
    exit(1);
  }
}


//Copy from other GPUs to this device
double normalMemCpyOther2This(float ** gpu_ptrs, int num_gpus, int bytes_to_transfer_each, int curr_gpu) {
    std::chrono::high_resolution_clock::time_point start = std::chrono::high_resolution_clock::now();
    for (int i = 0; i < num_gpus; i++) {
        if (i != curr_gpu) {
            CUDA_CHECK(hipMemcpy(gpu_ptrs[curr_gpu] + (bytes_to_transfer_each/4/num_gpus)*i,
             gpu_ptrs[i] + (bytes_to_transfer_each/4/num_gpus)*i,
             bytes_to_transfer_each/num_gpus,
                hipMemcpyDefault));
        }
        hipSetDevice(i);
        hipDeviceSynchronize();
        hipSetDevice(curr_gpu);
        hipDeviceSynchronize();

    }
    std::chrono::high_resolution_clock::time_point end = std::chrono::high_resolution_clock::now();
    std::chrono::duration<double> secs = end-start;
    return secs.count();
}

//From this device to other GPUS, async
double normalMemCpyThis2Other(float ** gpu_ptrs, int num_gpus, int bytes_to_transfer_each, int curr_gpu) {
    std::chrono::high_resolution_clock::time_point start = std::chrono::high_resolution_clock::now();
    for (int i = 0; i < num_gpus; i++) {
        if (i != curr_gpu) {
            CUDA_CHECK(hipMemcpy(gpu_ptrs[i] + (bytes_to_transfer_each/4/num_gpus)*i,
             gpu_ptrs[curr_gpu] + (bytes_to_transfer_each/4/num_gpus)*i,
             bytes_to_transfer_each/num_gpus,
                hipMemcpyDefault));
        }
        hipSetDevice(i);
        hipDeviceSynchronize();
        hipSetDevice(curr_gpu);
        hipDeviceSynchronize();

    }
    std::chrono::high_resolution_clock::time_point end = std::chrono::high_resolution_clock::now();
    std::chrono::duration<double> secs = end-start;
    return secs.count();
}

//Copy from other GPUs to this device, async
double normalMemCpyOther2ThisAsync(float ** gpu_ptrs, int num_gpus, int bytes_to_transfer_each, int curr_gpu) {
    hipStream_t streams[num_gpus];
    for (int i = 0; i < num_gpus; i++) {
        CUDA_CHECK(hipStreamCreate(&streams[i]));
    }

    std::chrono::high_resolution_clock::time_point start = std::chrono::high_resolution_clock::now();
    for (int i = 0; i < num_gpus; i++) {
        if (i != curr_gpu) {
            CUDA_CHECK(hipMemcpyAsync(gpu_ptrs[curr_gpu] + (bytes_to_transfer_each/4/num_gpus)*i,
             gpu_ptrs[i] + (bytes_to_transfer_each/4/num_gpus)*i,
             bytes_to_transfer_each/num_gpus,
                hipMemcpyDefault, streams[i]));
        }
        hipSetDevice(i);
        hipDeviceSynchronize();
        hipSetDevice(curr_gpu);
        hipDeviceSynchronize();
    }
    
    for (int i = 0; i < num_gpus; i++) {
        CUDA_CHECK(hipStreamSynchronize(streams[i]));
    }
    
    std::chrono::high_resolution_clock::time_point end = std::chrono::high_resolution_clock::now();
    std::chrono::duration<double> secs = end-start;
    return secs.count();
}

//From this device to other GPUS, async
double normalMemCpyThis2OtherAsync(float ** gpu_ptrs, int num_gpus, int bytes_to_transfer_each, int curr_gpu) {
    hipStream_t streams[num_gpus];
    for (int i = 0; i < num_gpus; i++) {
        CUDA_CHECK(hipStreamCreate(&streams[i]));
    }

    std::chrono::high_resolution_clock::time_point start = std::chrono::high_resolution_clock::now();
    for (int i = 0; i < num_gpus; i++) {
        if (i != curr_gpu) {
            CUDA_CHECK(hipMemcpyAsync(gpu_ptrs[i] + (bytes_to_transfer_each/4/num_gpus)*i,
             gpu_ptrs[curr_gpu] + (bytes_to_transfer_each/4/num_gpus)*i,
             bytes_to_transfer_each/num_gpus,
                hipMemcpyDefault, streams[i]));
        }
        hipSetDevice(i);
        hipDeviceSynchronize();
        hipSetDevice(curr_gpu);
        hipDeviceSynchronize();

    }

    for (int i = 0; i < num_gpus; i++) {
        CUDA_CHECK(hipStreamSynchronize(streams[i]));
    }

    std::chrono::high_resolution_clock::time_point end = std::chrono::high_resolution_clock::now();
    std::chrono::duration<double> secs = end-start;
    return secs.count();
}

int main(int argc, char * argv[]) {

    if (argc != 3) {
        std::cerr << "Usage: " << argv[0] << " num gpus total_model_memory" << std::endl;
        exit(1);
    }

    int num_gpus = atoi(argv[1]);

    int bytes_to_transfer_each = atoi(argv[2])*1024*1024;

    float ** gpu_ptrs = new float*[num_gpus];

    //Initiate with random memory, we don't care what it is. Also init the streams
    for (int i = 0; i < num_gpus; i++) {
        hipSetDevice(i);
        CUDA_CHECK(hipMalloc(&gpu_ptrs[i], bytes_to_transfer_each));
    }

    auto other_this = normalMemCpyOther2This(gpu_ptrs, num_gpus, bytes_to_transfer_each, 0);
    auto this_other = normalMemCpyThis2Other(gpu_ptrs, num_gpus, bytes_to_transfer_each, 0);
    auto other_thisAsync = normalMemCpyOther2ThisAsync(gpu_ptrs, num_gpus, bytes_to_transfer_each, 0);
    auto this_otherAsync = normalMemCpyThis2OtherAsync(gpu_ptrs, num_gpus, bytes_to_transfer_each, 0);
    
    for (int i = 1; i < num_gpus; i++) {
      other_this += normalMemCpyOther2This(gpu_ptrs, num_gpus, bytes_to_transfer_each, i);
      this_other += normalMemCpyThis2Other(gpu_ptrs, num_gpus, bytes_to_transfer_each, i);
      other_thisAsync += normalMemCpyOther2ThisAsync(gpu_ptrs, num_gpus, bytes_to_transfer_each, i);
      this_otherAsync += normalMemCpyThis2OtherAsync(gpu_ptrs, num_gpus, bytes_to_transfer_each, i);
    }

    std::cout << std::fixed << "Other to this took: " << other_this << " seconds." << std::endl;
    std::cout << std::fixed << "This to other took: " << this_other << " seconds." << std::endl;
    std::cout << std::fixed << "Other to this Async took: " << other_this << " seconds." << std::endl;
    std::cout << std::fixed << "This to other Async took: " << this_other << " seconds." << std::endl;

    //Attempt to enable peer access
    for (int i = 0; i<num_gpus; i++) {
        for (int j = 0; j<num_gpus; j++) {
            if (i != j) {
                int result;
                CUDA_CHECK(hipDeviceCanAccessPeer(&result, i, j));
                if (result) {
                    hipSetDevice(i);
                    hipDeviceEnablePeerAccess (j, 0);
                } else {
                    std::cout << std::fixed << "Peer access unavailable between devices: " << i << " and " << j << std::endl;
                }
            }
        }
    }

    //Redo the benchmarks, see if it is any different

    other_this = normalMemCpyOther2This(gpu_ptrs, num_gpus, bytes_to_transfer_each, 0);
    this_other = normalMemCpyThis2Other(gpu_ptrs, num_gpus, bytes_to_transfer_each, 0);
    other_thisAsync = normalMemCpyOther2ThisAsync(gpu_ptrs, num_gpus, bytes_to_transfer_each, 0);
    this_otherAsync = normalMemCpyThis2OtherAsync(gpu_ptrs, num_gpus, bytes_to_transfer_each, 0);
    
    for (int i = 1; i < num_gpus; i++) {
      other_this += normalMemCpyOther2This(gpu_ptrs, num_gpus, bytes_to_transfer_each, i);
      this_other += normalMemCpyThis2Other(gpu_ptrs, num_gpus, bytes_to_transfer_each, i);
      other_thisAsync += normalMemCpyOther2ThisAsync(gpu_ptrs, num_gpus, bytes_to_transfer_each, i);
      this_otherAsync += normalMemCpyThis2OtherAsync(gpu_ptrs, num_gpus, bytes_to_transfer_each, i);
    }

    std::cout << std::fixed << "Peer other to this took: " << other_this << " seconds." << std::endl;
    std::cout << std::fixed << "Peer this to other took: " << this_other << " seconds." << std::endl;
    std::cout << std::fixed << "Peer other to this Async took: " << other_this << " seconds." << std::endl;
    std::cout << std::fixed << "Peer this to other Async took: " << this_other << " seconds." << std::endl;

    return 0;

}

	.text
	.file	"memory_tester.hip"
                                        # Start of file scope inline assembly
	.globl	_ZSt21ios_base_library_initv

                                        # End of file scope inline assembly
	.section	.rodata.cst8,"aM",@progbits,8
	.p2align	3, 0x0                          # -- Begin function _Z22normalMemCpyOther2ThisPPfiii
.LCPI0_0:
	.quad	0x41cdcd6500000000              # double 1.0E+9
	.text
	.globl	_Z22normalMemCpyOther2ThisPPfiii
	.type	_Z22normalMemCpyOther2ThisPPfiii,@function
_Z22normalMemCpyOther2ThisPPfiii:       # @_Z22normalMemCpyOther2ThisPPfiii
	.cfi_startproc
# %bb.0:
	pushq	%rbp
	.cfi_def_cfa_offset 16
	pushq	%r15
	.cfi_def_cfa_offset 24
	pushq	%r14
	.cfi_def_cfa_offset 32
	pushq	%r13
	.cfi_def_cfa_offset 40
	pushq	%r12
	.cfi_def_cfa_offset 48
	pushq	%rbx
	.cfi_def_cfa_offset 56
	subq	$40, %rsp
	.cfi_def_cfa_offset 96
	.cfi_offset %rbx, -56
	.cfi_offset %r12, -48
	.cfi_offset %r13, -40
	.cfi_offset %r14, -32
	.cfi_offset %r15, -24
	.cfi_offset %rbp, -16
	movl	%ecx, %ebx
                                        # kill: def $edx killed $edx def $rdx
	movq	%rdx, 16(%rsp)                  # 8-byte Spill
	movl	%esi, %ebp
	movq	%rdi, %r15
	callq	_ZNSt6chrono3_V212system_clock3nowEv
	movq	%rax, 24(%rsp)                  # 8-byte Spill
	movl	%ebp, 12(%rsp)                  # 4-byte Spill
	testl	%ebp, %ebp
	jle	.LBB0_5
# %bb.1:                                # %.lr.ph
	movl	12(%rsp), %r14d                 # 4-byte Reload
	movl	%ebx, %r12d
	movslq	%ebx, %rax
	movq	%rax, 32(%rsp)                  # 8-byte Spill
	movq	16(%rsp), %rax                  # 8-byte Reload
	leal	3(%rax), %r13d
	testl	%eax, %eax
	cmovnsl	%eax, %r13d
	sarl	$2, %r13d
	xorl	%ebx, %ebx
.LBB0_2:                                # =>This Inner Loop Header: Depth=1
	cmpq	%rbx, %r12
	je	.LBB0_4
# %bb.3:                                #   in Loop: Header=BB0_2 Depth=1
	movl	%r13d, %eax
	cltd
	idivl	%r14d
	imull	%ebx, %eax
	movslq	%eax, %rsi
	shlq	$2, %rsi
	movq	32(%rsp), %rax                  # 8-byte Reload
	movq	(%r15,%rax,8), %rdi
	addq	%rsi, %rdi
	addq	(%r15,%rbx,8), %rsi
	movq	16(%rsp), %rax                  # 8-byte Reload
                                        # kill: def $eax killed $eax killed $rax
	cltd
	idivl	12(%rsp)                        # 4-byte Folded Reload
	movslq	%eax, %rdx
	movl	$4, %ecx
	callq	hipMemcpy
	testl	%eax, %eax
	jne	.LBB0_6
.LBB0_4:                                # %_Z9gpuAssert10hipError_tPKcib.exit
                                        #   in Loop: Header=BB0_2 Depth=1
	movl	%ebx, %edi
	callq	hipSetDevice
	callq	hipDeviceSynchronize
	movl	%r12d, %edi
	callq	hipSetDevice
	callq	hipDeviceSynchronize
	incq	%rbx
	cmpq	%rbx, %r14
	jne	.LBB0_2
.LBB0_5:                                # %._crit_edge
	callq	_ZNSt6chrono3_V212system_clock3nowEv
	subq	24(%rsp), %rax                  # 8-byte Folded Reload
	cvtsi2sd	%rax, %xmm0
	divsd	.LCPI0_0(%rip), %xmm0
	addq	$40, %rsp
	.cfi_def_cfa_offset 56
	popq	%rbx
	.cfi_def_cfa_offset 48
	popq	%r12
	.cfi_def_cfa_offset 40
	popq	%r13
	.cfi_def_cfa_offset 32
	popq	%r14
	.cfi_def_cfa_offset 24
	popq	%r15
	.cfi_def_cfa_offset 16
	popq	%rbp
	.cfi_def_cfa_offset 8
	retq
.LBB0_6:
	.cfi_def_cfa_offset 96
	movl	%eax, %ebp
	movl	$_ZSt4cerr, %edi
	movl	$.L.str.14, %esi
	callq	_ZStlsISt11char_traitsIcEERSt13basic_ostreamIcT_ES5_PKc
	movq	%rax, %rbx
	movl	%ebp, %edi
	callq	hipGetErrorString
	movq	%rbx, %rdi
	movq	%rax, %rsi
	callq	_ZStlsISt11char_traitsIcEERSt13basic_ostreamIcT_ES5_PKc
	movl	$.L.str.15, %esi
	movq	%rax, %rdi
	callq	_ZStlsISt11char_traitsIcEERSt13basic_ostreamIcT_ES5_PKc
	movl	$.L.str, %esi
	movq	%rax, %rdi
	callq	_ZStlsISt11char_traitsIcEERSt13basic_ostreamIcT_ES5_PKc
	movl	$.L.str.15, %esi
	movq	%rax, %rdi
	callq	_ZStlsISt11char_traitsIcEERSt13basic_ostreamIcT_ES5_PKc
	movq	%rax, %rdi
	movl	$27, %esi
	callq	_ZNSolsEi
	movq	%rax, %rdi
	callq	_ZSt4endlIcSt11char_traitsIcEERSt13basic_ostreamIT_T0_ES6_
	movl	$1, %edi
	callq	exit
.Lfunc_end0:
	.size	_Z22normalMemCpyOther2ThisPPfiii, .Lfunc_end0-_Z22normalMemCpyOther2ThisPPfiii
	.cfi_endproc
                                        # -- End function
	.section	.rodata.cst8,"aM",@progbits,8
	.p2align	3, 0x0                          # -- Begin function _Z22normalMemCpyThis2OtherPPfiii
.LCPI1_0:
	.quad	0x41cdcd6500000000              # double 1.0E+9
	.text
	.globl	_Z22normalMemCpyThis2OtherPPfiii
	.type	_Z22normalMemCpyThis2OtherPPfiii,@function
_Z22normalMemCpyThis2OtherPPfiii:       # @_Z22normalMemCpyThis2OtherPPfiii
	.cfi_startproc
# %bb.0:
	pushq	%rbp
	.cfi_def_cfa_offset 16
	pushq	%r15
	.cfi_def_cfa_offset 24
	pushq	%r14
	.cfi_def_cfa_offset 32
	pushq	%r13
	.cfi_def_cfa_offset 40
	pushq	%r12
	.cfi_def_cfa_offset 48
	pushq	%rbx
	.cfi_def_cfa_offset 56
	subq	$40, %rsp
	.cfi_def_cfa_offset 96
	.cfi_offset %rbx, -56
	.cfi_offset %r12, -48
	.cfi_offset %r13, -40
	.cfi_offset %r14, -32
	.cfi_offset %r15, -24
	.cfi_offset %rbp, -16
	movl	%ecx, %ebx
                                        # kill: def $edx killed $edx def $rdx
	movq	%rdx, 16(%rsp)                  # 8-byte Spill
	movl	%esi, %ebp
	movq	%rdi, %r15
	callq	_ZNSt6chrono3_V212system_clock3nowEv
	movq	%rax, 24(%rsp)                  # 8-byte Spill
	movl	%ebp, 12(%rsp)                  # 4-byte Spill
	testl	%ebp, %ebp
	jle	.LBB1_5
# %bb.1:                                # %.lr.ph
	movl	12(%rsp), %r14d                 # 4-byte Reload
	movq	16(%rsp), %rax                  # 8-byte Reload
	leal	3(%rax), %r12d
	testl	%eax, %eax
	cmovnsl	%eax, %r12d
	movl	%ebx, %r13d
	sarl	$2, %r12d
	movslq	%ebx, %rax
	movq	%rax, 32(%rsp)                  # 8-byte Spill
	xorl	%ebx, %ebx
.LBB1_2:                                # =>This Inner Loop Header: Depth=1
	cmpq	%rbx, %r13
	je	.LBB1_4
# %bb.3:                                #   in Loop: Header=BB1_2 Depth=1
	movl	%r12d, %eax
	cltd
	idivl	%r14d
	imull	%ebx, %eax
	movslq	%eax, %rsi
	shlq	$2, %rsi
	movq	(%r15,%rbx,8), %rdi
	addq	%rsi, %rdi
	movq	32(%rsp), %rax                  # 8-byte Reload
	addq	(%r15,%rax,8), %rsi
	movq	16(%rsp), %rax                  # 8-byte Reload
                                        # kill: def $eax killed $eax killed $rax
	cltd
	idivl	12(%rsp)                        # 4-byte Folded Reload
	movslq	%eax, %rdx
	movl	$4, %ecx
	callq	hipMemcpy
	testl	%eax, %eax
	jne	.LBB1_6
.LBB1_4:                                # %_Z9gpuAssert10hipError_tPKcib.exit
                                        #   in Loop: Header=BB1_2 Depth=1
	movl	%ebx, %edi
	callq	hipSetDevice
	callq	hipDeviceSynchronize
	movl	%r13d, %edi
	callq	hipSetDevice
	callq	hipDeviceSynchronize
	incq	%rbx
	cmpq	%rbx, %r14
	jne	.LBB1_2
.LBB1_5:                                # %._crit_edge
	callq	_ZNSt6chrono3_V212system_clock3nowEv
	subq	24(%rsp), %rax                  # 8-byte Folded Reload
	cvtsi2sd	%rax, %xmm0
	divsd	.LCPI1_0(%rip), %xmm0
	addq	$40, %rsp
	.cfi_def_cfa_offset 56
	popq	%rbx
	.cfi_def_cfa_offset 48
	popq	%r12
	.cfi_def_cfa_offset 40
	popq	%r13
	.cfi_def_cfa_offset 32
	popq	%r14
	.cfi_def_cfa_offset 24
	popq	%r15
	.cfi_def_cfa_offset 16
	popq	%rbp
	.cfi_def_cfa_offset 8
	retq
.LBB1_6:
	.cfi_def_cfa_offset 96
	movl	%eax, %ebp
	movl	$_ZSt4cerr, %edi
	movl	$.L.str.14, %esi
	callq	_ZStlsISt11char_traitsIcEERSt13basic_ostreamIcT_ES5_PKc
	movq	%rax, %rbx
	movl	%ebp, %edi
	callq	hipGetErrorString
	movq	%rbx, %rdi
	movq	%rax, %rsi
	callq	_ZStlsISt11char_traitsIcEERSt13basic_ostreamIcT_ES5_PKc
	movl	$.L.str.15, %esi
	movq	%rax, %rdi
	callq	_ZStlsISt11char_traitsIcEERSt13basic_ostreamIcT_ES5_PKc
	movl	$.L.str, %esi
	movq	%rax, %rdi
	callq	_ZStlsISt11char_traitsIcEERSt13basic_ostreamIcT_ES5_PKc
	movl	$.L.str.15, %esi
	movq	%rax, %rdi
	callq	_ZStlsISt11char_traitsIcEERSt13basic_ostreamIcT_ES5_PKc
	movq	%rax, %rdi
	movl	$48, %esi
	callq	_ZNSolsEi
	movq	%rax, %rdi
	callq	_ZSt4endlIcSt11char_traitsIcEERSt13basic_ostreamIT_T0_ES6_
	movl	$1, %edi
	callq	exit
.Lfunc_end1:
	.size	_Z22normalMemCpyThis2OtherPPfiii, .Lfunc_end1-_Z22normalMemCpyThis2OtherPPfiii
	.cfi_endproc
                                        # -- End function
	.section	.rodata.cst8,"aM",@progbits,8
	.p2align	3, 0x0                          # -- Begin function _Z27normalMemCpyOther2ThisAsyncPPfiii
.LCPI2_0:
	.quad	0x41cdcd6500000000              # double 1.0E+9
	.text
	.globl	_Z27normalMemCpyOther2ThisAsyncPPfiii
	.type	_Z27normalMemCpyOther2ThisAsyncPPfiii,@function
_Z27normalMemCpyOther2ThisAsyncPPfiii:  # @_Z27normalMemCpyOther2ThisAsyncPPfiii
	.cfi_startproc
# %bb.0:
	pushq	%rbp
	.cfi_def_cfa_offset 16
	.cfi_offset %rbp, -16
	movq	%rsp, %rbp
	.cfi_def_cfa_register %rbp
	pushq	%r15
	pushq	%r14
	pushq	%r13
	pushq	%r12
	pushq	%rbx
	subq	$56, %rsp
	.cfi_offset %rbx, -56
	.cfi_offset %r12, -48
	.cfi_offset %r13, -40
	.cfi_offset %r14, -32
	.cfi_offset %r15, -24
                                        # kill: def $edx killed $edx def $rdx
	movq	%rdx, -72(%rbp)                 # 8-byte Spill
	movq	%rdi, -96(%rbp)                 # 8-byte Spill
	movl	%esi, %r15d
	movq	%rsp, %r12
	leaq	15(,%r15,8), %rax
	andq	$-16, %rax
	subq	%rax, %r12
	movq	%r12, %rsp
	movl	%esi, -44(%rbp)                 # 4-byte Spill
	testl	%esi, %esi
	jle	.LBB2_5
# %bb.1:
	movl	%ecx, %r14d
	movl	%ecx, %r13d
	movq	%r15, -80(%rbp)                 # 8-byte Spill
	movq	%r12, -64(%rbp)                 # 8-byte Spill
.LBB2_2:                                # %_Z9gpuAssert10hipError_tPKcib.exit
                                        # =>This Inner Loop Header: Depth=1
	movq	%r12, %rdi
	callq	hipStreamCreate
	testl	%eax, %eax
	jne	.LBB2_3
# %bb.6:                                #   in Loop: Header=BB2_2 Depth=1
	addq	$8, %r12
	decq	%r15
	jne	.LBB2_2
# %bb.7:                                # %._crit_edge
	callq	_ZNSt6chrono3_V212system_clock3nowEv
	movq	%rax, %rbx
	cmpl	$0, -44(%rbp)                   # 4-byte Folded Reload
	jle	.LBB2_18
# %bb.8:                                # %.lr.ph
	movq	%rbx, -56(%rbp)                 # 8-byte Spill
	movslq	%r14d, %rax
	movq	%rax, -88(%rbp)                 # 8-byte Spill
	movq	-72(%rbp), %rax                 # 8-byte Reload
	leal	3(%rax), %r12d
	testl	%eax, %eax
	cmovnsl	%eax, %r12d
	sarl	$2, %r12d
	xorl	%r14d, %r14d
	movq	-80(%rbp), %r15                 # 8-byte Reload
.LBB2_9:                                # =>This Inner Loop Header: Depth=1
	cmpq	%r14, %r13
	je	.LBB2_12
# %bb.10:                               #   in Loop: Header=BB2_9 Depth=1
	movl	%r12d, %eax
	cltd
	idivl	%r15d
	imull	%r14d, %eax
	movslq	%eax, %rsi
	shlq	$2, %rsi
	movq	-96(%rbp), %rax                 # 8-byte Reload
	movq	-88(%rbp), %rcx                 # 8-byte Reload
	movq	(%rax,%rcx,8), %rdi
	addq	%rsi, %rdi
	addq	(%rax,%r14,8), %rsi
	movq	-72(%rbp), %rax                 # 8-byte Reload
                                        # kill: def $eax killed $eax killed $rax
	cltd
	idivl	-44(%rbp)                       # 4-byte Folded Reload
	movslq	%eax, %rdx
	movq	-64(%rbp), %rax                 # 8-byte Reload
	movq	(%rax,%r14,8), %r8
	movl	$4, %ecx
	callq	hipMemcpyAsync
	testl	%eax, %eax
	jne	.LBB2_11
.LBB2_12:                               # %_Z9gpuAssert10hipError_tPKcib.exit34
                                        #   in Loop: Header=BB2_9 Depth=1
	movl	%r14d, %edi
	callq	hipSetDevice
	callq	hipDeviceSynchronize
	movl	%r13d, %edi
	callq	hipSetDevice
	callq	hipDeviceSynchronize
	incq	%r14
	cmpq	%r14, %r15
	jne	.LBB2_9
# %bb.13:                               # %.preheader
	cmpl	$0, -44(%rbp)                   # 4-byte Folded Reload
	jle	.LBB2_17
# %bb.14:                               # %_Z9gpuAssert10hipError_tPKcib.exit36.preheader
	xorl	%r14d, %r14d
	movq	-64(%rbp), %r12                 # 8-byte Reload
.LBB2_15:                               # %_Z9gpuAssert10hipError_tPKcib.exit36
                                        # =>This Inner Loop Header: Depth=1
	movq	(%r12,%r14,8), %rdi
	callq	hipStreamSynchronize
	testl	%eax, %eax
	jne	.LBB2_19
# %bb.16:                               #   in Loop: Header=BB2_15 Depth=1
	incq	%r14
	cmpq	%r14, %r15
	jne	.LBB2_15
.LBB2_17:
	movq	-56(%rbp), %rbx                 # 8-byte Reload
	jmp	.LBB2_18
.LBB2_5:                                # %._crit_edge.thread
	callq	_ZNSt6chrono3_V212system_clock3nowEv
	movq	%rax, %rbx
.LBB2_18:                               # %._crit_edge46
	callq	_ZNSt6chrono3_V212system_clock3nowEv
	subq	%rbx, %rax
	cvtsi2sd	%rax, %xmm0
	divsd	.LCPI2_0(%rip), %xmm0
	leaq	-40(%rbp), %rsp
	popq	%rbx
	popq	%r12
	popq	%r13
	popq	%r14
	popq	%r15
	popq	%rbp
	.cfi_def_cfa %rsp, 8
	retq
.LBB2_3:
	.cfi_def_cfa %rbp, 16
	movl	%eax, %ebx
	movl	$_ZSt4cerr, %edi
	movl	$.L.str.14, %esi
	callq	_ZStlsISt11char_traitsIcEERSt13basic_ostreamIcT_ES5_PKc
	movq	%rax, %r14
	movl	%ebx, %edi
	callq	hipGetErrorString
	movq	%r14, %rdi
	movq	%rax, %rsi
	callq	_ZStlsISt11char_traitsIcEERSt13basic_ostreamIcT_ES5_PKc
	movl	$.L.str.15, %esi
	movq	%rax, %rdi
	callq	_ZStlsISt11char_traitsIcEERSt13basic_ostreamIcT_ES5_PKc
	movl	$.L.str, %esi
	movq	%rax, %rdi
	callq	_ZStlsISt11char_traitsIcEERSt13basic_ostreamIcT_ES5_PKc
	movl	$.L.str.15, %esi
	movq	%rax, %rdi
	callq	_ZStlsISt11char_traitsIcEERSt13basic_ostreamIcT_ES5_PKc
	movq	%rax, %rdi
	movl	$65, %esi
	jmp	.LBB2_4
.LBB2_19:
	movl	%eax, %ebx
	movl	$_ZSt4cerr, %edi
	movl	$.L.str.14, %esi
	callq	_ZStlsISt11char_traitsIcEERSt13basic_ostreamIcT_ES5_PKc
	movq	%rax, %r14
	movl	%ebx, %edi
	callq	hipGetErrorString
	movq	%r14, %rdi
	movq	%rax, %rsi
	callq	_ZStlsISt11char_traitsIcEERSt13basic_ostreamIcT_ES5_PKc
	movl	$.L.str.15, %esi
	movq	%rax, %rdi
	callq	_ZStlsISt11char_traitsIcEERSt13basic_ostreamIcT_ES5_PKc
	movl	$.L.str, %esi
	movq	%rax, %rdi
	callq	_ZStlsISt11char_traitsIcEERSt13basic_ostreamIcT_ES5_PKc
	movl	$.L.str.15, %esi
	movq	%rax, %rdi
	callq	_ZStlsISt11char_traitsIcEERSt13basic_ostreamIcT_ES5_PKc
	movq	%rax, %rdi
	movl	$83, %esi
	jmp	.LBB2_4
.LBB2_11:
	movl	%eax, %ebx
	movl	$_ZSt4cerr, %edi
	movl	$.L.str.14, %esi
	callq	_ZStlsISt11char_traitsIcEERSt13basic_ostreamIcT_ES5_PKc
	movq	%rax, %r14
	movl	%ebx, %edi
	callq	hipGetErrorString
	movq	%r14, %rdi
	movq	%rax, %rsi
	callq	_ZStlsISt11char_traitsIcEERSt13basic_ostreamIcT_ES5_PKc
	movl	$.L.str.15, %esi
	movq	%rax, %rdi
	callq	_ZStlsISt11char_traitsIcEERSt13basic_ostreamIcT_ES5_PKc
	movl	$.L.str, %esi
	movq	%rax, %rdi
	callq	_ZStlsISt11char_traitsIcEERSt13basic_ostreamIcT_ES5_PKc
	movl	$.L.str.15, %esi
	movq	%rax, %rdi
	callq	_ZStlsISt11char_traitsIcEERSt13basic_ostreamIcT_ES5_PKc
	movq	%rax, %rdi
	movl	$74, %esi
.LBB2_4:
	callq	_ZNSolsEi
	movq	%rax, %rdi
	callq	_ZSt4endlIcSt11char_traitsIcEERSt13basic_ostreamIT_T0_ES6_
	movl	$1, %edi
	callq	exit
.Lfunc_end2:
	.size	_Z27normalMemCpyOther2ThisAsyncPPfiii, .Lfunc_end2-_Z27normalMemCpyOther2ThisAsyncPPfiii
	.cfi_endproc
                                        # -- End function
	.section	.rodata.cst8,"aM",@progbits,8
	.p2align	3, 0x0                          # -- Begin function _Z27normalMemCpyThis2OtherAsyncPPfiii
.LCPI3_0:
	.quad	0x41cdcd6500000000              # double 1.0E+9
	.text
	.globl	_Z27normalMemCpyThis2OtherAsyncPPfiii
	.type	_Z27normalMemCpyThis2OtherAsyncPPfiii,@function
_Z27normalMemCpyThis2OtherAsyncPPfiii:  # @_Z27normalMemCpyThis2OtherAsyncPPfiii
	.cfi_startproc
# %bb.0:
	pushq	%rbp
	.cfi_def_cfa_offset 16
	.cfi_offset %rbp, -16
	movq	%rsp, %rbp
	.cfi_def_cfa_register %rbp
	pushq	%r15
	pushq	%r14
	pushq	%r13
	pushq	%r12
	pushq	%rbx
	subq	$56, %rsp
	.cfi_offset %rbx, -56
	.cfi_offset %r12, -48
	.cfi_offset %r13, -40
	.cfi_offset %r14, -32
	.cfi_offset %r15, -24
                                        # kill: def $edx killed $edx def $rdx
	movq	%rdx, -72(%rbp)                 # 8-byte Spill
	movq	%rdi, -96(%rbp)                 # 8-byte Spill
	movl	%esi, %r15d
	movq	%rsp, %r12
	leaq	15(,%r15,8), %rax
	andq	$-16, %rax
	subq	%rax, %r12
	movq	%r12, %rsp
	movl	%esi, -44(%rbp)                 # 4-byte Spill
	testl	%esi, %esi
	jle	.LBB3_5
# %bb.1:
	movl	%ecx, %r14d
	movl	%ecx, %r13d
	movq	%r15, -80(%rbp)                 # 8-byte Spill
	movq	%r12, -64(%rbp)                 # 8-byte Spill
.LBB3_2:                                # %_Z9gpuAssert10hipError_tPKcib.exit
                                        # =>This Inner Loop Header: Depth=1
	movq	%r12, %rdi
	callq	hipStreamCreate
	testl	%eax, %eax
	jne	.LBB3_3
# %bb.6:                                #   in Loop: Header=BB3_2 Depth=1
	addq	$8, %r12
	decq	%r15
	jne	.LBB3_2
# %bb.7:                                # %._crit_edge
	callq	_ZNSt6chrono3_V212system_clock3nowEv
	movq	%rax, %rbx
	cmpl	$0, -44(%rbp)                   # 4-byte Folded Reload
	jle	.LBB3_18
# %bb.8:                                # %.lr.ph
	movq	%rbx, -56(%rbp)                 # 8-byte Spill
	movq	-72(%rbp), %rax                 # 8-byte Reload
	leal	3(%rax), %r12d
	testl	%eax, %eax
	cmovnsl	%eax, %r12d
	sarl	$2, %r12d
	movslq	%r14d, %rax
	movq	%rax, -88(%rbp)                 # 8-byte Spill
	xorl	%r14d, %r14d
	movq	-80(%rbp), %r15                 # 8-byte Reload
.LBB3_9:                                # =>This Inner Loop Header: Depth=1
	cmpq	%r14, %r13
	je	.LBB3_12
# %bb.10:                               #   in Loop: Header=BB3_9 Depth=1
	movl	%r12d, %eax
	cltd
	idivl	%r15d
	imull	%r14d, %eax
	movslq	%eax, %rsi
	shlq	$2, %rsi
	movq	-96(%rbp), %rax                 # 8-byte Reload
	movq	(%rax,%r14,8), %rdi
	addq	%rsi, %rdi
	movq	-88(%rbp), %rcx                 # 8-byte Reload
	addq	(%rax,%rcx,8), %rsi
	movq	-72(%rbp), %rax                 # 8-byte Reload
                                        # kill: def $eax killed $eax killed $rax
	cltd
	idivl	-44(%rbp)                       # 4-byte Folded Reload
	movslq	%eax, %rdx
	movq	-64(%rbp), %rax                 # 8-byte Reload
	movq	(%rax,%r14,8), %r8
	movl	$4, %ecx
	callq	hipMemcpyAsync
	testl	%eax, %eax
	jne	.LBB3_11
.LBB3_12:                               # %_Z9gpuAssert10hipError_tPKcib.exit34
                                        #   in Loop: Header=BB3_9 Depth=1
	movl	%r14d, %edi
	callq	hipSetDevice
	callq	hipDeviceSynchronize
	movl	%r13d, %edi
	callq	hipSetDevice
	callq	hipDeviceSynchronize
	incq	%r14
	cmpq	%r14, %r15
	jne	.LBB3_9
# %bb.13:                               # %.preheader
	cmpl	$0, -44(%rbp)                   # 4-byte Folded Reload
	jle	.LBB3_17
# %bb.14:                               # %_Z9gpuAssert10hipError_tPKcib.exit36.preheader
	xorl	%r14d, %r14d
	movq	-64(%rbp), %r12                 # 8-byte Reload
.LBB3_15:                               # %_Z9gpuAssert10hipError_tPKcib.exit36
                                        # =>This Inner Loop Header: Depth=1
	movq	(%r12,%r14,8), %rdi
	callq	hipStreamSynchronize
	testl	%eax, %eax
	jne	.LBB3_19
# %bb.16:                               #   in Loop: Header=BB3_15 Depth=1
	incq	%r14
	cmpq	%r14, %r15
	jne	.LBB3_15
.LBB3_17:
	movq	-56(%rbp), %rbx                 # 8-byte Reload
	jmp	.LBB3_18
.LBB3_5:                                # %._crit_edge.thread
	callq	_ZNSt6chrono3_V212system_clock3nowEv
	movq	%rax, %rbx
.LBB3_18:                               # %._crit_edge46
	callq	_ZNSt6chrono3_V212system_clock3nowEv
	subq	%rbx, %rax
	cvtsi2sd	%rax, %xmm0
	divsd	.LCPI3_0(%rip), %xmm0
	leaq	-40(%rbp), %rsp
	popq	%rbx
	popq	%r12
	popq	%r13
	popq	%r14
	popq	%r15
	popq	%rbp
	.cfi_def_cfa %rsp, 8
	retq
.LBB3_3:
	.cfi_def_cfa %rbp, 16
	movl	%eax, %ebx
	movl	$_ZSt4cerr, %edi
	movl	$.L.str.14, %esi
	callq	_ZStlsISt11char_traitsIcEERSt13basic_ostreamIcT_ES5_PKc
	movq	%rax, %r14
	movl	%ebx, %edi
	callq	hipGetErrorString
	movq	%r14, %rdi
	movq	%rax, %rsi
	callq	_ZStlsISt11char_traitsIcEERSt13basic_ostreamIcT_ES5_PKc
	movl	$.L.str.15, %esi
	movq	%rax, %rdi
	callq	_ZStlsISt11char_traitsIcEERSt13basic_ostreamIcT_ES5_PKc
	movl	$.L.str, %esi
	movq	%rax, %rdi
	callq	_ZStlsISt11char_traitsIcEERSt13basic_ostreamIcT_ES5_PKc
	movl	$.L.str.15, %esi
	movq	%rax, %rdi
	callq	_ZStlsISt11char_traitsIcEERSt13basic_ostreamIcT_ES5_PKc
	movq	%rax, %rdi
	movl	$95, %esi
	jmp	.LBB3_4
.LBB3_19:
	movl	%eax, %ebx
	movl	$_ZSt4cerr, %edi
	movl	$.L.str.14, %esi
	callq	_ZStlsISt11char_traitsIcEERSt13basic_ostreamIcT_ES5_PKc
	movq	%rax, %r14
	movl	%ebx, %edi
	callq	hipGetErrorString
	movq	%r14, %rdi
	movq	%rax, %rsi
	callq	_ZStlsISt11char_traitsIcEERSt13basic_ostreamIcT_ES5_PKc
	movl	$.L.str.15, %esi
	movq	%rax, %rdi
	callq	_ZStlsISt11char_traitsIcEERSt13basic_ostreamIcT_ES5_PKc
	movl	$.L.str, %esi
	movq	%rax, %rdi
	callq	_ZStlsISt11char_traitsIcEERSt13basic_ostreamIcT_ES5_PKc
	movl	$.L.str.15, %esi
	movq	%rax, %rdi
	callq	_ZStlsISt11char_traitsIcEERSt13basic_ostreamIcT_ES5_PKc
	movq	%rax, %rdi
	movl	$114, %esi
	jmp	.LBB3_4
.LBB3_11:
	movl	%eax, %ebx
	movl	$_ZSt4cerr, %edi
	movl	$.L.str.14, %esi
	callq	_ZStlsISt11char_traitsIcEERSt13basic_ostreamIcT_ES5_PKc
	movq	%rax, %r14
	movl	%ebx, %edi
	callq	hipGetErrorString
	movq	%r14, %rdi
	movq	%rax, %rsi
	callq	_ZStlsISt11char_traitsIcEERSt13basic_ostreamIcT_ES5_PKc
	movl	$.L.str.15, %esi
	movq	%rax, %rdi
	callq	_ZStlsISt11char_traitsIcEERSt13basic_ostreamIcT_ES5_PKc
	movl	$.L.str, %esi
	movq	%rax, %rdi
	callq	_ZStlsISt11char_traitsIcEERSt13basic_ostreamIcT_ES5_PKc
	movl	$.L.str.15, %esi
	movq	%rax, %rdi
	callq	_ZStlsISt11char_traitsIcEERSt13basic_ostreamIcT_ES5_PKc
	movq	%rax, %rdi
	movl	$104, %esi
.LBB3_4:
	callq	_ZNSolsEi
	movq	%rax, %rdi
	callq	_ZSt4endlIcSt11char_traitsIcEERSt13basic_ostreamIT_T0_ES6_
	movl	$1, %edi
	callq	exit
.Lfunc_end3:
	.size	_Z27normalMemCpyThis2OtherAsyncPPfiii, .Lfunc_end3-_Z27normalMemCpyThis2OtherAsyncPPfiii
	.cfi_endproc
                                        # -- End function
	.globl	main                            # -- Begin function main
	.type	main,@function
main:                                   # @main
	.cfi_startproc
# %bb.0:
	pushq	%rbp
	.cfi_def_cfa_offset 16
	pushq	%r15
	.cfi_def_cfa_offset 24
	pushq	%r14
	.cfi_def_cfa_offset 32
	pushq	%r13
	.cfi_def_cfa_offset 40
	pushq	%r12
	.cfi_def_cfa_offset 48
	pushq	%rbx
	.cfi_def_cfa_offset 56
	subq	$40, %rsp
	.cfi_def_cfa_offset 96
	.cfi_offset %rbx, -56
	.cfi_offset %r12, -48
	.cfi_offset %r13, -40
	.cfi_offset %r14, -32
	.cfi_offset %r15, -24
	.cfi_offset %rbp, -16
	movq	%rsi, %r14
	cmpl	$3, %edi
	jne	.LBB4_1
# %bb.2:
	movq	8(%r14), %rdi
	callq	atoi
	movl	%eax, %ebx
	movq	16(%r14), %rdi
	callq	atoi
	shll	$20, %eax
	movl	%eax, 20(%rsp)                  # 4-byte Spill
	movslq	%ebx, %r14
	leaq	(,%r14,8), %rax
	testl	%r14d, %r14d
	movq	$-1, %rdi
	cmovnsq	%rax, %rdi
	callq	_Znam
	movq	%rax, 24(%rsp)                  # 8-byte Spill
	testl	%r14d, %r14d
	jle	.LBB4_6
# %bb.3:                                # %_Z9gpuAssert10hipError_tPKcib.exit.lr.ph
	movslq	20(%rsp), %r15                  # 4-byte Folded Reload
	movl	%ebx, %r14d
	xorl	%r12d, %r12d
	movq	24(%rsp), %r13                  # 8-byte Reload
.LBB4_4:                                # %_Z9gpuAssert10hipError_tPKcib.exit
                                        # =>This Inner Loop Header: Depth=1
	movl	%r12d, %edi
	callq	hipSetDevice
	movq	%r13, %rdi
	movq	%r15, %rsi
	callq	hipMalloc
	testl	%eax, %eax
	jne	.LBB4_15
# %bb.5:                                #   in Loop: Header=BB4_4 Depth=1
	incq	%r12
	addq	$8, %r13
	cmpq	%r12, %r14
	jne	.LBB4_4
.LBB4_6:                                # %._crit_edge
	movq	24(%rsp), %r12                  # 8-byte Reload
	movq	%r12, %rdi
	movl	%ebx, %esi
	movl	20(%rsp), %ebp                  # 4-byte Reload
	movl	%ebp, %edx
	xorl	%ecx, %ecx
	callq	_Z22normalMemCpyOther2ThisPPfiii
	movsd	%xmm0, (%rsp)                   # 8-byte Spill
	movq	%r12, %rdi
	movl	%ebx, %esi
	movl	%ebp, %edx
	xorl	%ecx, %ecx
	callq	_Z22normalMemCpyThis2OtherPPfiii
	movsd	%xmm0, 8(%rsp)                  # 8-byte Spill
	movq	%r12, %rdi
	movl	%ebx, %esi
	movl	%ebp, %edx
	xorl	%ecx, %ecx
	callq	_Z27normalMemCpyOther2ThisAsyncPPfiii
	movq	%r12, %rdi
	movl	%ebx, %esi
	movl	%ebp, %edx
	xorl	%ecx, %ecx
	callq	_Z27normalMemCpyThis2OtherAsyncPPfiii
	cmpl	$2, %ebx
	jl	.LBB4_9
# %bb.7:                                # %.lr.ph.preheader
	movl	$1, %r15d
.LBB4_8:                                # %.lr.ph
                                        # =>This Inner Loop Header: Depth=1
	movq	%r12, %rdi
	movl	%ebx, %esi
	movl	%ebp, %edx
	movl	%r15d, %ecx
	callq	_Z22normalMemCpyOther2ThisPPfiii
	movsd	(%rsp), %xmm1                   # 8-byte Reload
                                        # xmm1 = mem[0],zero
	addsd	%xmm0, %xmm1
	movsd	%xmm1, (%rsp)                   # 8-byte Spill
	movq	%r12, %rdi
	movl	%ebx, %esi
	movl	%ebp, %edx
	movl	%r15d, %ecx
	callq	_Z22normalMemCpyThis2OtherPPfiii
	movsd	8(%rsp), %xmm1                  # 8-byte Reload
                                        # xmm1 = mem[0],zero
	addsd	%xmm0, %xmm1
	movsd	%xmm1, 8(%rsp)                  # 8-byte Spill
	movq	%r12, %rdi
	movl	%ebx, %esi
	movl	%ebp, %edx
	movl	%r15d, %ecx
	callq	_Z27normalMemCpyOther2ThisAsyncPPfiii
	movq	%r12, %rdi
	movl	%ebx, %esi
	movl	%ebp, %edx
	movl	%r15d, %ecx
	callq	_Z27normalMemCpyThis2OtherAsyncPPfiii
	incl	%r15d
	cmpl	%r15d, %ebx
	jne	.LBB4_8
.LBB4_9:                                # %._crit_edge126
	movq	_ZSt4cout(%rip), %rax
	movq	-24(%rax), %rax
	movl	$-261, %ebp                     # imm = 0xFEFB
	movl	_ZSt4cout+24(%rax), %ecx
	andl	%ebp, %ecx
	orl	$4, %ecx
	movl	%ecx, _ZSt4cout+24(%rax)
	movl	$_ZSt4cout, %edi
	movl	$.L.str.3, %esi
	movl	$20, %edx
	callq	_ZSt16__ostream_insertIcSt11char_traitsIcEERSt13basic_ostreamIT_T0_ES6_PKS3_l
	movl	$_ZSt4cout, %edi
	movsd	(%rsp), %xmm0                   # 8-byte Reload
                                        # xmm0 = mem[0],zero
	callq	_ZNSo9_M_insertIdEERSoT_
	movq	%rax, %r15
	movl	$.L.str.4, %esi
	movl	$9, %edx
	movq	%rax, %rdi
	callq	_ZSt16__ostream_insertIcSt11char_traitsIcEERSt13basic_ostreamIT_T0_ES6_PKS3_l
	movq	(%r15), %rax
	movq	-24(%rax), %rdi
	addq	%r15, %rdi
	movl	$10, %esi
	callq	_ZNKSt9basic_iosIcSt11char_traitsIcEE5widenEc
	movsbl	%al, %esi
	movq	%r15, %rdi
	callq	_ZNSo3putEc
	movq	%rax, %rdi
	callq	_ZNSo5flushEv
	movq	_ZSt4cout(%rip), %rax
	movq	-24(%rax), %rax
	movl	_ZSt4cout+24(%rax), %ecx
	andl	%ebp, %ecx
	orl	$4, %ecx
	movl	%ecx, _ZSt4cout+24(%rax)
	movl	$_ZSt4cout, %edi
	movl	$.L.str.5, %esi
	movl	$20, %edx
	callq	_ZSt16__ostream_insertIcSt11char_traitsIcEERSt13basic_ostreamIT_T0_ES6_PKS3_l
	movl	$_ZSt4cout, %edi
	movsd	8(%rsp), %xmm0                  # 8-byte Reload
                                        # xmm0 = mem[0],zero
	callq	_ZNSo9_M_insertIdEERSoT_
	movq	%rax, %r15
	movl	$.L.str.4, %esi
	movl	$9, %edx
	movq	%rax, %rdi
	callq	_ZSt16__ostream_insertIcSt11char_traitsIcEERSt13basic_ostreamIT_T0_ES6_PKS3_l
	movq	(%r15), %rax
	movq	-24(%rax), %rdi
	addq	%r15, %rdi
	movl	$10, %esi
	callq	_ZNKSt9basic_iosIcSt11char_traitsIcEE5widenEc
	movsbl	%al, %esi
	movq	%r15, %rdi
	callq	_ZNSo3putEc
	movq	%rax, %rdi
	callq	_ZNSo5flushEv
	movq	_ZSt4cout(%rip), %rax
	movq	-24(%rax), %rax
	movl	_ZSt4cout+24(%rax), %ecx
	andl	%ebp, %ecx
	orl	$4, %ecx
	movl	%ecx, _ZSt4cout+24(%rax)
	movl	$_ZSt4cout, %edi
	movl	$.L.str.6, %esi
	movl	$26, %edx
	callq	_ZSt16__ostream_insertIcSt11char_traitsIcEERSt13basic_ostreamIT_T0_ES6_PKS3_l
	movl	$_ZSt4cout, %edi
	movsd	(%rsp), %xmm0                   # 8-byte Reload
                                        # xmm0 = mem[0],zero
	callq	_ZNSo9_M_insertIdEERSoT_
	movq	%rax, %r15
	movl	$.L.str.4, %esi
	movl	$9, %edx
	movq	%rax, %rdi
	callq	_ZSt16__ostream_insertIcSt11char_traitsIcEERSt13basic_ostreamIT_T0_ES6_PKS3_l
	movq	(%r15), %rax
	movq	-24(%rax), %rdi
	addq	%r15, %rdi
	movl	$10, %esi
	callq	_ZNKSt9basic_iosIcSt11char_traitsIcEE5widenEc
	movsbl	%al, %esi
	movq	%r15, %rdi
	callq	_ZNSo3putEc
	movq	%rax, %rdi
	callq	_ZNSo5flushEv
	movq	_ZSt4cout(%rip), %rax
	movq	-24(%rax), %rax
	movl	_ZSt4cout+24(%rax), %ecx
	movl	$-261, %r14d                    # imm = 0xFEFB
	andl	%ebp, %ecx
	orl	$4, %ecx
	movl	%ecx, _ZSt4cout+24(%rax)
	movl	$_ZSt4cout, %edi
	movl	$.L.str.7, %esi
	movl	$26, %edx
	callq	_ZSt16__ostream_insertIcSt11char_traitsIcEERSt13basic_ostreamIT_T0_ES6_PKS3_l
	movl	$_ZSt4cout, %edi
	movsd	8(%rsp), %xmm0                  # 8-byte Reload
                                        # xmm0 = mem[0],zero
	callq	_ZNSo9_M_insertIdEERSoT_
	movq	%rax, %r15
	movl	$.L.str.4, %esi
	movl	$9, %edx
	movq	%rax, %rdi
	callq	_ZSt16__ostream_insertIcSt11char_traitsIcEERSt13basic_ostreamIT_T0_ES6_PKS3_l
	movq	(%r15), %rax
	movq	-24(%rax), %rdi
	addq	%r15, %rdi
	movl	$10, %esi
	callq	_ZNKSt9basic_iosIcSt11char_traitsIcEE5widenEc
	movsbl	%al, %esi
	movq	%r15, %rdi
	callq	_ZNSo3putEc
	movq	%rax, %rdi
	callq	_ZNSo5flushEv
	testl	%ebx, %ebx
	jle	.LBB4_18
# %bb.10:                               # %.lr.ph130.preheader.preheader
	xorl	%r15d, %r15d
	leaq	36(%rsp), %r12
.LBB4_11:                               # %.lr.ph130.preheader
                                        # =>This Loop Header: Depth=1
                                        #     Child Loop BB4_12 Depth 2
	xorl	%r13d, %r13d
.LBB4_12:                               # %.lr.ph130
                                        #   Parent Loop BB4_11 Depth=1
                                        # =>  This Inner Loop Header: Depth=2
	cmpl	%r13d, %r15d
	je	.LBB4_25
# %bb.13:                               #   in Loop: Header=BB4_12 Depth=2
	movq	%r12, %rdi
	movl	%r15d, %esi
	movl	%r13d, %edx
	callq	hipDeviceCanAccessPeer
	testl	%eax, %eax
	jne	.LBB4_14
# %bb.22:                               # %_Z9gpuAssert10hipError_tPKcib.exit119
                                        #   in Loop: Header=BB4_12 Depth=2
	cmpl	$0, 36(%rsp)
	je	.LBB4_24
# %bb.23:                               #   in Loop: Header=BB4_12 Depth=2
	movl	%r15d, %edi
	callq	hipSetDevice
	movl	%r13d, %edi
	xorl	%esi, %esi
	callq	hipDeviceEnablePeerAccess
	jmp	.LBB4_25
.LBB4_24:                               #   in Loop: Header=BB4_12 Depth=2
	movq	_ZSt4cout(%rip), %rax
	movq	-24(%rax), %rax
	movl	_ZSt4cout+24(%rax), %ecx
	andl	%r14d, %ecx
	orl	$4, %ecx
	movl	%ecx, _ZSt4cout+24(%rax)
	movl	$_ZSt4cout, %edi
	movl	$.L.str.8, %esi
	movl	$41, %edx
	callq	_ZSt16__ostream_insertIcSt11char_traitsIcEERSt13basic_ostreamIT_T0_ES6_PKS3_l
	movl	$_ZSt4cout, %edi
	movl	%r15d, %esi
	callq	_ZNSolsEi
	movq	%rax, %rbp
	movl	$.L.str.9, %esi
	movl	$5, %edx
	movq	%rax, %rdi
	callq	_ZSt16__ostream_insertIcSt11char_traitsIcEERSt13basic_ostreamIT_T0_ES6_PKS3_l
	movq	%rbp, %rdi
	movl	%r13d, %esi
	callq	_ZNSolsEi
	movq	%rax, %rbp
	movq	(%rax), %rax
	movq	-24(%rax), %rdi
	addq	%rbp, %rdi
	movl	$10, %esi
	callq	_ZNKSt9basic_iosIcSt11char_traitsIcEE5widenEc
	movsbl	%al, %esi
	movq	%rbp, %rdi
	callq	_ZNSo3putEc
	movq	%rax, %rdi
	callq	_ZNSo5flushEv
.LBB4_25:                               #   in Loop: Header=BB4_12 Depth=2
	incl	%r13d
	cmpl	%r13d, %ebx
	jne	.LBB4_12
# %bb.26:                               # %._crit_edge131
                                        #   in Loop: Header=BB4_11 Depth=1
	incl	%r15d
	cmpl	%ebx, %r15d
	jne	.LBB4_11
.LBB4_18:                               # %._crit_edge133
	movq	24(%rsp), %r12                  # 8-byte Reload
	movq	%r12, %rdi
	movl	%ebx, %esi
	movl	20(%rsp), %ebp                  # 4-byte Reload
	movl	%ebp, %edx
	xorl	%ecx, %ecx
	callq	_Z22normalMemCpyOther2ThisPPfiii
	movsd	%xmm0, (%rsp)                   # 8-byte Spill
	movq	%r12, %rdi
	movl	%ebx, %esi
	movl	%ebp, %edx
	xorl	%ecx, %ecx
	callq	_Z22normalMemCpyThis2OtherPPfiii
	movsd	%xmm0, 8(%rsp)                  # 8-byte Spill
	movq	%r12, %rdi
	movl	%ebx, %esi
	movl	%ebp, %edx
	xorl	%ecx, %ecx
	callq	_Z27normalMemCpyOther2ThisAsyncPPfiii
	movq	%r12, %rdi
	movl	%ebx, %esi
	movl	%ebp, %edx
	xorl	%ecx, %ecx
	callq	_Z27normalMemCpyThis2OtherAsyncPPfiii
	cmpl	$2, %ebx
	jl	.LBB4_21
# %bb.19:                               # %.lr.ph138.preheader
	movl	$1, %r15d
.LBB4_20:                               # %.lr.ph138
                                        # =>This Inner Loop Header: Depth=1
	movq	%r12, %rdi
	movl	%ebx, %esi
	movl	%ebp, %edx
	movl	%r15d, %ecx
	callq	_Z22normalMemCpyOther2ThisPPfiii
	movsd	(%rsp), %xmm1                   # 8-byte Reload
                                        # xmm1 = mem[0],zero
	addsd	%xmm0, %xmm1
	movsd	%xmm1, (%rsp)                   # 8-byte Spill
	movq	%r12, %rdi
	movl	%ebx, %esi
	movl	%ebp, %edx
	movl	%r15d, %ecx
	callq	_Z22normalMemCpyThis2OtherPPfiii
	movsd	8(%rsp), %xmm1                  # 8-byte Reload
                                        # xmm1 = mem[0],zero
	addsd	%xmm0, %xmm1
	movsd	%xmm1, 8(%rsp)                  # 8-byte Spill
	movq	%r12, %rdi
	movl	%ebx, %esi
	movl	%ebp, %edx
	movl	%r15d, %ecx
	callq	_Z27normalMemCpyOther2ThisAsyncPPfiii
	movq	%r12, %rdi
	movl	%ebx, %esi
	movl	%ebp, %edx
	movl	%r15d, %ecx
	callq	_Z27normalMemCpyThis2OtherAsyncPPfiii
	incl	%r15d
	cmpl	%r15d, %ebx
	jne	.LBB4_20
.LBB4_21:                               # %._crit_edge139
	movq	_ZSt4cout(%rip), %rax
	movq	-24(%rax), %rax
	movl	$-261, %ebp                     # imm = 0xFEFB
	movl	_ZSt4cout+24(%rax), %ecx
	andl	%ebp, %ecx
	orl	$4, %ecx
	movl	%ecx, _ZSt4cout+24(%rax)
	movl	$_ZSt4cout, %edi
	movl	$.L.str.10, %esi
	movl	$25, %edx
	callq	_ZSt16__ostream_insertIcSt11char_traitsIcEERSt13basic_ostreamIT_T0_ES6_PKS3_l
	movl	$_ZSt4cout, %edi
	movsd	(%rsp), %xmm0                   # 8-byte Reload
                                        # xmm0 = mem[0],zero
	callq	_ZNSo9_M_insertIdEERSoT_
	movq	%rax, %rbx
	movl	$.L.str.4, %esi
	movl	$9, %edx
	movq	%rax, %rdi
	callq	_ZSt16__ostream_insertIcSt11char_traitsIcEERSt13basic_ostreamIT_T0_ES6_PKS3_l
	movq	(%rbx), %rax
	movq	-24(%rax), %rdi
	addq	%rbx, %rdi
	movl	$10, %esi
	callq	_ZNKSt9basic_iosIcSt11char_traitsIcEE5widenEc
	movsbl	%al, %esi
	movq	%rbx, %rdi
	callq	_ZNSo3putEc
	movq	%rax, %rdi
	callq	_ZNSo5flushEv
	movq	_ZSt4cout(%rip), %rax
	movq	-24(%rax), %rax
	movl	_ZSt4cout+24(%rax), %ecx
	andl	%ebp, %ecx
	orl	$4, %ecx
	movl	%ecx, _ZSt4cout+24(%rax)
	movl	$_ZSt4cout, %edi
	movl	$.L.str.11, %esi
	movl	$25, %edx
	callq	_ZSt16__ostream_insertIcSt11char_traitsIcEERSt13basic_ostreamIT_T0_ES6_PKS3_l
	movl	$_ZSt4cout, %edi
	movsd	8(%rsp), %xmm0                  # 8-byte Reload
                                        # xmm0 = mem[0],zero
	callq	_ZNSo9_M_insertIdEERSoT_
	movq	%rax, %rbx
	movl	$.L.str.4, %esi
	movl	$9, %edx
	movq	%rax, %rdi
	callq	_ZSt16__ostream_insertIcSt11char_traitsIcEERSt13basic_ostreamIT_T0_ES6_PKS3_l
	movq	(%rbx), %rax
	movq	-24(%rax), %rdi
	addq	%rbx, %rdi
	movl	$10, %esi
	callq	_ZNKSt9basic_iosIcSt11char_traitsIcEE5widenEc
	movsbl	%al, %esi
	movq	%rbx, %rdi
	callq	_ZNSo3putEc
	movq	%rax, %rdi
	callq	_ZNSo5flushEv
	movq	_ZSt4cout(%rip), %rax
	movq	-24(%rax), %rax
	movl	_ZSt4cout+24(%rax), %ecx
	andl	%ebp, %ecx
	orl	$4, %ecx
	movl	%ecx, _ZSt4cout+24(%rax)
	movl	$_ZSt4cout, %edi
	movl	$.L.str.12, %esi
	movl	$31, %edx
	callq	_ZSt16__ostream_insertIcSt11char_traitsIcEERSt13basic_ostreamIT_T0_ES6_PKS3_l
	movl	$_ZSt4cout, %edi
	movsd	(%rsp), %xmm0                   # 8-byte Reload
                                        # xmm0 = mem[0],zero
	callq	_ZNSo9_M_insertIdEERSoT_
	movq	%rax, %rbx
	movl	$.L.str.4, %esi
	movl	$9, %edx
	movq	%rax, %rdi
	callq	_ZSt16__ostream_insertIcSt11char_traitsIcEERSt13basic_ostreamIT_T0_ES6_PKS3_l
	movq	(%rbx), %rax
	movq	-24(%rax), %rdi
	addq	%rbx, %rdi
	movl	$10, %esi
	callq	_ZNKSt9basic_iosIcSt11char_traitsIcEE5widenEc
	movsbl	%al, %esi
	movq	%rbx, %rdi
	callq	_ZNSo3putEc
	movq	%rax, %rdi
	callq	_ZNSo5flushEv
	movq	_ZSt4cout(%rip), %rax
	movq	-24(%rax), %rax
	andl	_ZSt4cout+24(%rax), %ebp
	orl	$4, %ebp
	movl	%ebp, _ZSt4cout+24(%rax)
	movl	$_ZSt4cout, %edi
	movl	$.L.str.13, %esi
	movl	$31, %edx
	callq	_ZSt16__ostream_insertIcSt11char_traitsIcEERSt13basic_ostreamIT_T0_ES6_PKS3_l
	movl	$_ZSt4cout, %edi
	movsd	8(%rsp), %xmm0                  # 8-byte Reload
                                        # xmm0 = mem[0],zero
	callq	_ZNSo9_M_insertIdEERSoT_
	movq	%rax, %rbx
	movl	$.L.str.4, %esi
	movl	$9, %edx
	movq	%rax, %rdi
	callq	_ZSt16__ostream_insertIcSt11char_traitsIcEERSt13basic_ostreamIT_T0_ES6_PKS3_l
	movq	(%rbx), %rax
	movq	-24(%rax), %rdi
	addq	%rbx, %rdi
	movl	$10, %esi
	callq	_ZNKSt9basic_iosIcSt11char_traitsIcEE5widenEc
	movsbl	%al, %esi
	movq	%rbx, %rdi
	callq	_ZNSo3putEc
	movq	%rax, %rdi
	callq	_ZNSo5flushEv
	xorl	%eax, %eax
	addq	$40, %rsp
	.cfi_def_cfa_offset 56
	popq	%rbx
	.cfi_def_cfa_offset 48
	popq	%r12
	.cfi_def_cfa_offset 40
	popq	%r13
	.cfi_def_cfa_offset 32
	popq	%r14
	.cfi_def_cfa_offset 24
	popq	%r15
	.cfi_def_cfa_offset 16
	popq	%rbp
	.cfi_def_cfa_offset 8
	retq
.LBB4_14:
	.cfi_def_cfa_offset 96
	movl	%eax, %ebp
	movl	$_ZSt4cerr, %edi
	movl	$.L.str.14, %esi
	callq	_ZStlsISt11char_traitsIcEERSt13basic_ostreamIcT_ES5_PKc
	movq	%rax, %rbx
	movl	%ebp, %edi
	callq	hipGetErrorString
	movq	%rbx, %rdi
	movq	%rax, %rsi
	callq	_ZStlsISt11char_traitsIcEERSt13basic_ostreamIcT_ES5_PKc
	movl	$.L.str.15, %esi
	movq	%rax, %rdi
	callq	_ZStlsISt11char_traitsIcEERSt13basic_ostreamIcT_ES5_PKc
	movl	$.L.str, %esi
	movq	%rax, %rdi
	callq	_ZStlsISt11char_traitsIcEERSt13basic_ostreamIcT_ES5_PKc
	movl	$.L.str.15, %esi
	movq	%rax, %rdi
	callq	_ZStlsISt11char_traitsIcEERSt13basic_ostreamIcT_ES5_PKc
	movq	%rax, %rdi
	movl	$163, %esi
.LBB4_16:
	callq	_ZNSolsEi
.LBB4_17:
	movq	%rax, %rdi
	callq	_ZSt4endlIcSt11char_traitsIcEERSt13basic_ostreamIT_T0_ES6_
	movl	$1, %edi
	callq	exit
.LBB4_15:
	movl	%eax, %ebp
	movl	$_ZSt4cerr, %edi
	movl	$.L.str.14, %esi
	callq	_ZStlsISt11char_traitsIcEERSt13basic_ostreamIcT_ES5_PKc
	movq	%rax, %rbx
	movl	%ebp, %edi
	callq	hipGetErrorString
	movq	%rbx, %rdi
	movq	%rax, %rsi
	callq	_ZStlsISt11char_traitsIcEERSt13basic_ostreamIcT_ES5_PKc
	movl	$.L.str.15, %esi
	movq	%rax, %rdi
	callq	_ZStlsISt11char_traitsIcEERSt13basic_ostreamIcT_ES5_PKc
	movl	$.L.str, %esi
	movq	%rax, %rdi
	callq	_ZStlsISt11char_traitsIcEERSt13basic_ostreamIcT_ES5_PKc
	movl	$.L.str.15, %esi
	movq	%rax, %rdi
	callq	_ZStlsISt11char_traitsIcEERSt13basic_ostreamIcT_ES5_PKc
	movq	%rax, %rdi
	movl	$138, %esi
	jmp	.LBB4_16
.LBB4_1:
	movl	$_ZSt4cerr, %edi
	movl	$.L.str.1, %esi
	callq	_ZStlsISt11char_traitsIcEERSt13basic_ostreamIcT_ES5_PKc
	movq	(%r14), %rsi
	movq	%rax, %rdi
	callq	_ZStlsISt11char_traitsIcEERSt13basic_ostreamIcT_ES5_PKc
	movl	$.L.str.2, %esi
	movq	%rax, %rdi
	callq	_ZStlsISt11char_traitsIcEERSt13basic_ostreamIcT_ES5_PKc
	jmp	.LBB4_17
.Lfunc_end4:
	.size	main, .Lfunc_end4-main
	.cfi_endproc
                                        # -- End function
	.type	.L.str,@object                  # @.str
	.section	.rodata.str1.1,"aMS",@progbits,1
.L.str:
	.asciz	"/home/ubuntu/Datasets/stackv2/train-structured-repos-hip-sm89/XapaJIaMnu/gpuPeerAccessTester/master/memory_tester.hip"
	.size	.L.str, 118

	.type	.L.str.1,@object                # @.str.1
.L.str.1:
	.asciz	"Usage: "
	.size	.L.str.1, 8

	.type	.L.str.2,@object                # @.str.2
.L.str.2:
	.asciz	" num gpus total_model_memory"
	.size	.L.str.2, 29

	.type	.L.str.3,@object                # @.str.3
.L.str.3:
	.asciz	"Other to this took: "
	.size	.L.str.3, 21

	.type	.L.str.4,@object                # @.str.4
.L.str.4:
	.asciz	" seconds."
	.size	.L.str.4, 10

	.type	.L.str.5,@object                # @.str.5
.L.str.5:
	.asciz	"This to other took: "
	.size	.L.str.5, 21

	.type	.L.str.6,@object                # @.str.6
.L.str.6:
	.asciz	"Other to this Async took: "
	.size	.L.str.6, 27

	.type	.L.str.7,@object                # @.str.7
.L.str.7:
	.asciz	"This to other Async took: "
	.size	.L.str.7, 27

	.type	.L.str.8,@object                # @.str.8
.L.str.8:
	.asciz	"Peer access unavailable between devices: "
	.size	.L.str.8, 42

	.type	.L.str.9,@object                # @.str.9
.L.str.9:
	.asciz	" and "
	.size	.L.str.9, 6

	.type	.L.str.10,@object               # @.str.10
.L.str.10:
	.asciz	"Peer other to this took: "
	.size	.L.str.10, 26

	.type	.L.str.11,@object               # @.str.11
.L.str.11:
	.asciz	"Peer this to other took: "
	.size	.L.str.11, 26

	.type	.L.str.12,@object               # @.str.12
.L.str.12:
	.asciz	"Peer other to this Async took: "
	.size	.L.str.12, 32

	.type	.L.str.13,@object               # @.str.13
.L.str.13:
	.asciz	"Peer this to other Async took: "
	.size	.L.str.13, 32

	.type	.L.str.14,@object               # @.str.14
.L.str.14:
	.asciz	"GPUassert: "
	.size	.L.str.14, 12

	.type	.L.str.15,@object               # @.str.15
.L.str.15:
	.asciz	" "
	.size	.L.str.15, 2

	.type	__hip_cuid_,@object             # @__hip_cuid_
	.bss
	.globl	__hip_cuid_
__hip_cuid_:
	.byte	0                               # 0x0
	.size	__hip_cuid_, 1

	.ident	"AMD clang version 18.0.0git (https://github.com/RadeonOpenCompute/llvm-project roc-6.3.2 25012 e5bf7e55c91490b07c49d8960fa7983d864936c4)"
	.section	".note.GNU-stack","",@progbits
	.addrsig
	.addrsig_sym _ZSt4cerr
	.addrsig_sym _ZSt4cout
	.addrsig_sym __hip_cuid_

	.text
	.p2alignl 7, 3214868480
	.fill 96, 4, 3214868480
	.type	__hip_cuid_,@object             ; @__hip_cuid_
	.section	.bss,"aw",@nobits
	.globl	__hip_cuid_
__hip_cuid_:
	.byte	0                               ; 0x0
	.size	__hip_cuid_, 1

	.ident	"AMD clang version 18.0.0git (https://github.com/RadeonOpenCompute/llvm-project roc-6.3.2 25012 e5bf7e55c91490b07c49d8960fa7983d864936c4)"
	.section	".note.GNU-stack","",@progbits
	.addrsig
	.addrsig_sym __hip_cuid_
	.amdgcn_target "amdgcn-amd-amdhsa--gfx1100"
	.amdgpu_metadata
---
amdhsa.kernels:  []
amdhsa.target:   amdgcn-amd-amdhsa--gfx1100
amdhsa.version:
  - 1
  - 2
...

	.end_amdgpu_metadata
