#include <cuda_runtime.h>
#include <stdlib.h>
#include <stdio.h>

#ifndef USE_MALLOC_HOST
#define USE_MALLOC_HOST
#endif
#undef USE_MALLOC_HOST

#define CUDA_SAFE_CALL(err)  __cudaSafeCall(err,__FILE__,__LINE__)

inline void __cudaSafeCall(cudaError_t err,const char *file, const int line) {
  if(cudaSuccess != err) {
    printf("%s(%i) : cudaSafeCall() Runtime API error : %s.\n",
           file, line, cudaGetErrorString(err) );
    exit(-1);
  }
}

int main(int argc, char * argv[]) {
  int numDevs=0, i, j, N, nBytes;
  cudaError_t err;
  cudaDeviceProp prop;
  cudaEvent_t start, stop;
  float *x_cpu, *y_cpu, *x_gpu;
  float dt,totalDtTo=0.0,totalDtFrom=0.0;

  /* create events */
  CUDA_SAFE_CALL(cudaEventCreate(&start));
  CUDA_SAFE_CALL(cudaEventCreate(&stop));

  N = atoi(argv[1]);
  nBytes = N*sizeof(float);
#ifdef USE_MALLOC_HOST
  CUDA_SAFE_CALL(cudaMallocHost((void**)&x_cpu,nBytes));
  CUDA_SAFE_CALL(cudaMallocHost((void**)&y_cpu,nBytes));
#else
  x_cpu = (float *) malloc(nBytes);
  y_cpu = (float *) malloc(nBytes);
#endif

  for (i=0; i<N; ++i) {
    x_cpu[i] = 1.0*i;
  }

  CUDA_SAFE_CALL(cudaGetDeviceCount(&numDevs));
  printf("Number of CUDA Devices = %d\n",numDevs);
  printf("===========================\n");

  for (i=0; i<numDevs; ++i) {
    CUDA_SAFE_CALL(cudaSetDevice(i));
    CUDA_SAFE_CALL(cudaMalloc((void**)&x_gpu,nBytes));
#ifndef USE_MALLOC_HOST
    CUDA_SAFE_CALL(cudaHostRegister(x_cpu, nBytes, cudaHostRegisterMapped));
    CUDA_SAFE_CALL(cudaHostRegister(y_cpu, nBytes, cudaHostRegisterMapped));
#endif

    CUDA_SAFE_CALL(cudaGetDeviceProperties(&prop,i));
    printf("Device %d has name %s with compute capability %d.%d canMapHostMemory=%d\n",i,prop.name,prop.major,prop.minor,prop.canMapHostMemory);
    printf("                           global memory = %1.5g\n",1.0*prop.totalGlobalMem/(1024*1024*1024));

    dt=0.0;
    CUDA_SAFE_CALL(cudaEventRecord(start, 0));
    for (j=0; j<100; ++j) {
      CUDA_SAFE_CALL(cudaMemcpy(x_gpu, x_cpu, nBytes, cudaMemcpyHostToDevice));
    }
    CUDA_SAFE_CALL(cudaEventRecord(stop, 0));
    CUDA_SAFE_CALL(cudaEventSynchronize(stop));
    CUDA_SAFE_CALL(cudaEventElapsedTime(&dt,start,stop));
    totalDtTo+=dt;

    dt=0.0;
    CUDA_SAFE_CALL(cudaEventRecord(start, 0));
    for (j=0; j<100; ++j) {
      CUDA_SAFE_CALL(cudaMemcpy(y_cpu, x_gpu, nBytes, cudaMemcpyDeviceToHost));
    }
    CUDA_SAFE_CALL(cudaEventRecord(stop, 0));
    CUDA_SAFE_CALL(cudaEventSynchronize(stop));
    CUDA_SAFE_CALL(cudaEventElapsedTime(&dt,start,stop));
    totalDtFrom+=dt;
    
    totalDtTo*=.001;
    totalDtFrom*=.001;

    printf("HostToDevice PCI Express BW=%g GB/s\n",100.0*nBytes/(1024*1024*1024)/totalDtTo);
    printf("DeviceToHost PCI Express BW=%g GB/s\n",100.0*nBytes/(1024*1024*1024)/totalDtFrom);

    totalDtTo=0.0;
    totalDtFrom=0.0;
#ifndef USE_MALLOC_HOST
    CUDA_SAFE_CALL(cudaHostUnregister(x_cpu));
    CUDA_SAFE_CALL(cudaHostUnregister(y_cpu));
#endif
    CUDA_SAFE_CALL(cudaFree(x_gpu));
  }

#ifndef USE_MALLOC_HOST
  free(x_cpu);
  free(y_cpu);
#else
  CUDA_SAFE_CALL(cudaFreeHost(x_cpu));
  CUDA_SAFE_CALL(cudaFreeHost(y_cpu));
#endif

  /* destroy events */
  CUDA_SAFE_CALL(cudaEventDestroy(start));
  CUDA_SAFE_CALL(cudaEventDestroy(stop));
  return 0;
}

	.file	"tmpxft_002b68ca_00000000-6_test.cudafe1.cpp"
	.text
#APP

#NO_APP
	.type	_ZL26__cudaUnregisterBinaryUtilv, @function
_ZL26__cudaUnregisterBinaryUtilv:
.LFB2031:
	.cfi_startproc
	endbr64
	movq	_ZL20__cudaFatCubinHandle(%rip), %rdi
	jmp	__cudaUnregisterFatBinary@PLT
	.cfi_endproc
.LFE2031:
	.size	_ZL26__cudaUnregisterBinaryUtilv, .-_ZL26__cudaUnregisterBinaryUtilv
	.section	.rodata._Z14__cudaSafeCall9cudaErrorPKci.str1.1,"aMS",@progbits,1
.LC0:
	.string	"%s(%i) : cudaSafeCall() Runtime API error : %s.\n"
	.section	.text._Z14__cudaSafeCall9cudaErrorPKci,"axG",@progbits,_Z14__cudaSafeCall9cudaErrorPKci,comdat
	.weak	_Z14__cudaSafeCall9cudaErrorPKci
	.type	_Z14__cudaSafeCall9cudaErrorPKci, @function
_Z14__cudaSafeCall9cudaErrorPKci:
.LFB2027:
	.cfi_startproc
	endbr64
	testl	%edi, %edi
	je	.L2
	pushq	%rbp
	.cfi_def_cfa_offset 16
	.cfi_offset 6, -16
	movl	%edx, %ebp
	pushq	%rbx
	.cfi_def_cfa_offset 24
	.cfi_offset 3, -24
	movq	%rsi, %rbx
	pushq	%rax
	.cfi_def_cfa_offset 32
	call	cudaGetErrorString@PLT
	movl	$2, %edi
	movl	%ebp, %ecx
	movq	%rbx, %rdx
	movq	%rax, %r8
	leaq	.LC0(%rip), %rsi
	xorl	%eax, %eax
	call	__printf_chk@PLT
	orl	$-1, %edi
	call	exit@PLT
.L2:
	.cfi_def_cfa_offset 8
	.cfi_restore 3
	.cfi_restore 6
	ret
	.cfi_endproc
.LFE2027:
	.size	_Z14__cudaSafeCall9cudaErrorPKci, .-_Z14__cudaSafeCall9cudaErrorPKci
	.section	.rodata.str1.1,"aMS",@progbits,1
.LC1:
	.string	"/home/ubuntu/Datasets/stackv2/train-structured/nricklin/ubuntu-gpu-test/master/test.cu"
.LC2:
	.string	"Number of CUDA Devices = %d\n"
.LC3:
	.string	"===========================\n"
.LC4:
	.string	"Device %d has name %s with compute capability %d.%d canMapHostMemory=%d\n"
.LC6:
	.string	"                           global memory = %1.5g\n"
.LC10:
	.string	"HostToDevice PCI Express BW=%g GB/s\n"
.LC11:
	.string	"DeviceToHost PCI Express BW=%g GB/s\n"
	.section	.text.startup,"ax",@progbits
	.globl	main
	.type	main, @function
main:
.LFB2028:
	.cfi_startproc
	endbr64
	pushq	%r15
	.cfi_def_cfa_offset 16
	.cfi_offset 15, -16
	pushq	%r14
	.cfi_def_cfa_offset 24
	.cfi_offset 14, -24
	pushq	%r13
	.cfi_def_cfa_offset 32
	.cfi_offset 13, -32
	pushq	%r12
	.cfi_def_cfa_offset 40
	.cfi_offset 12, -40
	pushq	%rbp
	.cfi_def_cfa_offset 48
	.cfi_offset 6, -48
	pushq	%rbx
	.cfi_def_cfa_offset 56
	.cfi_offset 3, -56
	movq	%rsi, %rbx
	xorl	%esi, %esi
	subq	$1112, %rsp
	.cfi_def_cfa_offset 1168
	movq	%fs:40, %rax
	movq	%rax, 1096(%rsp)
	xorl	%eax, %eax
	leaq	40(%rsp), %rdi
	movl	%esi, 32(%rsp)
	call	cudaEventCreate@PLT
	movl	$29, %edx
	leaq	.LC1(%rip), %rsi
	movl	%eax, %edi
	call	_Z14__cudaSafeCall9cudaErrorPKci
	leaq	48(%rsp), %rdi
	call	cudaEventCreate@PLT
	movl	$30, %edx
	leaq	.LC1(%rip), %rsi
	movl	%eax, %edi
	call	_Z14__cudaSafeCall9cudaErrorPKci
	movq	8(%rbx), %rdi
	call	atoi@PLT
	leal	0(,%rax,4), %r12d
	movl	%eax, %ebp
	movslq	%r12d, %r15
	movq	%r15, %rdi
	call	malloc@PLT
	movq	%r15, %rdi
	movq	%rax, %rbx
	call	malloc@PLT
	xorl	%edx, %edx
	movq	%rax, 8(%rsp)
.L9:
	cmpl	%edx, %ebp
	jle	.L21
	cvtsi2ssl	%edx, %xmm0
	movss	%xmm0, (%rbx,%rdx,4)
	incq	%rdx
	jmp	.L9
.L21:
	leaq	32(%rsp), %rdi
	xorl	%ebp, %ebp
	leaq	.LC4(%rip), %r14
	call	cudaGetDeviceCount@PLT
	movl	$46, %edx
	leaq	.LC1(%rip), %rsi
	movl	%eax, %edi
	call	_Z14__cudaSafeCall9cudaErrorPKci
	movl	32(%rsp), %edx
	movl	$2, %edi
	xorl	%eax, %eax
	leaq	.LC2(%rip), %rsi
	call	__printf_chk@PLT
	leaq	.LC3(%rip), %rsi
	movl	$2, %edi
	xorl	%eax, %eax
	call	__printf_chk@PLT
.L11:
	cmpl	%ebp, 32(%rsp)
	jle	.L22
	movl	%ebp, %edi
	leaq	64(%rsp), %r13
	call	cudaSetDevice@PLT
	movl	$51, %edx
	leaq	.LC1(%rip), %rsi
	movl	%eax, %edi
	call	_Z14__cudaSafeCall9cudaErrorPKci
	leaq	56(%rsp), %rdi
	movq	%r15, %rsi
	call	cudaMalloc@PLT
	movl	$52, %edx
	leaq	.LC1(%rip), %rsi
	movl	%eax, %edi
	call	_Z14__cudaSafeCall9cudaErrorPKci
	movl	$2, %edx
	movq	%r15, %rsi
	movq	%rbx, %rdi
	call	cudaHostRegister@PLT
	movl	$54, %edx
	leaq	.LC1(%rip), %rsi
	movl	%eax, %edi
	call	_Z14__cudaSafeCall9cudaErrorPKci
	movq	8(%rsp), %rdi
	movl	$2, %edx
	movq	%r15, %rsi
	call	cudaHostRegister@PLT
	movl	$55, %edx
	leaq	.LC1(%rip), %rsi
	movl	%eax, %edi
	call	_Z14__cudaSafeCall9cudaErrorPKci
	movl	%ebp, %esi
	movq	%r13, %rdi
	call	cudaGetDeviceProperties_v2@PLT
	movl	$58, %edx
	leaq	.LC1(%rip), %rsi
	movl	%eax, %edi
	call	_Z14__cudaSafeCall9cudaErrorPKci
	movq	%r13, %rcx
	movl	%ebp, %edx
	movq	%r14, %rsi
	pushq	%rax
	.cfi_def_cfa_offset 1176
	movl	472(%rsp), %eax
	movl	$2, %edi
	pushq	%rax
	.cfi_def_cfa_offset 1184
	movl	444(%rsp), %r9d
	xorl	%eax, %eax
	movl	440(%rsp), %r8d
	call	__printf_chk@PLT
	popq	%rdx
	.cfi_def_cfa_offset 1176
	popq	%rcx
	.cfi_def_cfa_offset 1168
	movq	352(%rsp), %rax
	testq	%rax, %rax
	js	.L12
	cvtsi2sdq	%rax, %xmm0
	jmp	.L13
.L12:
	movq	%rax, %rdx
	andl	$1, %eax
	shrq	%rdx
	orq	%rax, %rdx
	cvtsi2sdq	%rdx, %xmm0
	addsd	%xmm0, %xmm0
.L13:
	mulsd	.LC5(%rip), %xmm0
	leaq	.LC6(%rip), %rsi
	movl	$2, %edi
	movb	$1, %al
	movl	$100, %r13d
	call	__printf_chk@PLT
	movq	40(%rsp), %rdi
	xorl	%esi, %esi
	movl	$0x00000000, 36(%rsp)
	call	cudaEventRecord@PLT
	movl	$63, %edx
	leaq	.LC1(%rip), %rsi
	movl	%eax, %edi
	call	_Z14__cudaSafeCall9cudaErrorPKci
.L14:
	movq	56(%rsp), %rdi
	movq	%r15, %rdx
	movq	%rbx, %rsi
	movl	$1, %ecx
	call	cudaMemcpy@PLT
	movl	$65, %edx
	leaq	.LC1(%rip), %rsi
	movl	%eax, %edi
	call	_Z14__cudaSafeCall9cudaErrorPKci
	decl	%r13d
	jne	.L14
	movq	48(%rsp), %rdi
	xorl	%esi, %esi
	movl	$100, %r13d
	call	cudaEventRecord@PLT
	movl	$67, %edx
	leaq	.LC1(%rip), %rsi
	movl	%eax, %edi
	call	_Z14__cudaSafeCall9cudaErrorPKci
	movq	48(%rsp), %rdi
	call	cudaEventSynchronize@PLT
	movl	$68, %edx
	leaq	.LC1(%rip), %rsi
	movl	%eax, %edi
	call	_Z14__cudaSafeCall9cudaErrorPKci
	leaq	36(%rsp), %rax
	movq	48(%rsp), %rdx
	movq	40(%rsp), %rsi
	movq	%rax, %rdi
	movq	%rax, 16(%rsp)
	call	cudaEventElapsedTime@PLT
	movl	$69, %edx
	leaq	.LC1(%rip), %rsi
	movl	%eax, %edi
	call	_Z14__cudaSafeCall9cudaErrorPKci
	xorps	%xmm0, %xmm0
	addss	36(%rsp), %xmm0
	xorl	%esi, %esi
	movq	40(%rsp), %rdi
	movl	$0x00000000, 36(%rsp)
	movss	%xmm0, 28(%rsp)
	call	cudaEventRecord@PLT
	movl	$73, %edx
	leaq	.LC1(%rip), %rsi
	movl	%eax, %edi
	call	_Z14__cudaSafeCall9cudaErrorPKci
.L15:
	movq	56(%rsp), %rsi
	movq	8(%rsp), %rdi
	movq	%r15, %rdx
	movl	$2, %ecx
	call	cudaMemcpy@PLT
	movl	$75, %edx
	leaq	.LC1(%rip), %rsi
	movl	%eax, %edi
	call	_Z14__cudaSafeCall9cudaErrorPKci
	decl	%r13d
	jne	.L15
	movq	48(%rsp), %rdi
	xorl	%esi, %esi
	incl	%ebp
	call	cudaEventRecord@PLT
	movl	$77, %edx
	leaq	.LC1(%rip), %rsi
	movl	%eax, %edi
	call	_Z14__cudaSafeCall9cudaErrorPKci
	movq	48(%rsp), %rdi
	call	cudaEventSynchronize@PLT
	movl	$78, %edx
	leaq	.LC1(%rip), %rsi
	movl	%eax, %edi
	call	_Z14__cudaSafeCall9cudaErrorPKci
	movq	48(%rsp), %rdx
	movq	40(%rsp), %rsi
	movq	16(%rsp), %rdi
	call	cudaEventElapsedTime@PLT
	movl	$79, %edx
	leaq	.LC1(%rip), %rsi
	movl	%eax, %edi
	call	_Z14__cudaSafeCall9cudaErrorPKci
	cvtsi2sdl	%r12d, %xmm3
	xorps	%xmm1, %xmm1
	movb	$1, %al
	movsd	.LC8(%rip), %xmm0
	mulsd	.LC9(%rip), %xmm3
	cvtss2sd	28(%rsp), %xmm2
	leaq	.LC10(%rip), %rsi
	mulsd	.LC5(%rip), %xmm3
	addss	36(%rsp), %xmm1
	movl	$2, %edi
	mulsd	%xmm0, %xmm2
	cvtss2sd	%xmm1, %xmm1
	mulsd	%xmm0, %xmm1
	movaps	%xmm3, %xmm0
	movsd	%xmm3, 16(%rsp)
	cvtsd2ss	%xmm2, %xmm2
	cvtss2sd	%xmm2, %xmm2
	divsd	%xmm2, %xmm0
	cvtsd2ss	%xmm1, %xmm1
	movss	%xmm1, 28(%rsp)
	call	__printf_chk@PLT
	movss	28(%rsp), %xmm1
	movsd	16(%rsp), %xmm3
	movb	$1, %al
	leaq	.LC11(%rip), %rsi
	movl	$2, %edi
	cvtss2sd	%xmm1, %xmm1
	divsd	%xmm1, %xmm3
	movaps	%xmm3, %xmm0
	call	__printf_chk@PLT
	movq	%rbx, %rdi
	call	cudaHostUnregister@PLT
	movl	$91, %edx
	leaq	.LC1(%rip), %rsi
	movl	%eax, %edi
	call	_Z14__cudaSafeCall9cudaErrorPKci
	movq	8(%rsp), %rdi
	call	cudaHostUnregister@PLT
	movl	$92, %edx
	leaq	.LC1(%rip), %rsi
	movl	%eax, %edi
	call	_Z14__cudaSafeCall9cudaErrorPKci
	movq	56(%rsp), %rdi
	call	cudaFree@PLT
	movl	$94, %edx
	leaq	.LC1(%rip), %rsi
	movl	%eax, %edi
	call	_Z14__cudaSafeCall9cudaErrorPKci
	jmp	.L11
.L22:
	movq	%rbx, %rdi
	call	free@PLT
	movq	8(%rsp), %rdi
	call	free@PLT
	movq	40(%rsp), %rdi
	call	cudaEventDestroy@PLT
	movl	$106, %edx
	leaq	.LC1(%rip), %rsi
	movl	%eax, %edi
	call	_Z14__cudaSafeCall9cudaErrorPKci
	movq	48(%rsp), %rdi
	call	cudaEventDestroy@PLT
	movl	$107, %edx
	leaq	.LC1(%rip), %rsi
	movl	%eax, %edi
	call	_Z14__cudaSafeCall9cudaErrorPKci
	movq	1096(%rsp), %rax
	subq	%fs:40, %rax
	je	.L17
	call	__stack_chk_fail@PLT
.L17:
	addq	$1112, %rsp
	.cfi_def_cfa_offset 56
	xorl	%eax, %eax
	popq	%rbx
	.cfi_def_cfa_offset 48
	popq	%rbp
	.cfi_def_cfa_offset 40
	popq	%r12
	.cfi_def_cfa_offset 32
	popq	%r13
	.cfi_def_cfa_offset 24
	popq	%r14
	.cfi_def_cfa_offset 16
	popq	%r15
	.cfi_def_cfa_offset 8
	ret
	.cfi_endproc
.LFE2028:
	.size	main, .-main
	.type	_ZL24__sti____cudaRegisterAllv, @function
_ZL24__sti____cudaRegisterAllv:
.LFB2054:
	.cfi_startproc
	endbr64
	pushq	%rax
	.cfi_def_cfa_offset 16
	leaq	_ZL15__fatDeviceText(%rip), %rdi
	call	__cudaRegisterFatBinary@PLT
	movq	%rax, %rdi
	movq	%rax, _ZL20__cudaFatCubinHandle(%rip)
	call	__cudaRegisterFatBinaryEnd@PLT
	leaq	_ZL26__cudaUnregisterBinaryUtilv(%rip), %rdi
	popq	%rdx
	.cfi_def_cfa_offset 8
	jmp	atexit@PLT
	.cfi_endproc
.LFE2054:
	.size	_ZL24__sti____cudaRegisterAllv, .-_ZL24__sti____cudaRegisterAllv
	.section	.init_array,"aw"
	.align 8
	.quad	_ZL24__sti____cudaRegisterAllv
	.section	.nvFatBinSegment,"aw"
	.align 8
	.type	_ZL15__fatDeviceText, @object
	.size	_ZL15__fatDeviceText, 24
_ZL15__fatDeviceText:
	.long	1180844977
	.long	1
	.quad	fatbinData
	.quad	0
	.local	_ZL20__cudaFatCubinHandle
	.comm	_ZL20__cudaFatCubinHandle,8,8
	.section	.rodata.cst8,"aM",@progbits,8
	.align 8
.LC5:
	.long	0
	.long	1041235968
	.align 8
.LC8:
	.long	-755914244
	.long	1062232653
	.align 8
.LC9:
	.long	0
	.long	1079574528
	.ident	"GCC: (Ubuntu 13.3.0-6ubuntu2~24.04) 13.3.0"
	.section	.note.GNU-stack,"",@progbits
	.section	.note.gnu.property,"a"
	.align 8
	.long	1f - 0f
	.long	4f - 1f
	.long	5
0:
	.string	"GNU"
1:
	.align 8
	.long	0xc0000002
	.long	3f - 2f
2:
	.long	0x3
3:
	.align 8
4:

#include <hip/hip_runtime.h>
#include <stdlib.h>
#include <stdio.h>

#ifndef USE_MALLOC_HOST
#define USE_MALLOC_HOST
#endif
#undef USE_MALLOC_HOST

#define CUDA_SAFE_CALL(err)  __cudaSafeCall(err,__FILE__,__LINE__)

inline void __cudaSafeCall(hipError_t err,const char *file, const int line) {
  if(hipSuccess != err) {
    printf("%s(%i) : cudaSafeCall() Runtime API error : %s.\n",
           file, line, hipGetErrorString(err) );
    exit(-1);
  }
}

int main(int argc, char * argv[]) {
  int numDevs=0, i, j, N, nBytes;
  hipError_t err;
  hipDeviceProp_t prop;
  hipEvent_t start, stop;
  float *x_cpu, *y_cpu, *x_gpu;
  float dt,totalDtTo=0.0,totalDtFrom=0.0;

  /* create events */
  CUDA_SAFE_CALL(hipEventCreate(&start));
  CUDA_SAFE_CALL(hipEventCreate(&stop));

  N = atoi(argv[1]);
  nBytes = N*sizeof(float);
#ifdef USE_MALLOC_HOST
  CUDA_SAFE_CALL(hipHostMalloc((void**)&x_cpu,nBytes, hipHostMallocDefault));
  CUDA_SAFE_CALL(hipHostMalloc((void**)&y_cpu,nBytes, hipHostMallocDefault));
#else
  x_cpu = (float *) malloc(nBytes);
  y_cpu = (float *) malloc(nBytes);
#endif

  for (i=0; i<N; ++i) {
    x_cpu[i] = 1.0*i;
  }

  CUDA_SAFE_CALL(hipGetDeviceCount(&numDevs));
  printf("Number of CUDA Devices = %d\n",numDevs);
  printf("===========================\n");

  for (i=0; i<numDevs; ++i) {
    CUDA_SAFE_CALL(hipSetDevice(i));
    CUDA_SAFE_CALL(hipMalloc((void**)&x_gpu,nBytes));
#ifndef USE_MALLOC_HOST
    CUDA_SAFE_CALL(hipHostRegister(x_cpu, nBytes, hipHostRegisterMapped));
    CUDA_SAFE_CALL(hipHostRegister(y_cpu, nBytes, hipHostRegisterMapped));
#endif

    CUDA_SAFE_CALL(hipGetDeviceProperties(&prop,i));
    printf("Device %d has name %s with compute capability %d.%d canMapHostMemory=%d\n",i,prop.name,prop.major,prop.minor,prop.canMapHostMemory);
    printf("                           global memory = %1.5g\n",1.0*prop.totalGlobalMem/(1024*1024*1024));

    dt=0.0;
    CUDA_SAFE_CALL(hipEventRecord(start, 0));
    for (j=0; j<100; ++j) {
      CUDA_SAFE_CALL(hipMemcpy(x_gpu, x_cpu, nBytes, hipMemcpyHostToDevice));
    }
    CUDA_SAFE_CALL(hipEventRecord(stop, 0));
    CUDA_SAFE_CALL(hipEventSynchronize(stop));
    CUDA_SAFE_CALL(hipEventElapsedTime(&dt,start,stop));
    totalDtTo+=dt;

    dt=0.0;
    CUDA_SAFE_CALL(hipEventRecord(start, 0));
    for (j=0; j<100; ++j) {
      CUDA_SAFE_CALL(hipMemcpy(y_cpu, x_gpu, nBytes, hipMemcpyDeviceToHost));
    }
    CUDA_SAFE_CALL(hipEventRecord(stop, 0));
    CUDA_SAFE_CALL(hipEventSynchronize(stop));
    CUDA_SAFE_CALL(hipEventElapsedTime(&dt,start,stop));
    totalDtFrom+=dt;
    
    totalDtTo*=.001;
    totalDtFrom*=.001;

    printf("HostToDevice PCI Express BW=%g GB/s\n",100.0*nBytes/(1024*1024*1024)/totalDtTo);
    printf("DeviceToHost PCI Express BW=%g GB/s\n",100.0*nBytes/(1024*1024*1024)/totalDtFrom);

    totalDtTo=0.0;
    totalDtFrom=0.0;
#ifndef USE_MALLOC_HOST
    CUDA_SAFE_CALL(hipHostUnregister(x_cpu));
    CUDA_SAFE_CALL(hipHostUnregister(y_cpu));
#endif
    CUDA_SAFE_CALL(hipFree(x_gpu));
  }

#ifndef USE_MALLOC_HOST
  free(x_cpu);
  free(y_cpu);
#else
  CUDA_SAFE_CALL(hipHostFree(x_cpu));
  CUDA_SAFE_CALL(hipHostFree(y_cpu));
#endif

  /* destroy events */
  CUDA_SAFE_CALL(hipEventDestroy(start));
  CUDA_SAFE_CALL(hipEventDestroy(stop));
  return 0;
}

	.text
	.file	"test.hip"
	.section	.rodata.cst8,"aM",@progbits,8
	.p2align	3, 0x0                          # -- Begin function main
.LCPI0_0:
	.quad	0x4059000000000000              # double 100
.LCPI0_1:
	.quad	0x3e10000000000000              # double 9.3132257461547852E-10
.LCPI0_4:
	.quad	0x3f50624dd2f1a9fc              # double 0.001
	.section	.rodata.cst16,"aM",@progbits,16
	.p2align	4, 0x0
.LCPI0_2:
	.long	1127219200                      # 0x43300000
	.long	1160773632                      # 0x45300000
	.long	0                               # 0x0
	.long	0                               # 0x0
.LCPI0_3:
	.quad	0x4330000000000000              # double 4503599627370496
	.quad	0x4530000000000000              # double 1.9342813113834067E+25
	.section	.rodata.cst4,"aM",@progbits,4
	.p2align	2, 0x0
.LCPI0_5:
	.long	0x00000000                      # float 0
	.text
	.globl	main
	.type	main,@function
main:                                   # @main
	.cfi_startproc
# %bb.0:
	pushq	%rbp
	.cfi_def_cfa_offset 16
	pushq	%r15
	.cfi_def_cfa_offset 24
	pushq	%r14
	.cfi_def_cfa_offset 32
	pushq	%r13
	.cfi_def_cfa_offset 40
	pushq	%r12
	.cfi_def_cfa_offset 48
	pushq	%rbx
	.cfi_def_cfa_offset 56
	subq	$1528, %rsp                     # imm = 0x5F8
	.cfi_def_cfa_offset 1584
	.cfi_offset %rbx, -56
	.cfi_offset %r12, -48
	.cfi_offset %r13, -40
	.cfi_offset %r14, -32
	.cfi_offset %r15, -24
	.cfi_offset %rbp, -16
	movq	%rsi, %rbx
	movl	$0, 12(%rsp)
	leaq	32(%rsp), %rdi
	callq	hipEventCreate
	testl	%eax, %eax
	jne	.LBB0_1
# %bb.3:                                # %_Z14__cudaSafeCall10hipError_tPKci.exit
	leaq	16(%rsp), %rdi
	callq	hipEventCreate
	testl	%eax, %eax
	jne	.LBB0_4
# %bb.5:                                # %_Z14__cudaSafeCall10hipError_tPKci.exit44
	movq	8(%rbx), %rdi
	callq	atoi
	movl	%eax, %r12d
	leal	(,%r12,4), %ebp
	movslq	%ebp, %rbx
	movq	%rbx, %rdi
	callq	malloc
	movq	%rax, %r14
	movq	%rbx, %rdi
	callq	malloc
	movq	%rax, %r15
	testl	%r12d, %r12d
	jle	.LBB0_8
# %bb.6:                                # %.lr.ph.preheader
	movl	%r12d, %eax
	xorl	%ecx, %ecx
.LBB0_7:                                # %.lr.ph
                                        # =>This Inner Loop Header: Depth=1
	xorps	%xmm0, %xmm0
	cvtsi2ss	%ecx, %xmm0
	movss	%xmm0, (%r14,%rcx,4)
	incq	%rcx
	cmpq	%rcx, %rax
	jne	.LBB0_7
.LBB0_8:                                # %._crit_edge
	leaq	12(%rsp), %rdi
	callq	hipGetDeviceCount
	testl	%eax, %eax
	jne	.LBB0_9
# %bb.10:                               # %_Z14__cudaSafeCall10hipError_tPKci.exit46
	movl	12(%rsp), %esi
	movl	$.L.str.1, %edi
	xorl	%eax, %eax
	callq	printf
	movl	$.Lstr, %edi
	callq	puts@PLT
	cmpl	$0, 12(%rsp)
	jle	.LBB0_53
# %bb.11:                               # %.lr.ph244
	xorps	%xmm0, %xmm0
	cvtsi2sd	%ebp, %xmm0
	mulsd	.LCPI0_0(%rip), %xmm0
	mulsd	.LCPI0_1(%rip), %xmm0
	movsd	%xmm0, 48(%rsp)                 # 8-byte Spill
	leaq	56(%rsp), %r13
	xorl	%r12d, %r12d
.LBB0_13:                               # =>This Loop Header: Depth=1
                                        #     Child Loop BB0_27 Depth 2
                                        #     Child Loop BB0_39 Depth 2
	movl	%r12d, %edi
	callq	hipSetDevice
	testl	%eax, %eax
	jne	.LBB0_14
# %bb.15:                               # %_Z14__cudaSafeCall10hipError_tPKci.exit48
                                        #   in Loop: Header=BB0_13 Depth=1
	leaq	40(%rsp), %rdi
	movq	%rbx, %rsi
	callq	hipMalloc
	testl	%eax, %eax
	jne	.LBB0_16
# %bb.17:                               # %_Z14__cudaSafeCall10hipError_tPKci.exit50
                                        #   in Loop: Header=BB0_13 Depth=1
	movq	%r14, %rdi
	movq	%rbx, %rsi
	movl	$2, %edx
	callq	hipHostRegister
	testl	%eax, %eax
	jne	.LBB0_18
# %bb.19:                               # %_Z14__cudaSafeCall10hipError_tPKci.exit52
                                        #   in Loop: Header=BB0_13 Depth=1
	movq	%r15, %rdi
	movq	%rbx, %rsi
	movl	$2, %edx
	callq	hipHostRegister
	testl	%eax, %eax
	jne	.LBB0_20
# %bb.21:                               # %_Z14__cudaSafeCall10hipError_tPKci.exit54
                                        #   in Loop: Header=BB0_13 Depth=1
	movq	%r13, %rdi
	movl	%r12d, %esi
	callq	hipGetDevicePropertiesR0600
	testl	%eax, %eax
	jne	.LBB0_22
# %bb.23:                               # %_Z14__cudaSafeCall10hipError_tPKci.exit56
                                        #   in Loop: Header=BB0_13 Depth=1
	movl	416(%rsp), %ecx
	movl	420(%rsp), %r8d
	movl	456(%rsp), %r9d
	movl	$.L.str.3, %edi
	movl	%r12d, %esi
	movq	%r13, %rbp
	movq	%r13, %rdx
	xorl	%eax, %eax
	callq	printf
	movsd	344(%rsp), %xmm1                # xmm1 = mem[0],zero
	unpcklps	.LCPI0_2(%rip), %xmm1           # xmm1 = xmm1[0],mem[0],xmm1[1],mem[1]
	subpd	.LCPI0_3(%rip), %xmm1
	movapd	%xmm1, %xmm0
	unpckhpd	%xmm1, %xmm0                    # xmm0 = xmm0[1],xmm1[1]
	addsd	%xmm1, %xmm0
	mulsd	.LCPI0_1(%rip), %xmm0
	movl	$.L.str.4, %edi
	movb	$1, %al
	callq	printf
	movl	$0, 8(%rsp)
	movq	32(%rsp), %rdi
	xorl	%esi, %esi
	callq	hipEventRecord
	testl	%eax, %eax
	jne	.LBB0_25
# %bb.24:                               # %_Z14__cudaSafeCall10hipError_tPKci.exit60.preheader
                                        #   in Loop: Header=BB0_13 Depth=1
	movl	$100, %r13d
.LBB0_27:                               # %_Z14__cudaSafeCall10hipError_tPKci.exit60
                                        #   Parent Loop BB0_13 Depth=1
                                        # =>  This Inner Loop Header: Depth=2
	movq	40(%rsp), %rdi
	movq	%r14, %rsi
	movq	%rbx, %rdx
	movl	$1, %ecx
	callq	hipMemcpy
	testl	%eax, %eax
	jne	.LBB0_28
# %bb.26:                               # %_Z14__cudaSafeCall10hipError_tPKci.exit58
                                        #   in Loop: Header=BB0_27 Depth=2
	decl	%r13d
	jne	.LBB0_27
# %bb.29:                               #   in Loop: Header=BB0_13 Depth=1
	movq	16(%rsp), %rdi
	xorl	%esi, %esi
	callq	hipEventRecord
	testl	%eax, %eax
	jne	.LBB0_30
# %bb.31:                               # %_Z14__cudaSafeCall10hipError_tPKci.exit62
                                        #   in Loop: Header=BB0_13 Depth=1
	movq	16(%rsp), %rdi
	callq	hipEventSynchronize
	testl	%eax, %eax
	jne	.LBB0_32
# %bb.33:                               # %_Z14__cudaSafeCall10hipError_tPKci.exit64
                                        #   in Loop: Header=BB0_13 Depth=1
	movq	32(%rsp), %rsi
	movq	16(%rsp), %rdx
	leaq	8(%rsp), %rdi
	callq	hipEventElapsedTime
	testl	%eax, %eax
	jne	.LBB0_34
# %bb.35:                               # %_Z14__cudaSafeCall10hipError_tPKci.exit66
                                        #   in Loop: Header=BB0_13 Depth=1
	movss	8(%rsp), %xmm0                  # xmm0 = mem[0],zero,zero,zero
	addss	.LCPI0_5(%rip), %xmm0
	movss	%xmm0, 28(%rsp)                 # 4-byte Spill
	movl	$0, 8(%rsp)
	movq	32(%rsp), %rdi
	xorl	%esi, %esi
	callq	hipEventRecord
	testl	%eax, %eax
	jne	.LBB0_37
# %bb.36:                               # %_Z14__cudaSafeCall10hipError_tPKci.exit70.preheader
                                        #   in Loop: Header=BB0_13 Depth=1
	movl	$100, %r13d
.LBB0_39:                               # %_Z14__cudaSafeCall10hipError_tPKci.exit70
                                        #   Parent Loop BB0_13 Depth=1
                                        # =>  This Inner Loop Header: Depth=2
	movq	40(%rsp), %rsi
	movq	%r15, %rdi
	movq	%rbx, %rdx
	movl	$2, %ecx
	callq	hipMemcpy
	testl	%eax, %eax
	jne	.LBB0_40
# %bb.38:                               # %_Z14__cudaSafeCall10hipError_tPKci.exit68
                                        #   in Loop: Header=BB0_39 Depth=2
	decl	%r13d
	jne	.LBB0_39
# %bb.41:                               #   in Loop: Header=BB0_13 Depth=1
	movq	16(%rsp), %rdi
	xorl	%esi, %esi
	callq	hipEventRecord
	testl	%eax, %eax
	jne	.LBB0_42
# %bb.43:                               # %_Z14__cudaSafeCall10hipError_tPKci.exit72
                                        #   in Loop: Header=BB0_13 Depth=1
	movq	16(%rsp), %rdi
	callq	hipEventSynchronize
	testl	%eax, %eax
	jne	.LBB0_44
# %bb.45:                               # %_Z14__cudaSafeCall10hipError_tPKci.exit74
                                        #   in Loop: Header=BB0_13 Depth=1
	movq	%rbp, %r13
	movq	32(%rsp), %rsi
	movq	16(%rsp), %rdx
	leaq	8(%rsp), %rdi
	callq	hipEventElapsedTime
	testl	%eax, %eax
	jne	.LBB0_46
# %bb.47:                               # %_Z14__cudaSafeCall10hipError_tPKci.exit76
                                        #   in Loop: Header=BB0_13 Depth=1
	movss	8(%rsp), %xmm0                  # xmm0 = mem[0],zero,zero,zero
	addss	.LCPI0_5(%rip), %xmm0
	cvtss2sd	28(%rsp), %xmm1                 # 4-byte Folded Reload
	movsd	.LCPI0_4(%rip), %xmm2           # xmm2 = mem[0],zero
	mulsd	%xmm2, %xmm1
	cvtsd2ss	%xmm1, %xmm1
	cvtss2sd	%xmm0, %xmm0
	mulsd	%xmm2, %xmm0
	cvtsd2ss	%xmm0, %xmm0
	movss	%xmm0, 28(%rsp)                 # 4-byte Spill
	cvtss2sd	%xmm1, %xmm1
	movsd	48(%rsp), %xmm0                 # 8-byte Reload
                                        # xmm0 = mem[0],zero
	divsd	%xmm1, %xmm0
	movl	$.L.str.5, %edi
	movb	$1, %al
	callq	printf
	xorps	%xmm1, %xmm1
	cvtss2sd	28(%rsp), %xmm1                 # 4-byte Folded Reload
	movsd	48(%rsp), %xmm0                 # 8-byte Reload
                                        # xmm0 = mem[0],zero
	divsd	%xmm1, %xmm0
	movl	$.L.str.6, %edi
	movb	$1, %al
	callq	printf
	movq	%r14, %rdi
	callq	hipHostUnregister
	testl	%eax, %eax
	jne	.LBB0_48
# %bb.49:                               # %_Z14__cudaSafeCall10hipError_tPKci.exit78
                                        #   in Loop: Header=BB0_13 Depth=1
	movq	%r15, %rdi
	callq	hipHostUnregister
	testl	%eax, %eax
	jne	.LBB0_50
# %bb.51:                               # %_Z14__cudaSafeCall10hipError_tPKci.exit80
                                        #   in Loop: Header=BB0_13 Depth=1
	movq	40(%rsp), %rdi
	callq	hipFree
	testl	%eax, %eax
	jne	.LBB0_52
# %bb.12:                               #   in Loop: Header=BB0_13 Depth=1
	incl	%r12d
	cmpl	12(%rsp), %r12d
	jl	.LBB0_13
.LBB0_53:                               # %._crit_edge245
	movq	%r14, %rdi
	callq	free
	movq	%r15, %rdi
	callq	free
	movq	32(%rsp), %rdi
	callq	hipEventDestroy
	testl	%eax, %eax
	jne	.LBB0_54
# %bb.55:                               # %_Z14__cudaSafeCall10hipError_tPKci.exit84
	movq	16(%rsp), %rdi
	callq	hipEventDestroy
	testl	%eax, %eax
	jne	.LBB0_56
# %bb.57:                               # %_Z14__cudaSafeCall10hipError_tPKci.exit86
	xorl	%eax, %eax
	addq	$1528, %rsp                     # imm = 0x5F8
	.cfi_def_cfa_offset 56
	popq	%rbx
	.cfi_def_cfa_offset 48
	popq	%r12
	.cfi_def_cfa_offset 40
	popq	%r13
	.cfi_def_cfa_offset 32
	popq	%r14
	.cfi_def_cfa_offset 24
	popq	%r15
	.cfi_def_cfa_offset 16
	popq	%rbp
	.cfi_def_cfa_offset 8
	retq
.LBB0_28:
	.cfi_def_cfa_offset 1584
	movl	%eax, %edi
	callq	hipGetErrorString
	movl	$.L.str.7, %edi
	movl	$.L.str, %esi
	movl	$65, %edx
	jmp	.LBB0_2
.LBB0_40:
	movl	%eax, %edi
	callq	hipGetErrorString
	movl	$.L.str.7, %edi
	movl	$.L.str, %esi
	movl	$75, %edx
.LBB0_2:
	movq	%rax, %rcx
	xorl	%eax, %eax
	callq	printf
	movl	$-1, %edi
	callq	exit
.LBB0_25:
	movl	%eax, %edi
	callq	hipGetErrorString
	movl	$.L.str.7, %edi
	movl	$.L.str, %esi
	movl	$63, %edx
	jmp	.LBB0_2
.LBB0_22:
	movl	%eax, %edi
	callq	hipGetErrorString
	movl	$.L.str.7, %edi
	movl	$.L.str, %esi
	movl	$58, %edx
	jmp	.LBB0_2
.LBB0_20:
	movl	%eax, %edi
	callq	hipGetErrorString
	movl	$.L.str.7, %edi
	movl	$.L.str, %esi
	movl	$55, %edx
	jmp	.LBB0_2
.LBB0_18:
	movl	%eax, %edi
	callq	hipGetErrorString
	movl	$.L.str.7, %edi
	movl	$.L.str, %esi
	movl	$54, %edx
	jmp	.LBB0_2
.LBB0_16:
	movl	%eax, %edi
	callq	hipGetErrorString
	movl	$.L.str.7, %edi
	movl	$.L.str, %esi
	movl	$52, %edx
	jmp	.LBB0_2
.LBB0_14:
	movl	%eax, %edi
	callq	hipGetErrorString
	movl	$.L.str.7, %edi
	movl	$.L.str, %esi
	movl	$51, %edx
	jmp	.LBB0_2
.LBB0_37:
	movl	%eax, %edi
	callq	hipGetErrorString
	movl	$.L.str.7, %edi
	movl	$.L.str, %esi
	movl	$73, %edx
	jmp	.LBB0_2
.LBB0_34:
	movl	%eax, %edi
	callq	hipGetErrorString
	movl	$.L.str.7, %edi
	movl	$.L.str, %esi
	movl	$69, %edx
	jmp	.LBB0_2
.LBB0_32:
	movl	%eax, %edi
	callq	hipGetErrorString
	movl	$.L.str.7, %edi
	movl	$.L.str, %esi
	movl	$68, %edx
	jmp	.LBB0_2
.LBB0_30:
	movl	%eax, %edi
	callq	hipGetErrorString
	movl	$.L.str.7, %edi
	movl	$.L.str, %esi
	movl	$67, %edx
	jmp	.LBB0_2
.LBB0_52:
	movl	%eax, %edi
	callq	hipGetErrorString
	movl	$.L.str.7, %edi
	movl	$.L.str, %esi
	movl	$94, %edx
	jmp	.LBB0_2
.LBB0_50:
	movl	%eax, %edi
	callq	hipGetErrorString
	movl	$.L.str.7, %edi
	movl	$.L.str, %esi
	movl	$92, %edx
	jmp	.LBB0_2
.LBB0_48:
	movl	%eax, %edi
	callq	hipGetErrorString
	movl	$.L.str.7, %edi
	movl	$.L.str, %esi
	movl	$91, %edx
	jmp	.LBB0_2
.LBB0_46:
	movl	%eax, %edi
	callq	hipGetErrorString
	movl	$.L.str.7, %edi
	movl	$.L.str, %esi
	movl	$79, %edx
	jmp	.LBB0_2
.LBB0_44:
	movl	%eax, %edi
	callq	hipGetErrorString
	movl	$.L.str.7, %edi
	movl	$.L.str, %esi
	movl	$78, %edx
	jmp	.LBB0_2
.LBB0_42:
	movl	%eax, %edi
	callq	hipGetErrorString
	movl	$.L.str.7, %edi
	movl	$.L.str, %esi
	movl	$77, %edx
	jmp	.LBB0_2
.LBB0_1:
	movl	%eax, %edi
	callq	hipGetErrorString
	movl	$.L.str.7, %edi
	movl	$.L.str, %esi
	movl	$29, %edx
	jmp	.LBB0_2
.LBB0_4:
	movl	%eax, %edi
	callq	hipGetErrorString
	movl	$.L.str.7, %edi
	movl	$.L.str, %esi
	movl	$30, %edx
	jmp	.LBB0_2
.LBB0_9:
	movl	%eax, %edi
	callq	hipGetErrorString
	movl	$.L.str.7, %edi
	movl	$.L.str, %esi
	movl	$46, %edx
	jmp	.LBB0_2
.LBB0_54:
	movl	%eax, %edi
	callq	hipGetErrorString
	movl	$.L.str.7, %edi
	movl	$.L.str, %esi
	movl	$106, %edx
	jmp	.LBB0_2
.LBB0_56:
	movl	%eax, %edi
	callq	hipGetErrorString
	movl	$.L.str.7, %edi
	movl	$.L.str, %esi
	movl	$107, %edx
	jmp	.LBB0_2
.Lfunc_end0:
	.size	main, .Lfunc_end0-main
	.cfi_endproc
                                        # -- End function
	.type	.L.str,@object                  # @.str
	.section	.rodata.str1.1,"aMS",@progbits,1
.L.str:
	.asciz	"/home/ubuntu/Datasets/stackv2/train-structured-repos-hip-sm89/nricklin/ubuntu-gpu-test/master/test.hip"
	.size	.L.str, 103

	.type	.L.str.1,@object                # @.str.1
.L.str.1:
	.asciz	"Number of CUDA Devices = %d\n"
	.size	.L.str.1, 29

	.type	.L.str.3,@object                # @.str.3
.L.str.3:
	.asciz	"Device %d has name %s with compute capability %d.%d canMapHostMemory=%d\n"
	.size	.L.str.3, 73

	.type	.L.str.4,@object                # @.str.4
.L.str.4:
	.asciz	"                           global memory = %1.5g\n"
	.size	.L.str.4, 50

	.type	.L.str.5,@object                # @.str.5
.L.str.5:
	.asciz	"HostToDevice PCI Express BW=%g GB/s\n"
	.size	.L.str.5, 37

	.type	.L.str.6,@object                # @.str.6
.L.str.6:
	.asciz	"DeviceToHost PCI Express BW=%g GB/s\n"
	.size	.L.str.6, 37

	.type	.L.str.7,@object                # @.str.7
.L.str.7:
	.asciz	"%s(%i) : cudaSafeCall() Runtime API error : %s.\n"
	.size	.L.str.7, 49

	.type	__hip_cuid_,@object             # @__hip_cuid_
	.bss
	.globl	__hip_cuid_
__hip_cuid_:
	.byte	0                               # 0x0
	.size	__hip_cuid_, 1

	.type	.Lstr,@object                   # @str
	.section	.rodata.str1.1,"aMS",@progbits,1
.Lstr:
	.asciz	"==========================="
	.size	.Lstr, 28

	.ident	"AMD clang version 18.0.0git (https://github.com/RadeonOpenCompute/llvm-project roc-6.3.2 25012 e5bf7e55c91490b07c49d8960fa7983d864936c4)"
	.section	".note.GNU-stack","",@progbits
	.addrsig
	.addrsig_sym __hip_cuid_

	.text
	.p2alignl 7, 3214868480
	.fill 96, 4, 3214868480
	.type	__hip_cuid_,@object             ; @__hip_cuid_
	.section	.bss,"aw",@nobits
	.globl	__hip_cuid_
__hip_cuid_:
	.byte	0                               ; 0x0
	.size	__hip_cuid_, 1

	.ident	"AMD clang version 18.0.0git (https://github.com/RadeonOpenCompute/llvm-project roc-6.3.2 25012 e5bf7e55c91490b07c49d8960fa7983d864936c4)"
	.section	".note.GNU-stack","",@progbits
	.addrsig
	.addrsig_sym __hip_cuid_
	.amdgcn_target "amdgcn-amd-amdhsa--gfx1100"
	.amdgpu_metadata
---
amdhsa.kernels:  []
amdhsa.target:   amdgcn-amd-amdhsa--gfx1100
amdhsa.version:
  - 1
  - 2
...

	.end_amdgpu_metadata
