
/*****************************************************************************


  Example     : cuda-vector-vector-multiplication-mGPU.cu

  Objective   : Write a CUDA Program to perform Vector Vector multiplication
                using global memory implementation to be executed on multiple GPUs.

  Input       : None

  Output      : Execution time in seconds , Gflops achieved

  Created     : Aug 2011

  E-mail      : RarchK

****************************************************************************/



#include<stdio.h>
#include<cuda.h>
#include<pthread.h>
#include<error.h>

#define EPS 1.0e-12
#define GRIDSIZE 10
#define BLOCKSIZE 16

#define SIZE 128

int blocksPerGrid;
int gridsPerBlock;
struct Data
{
	int deviceId;
	int size;
	double* a;
	double* b;
	double retVal;	
	double Tsec;
};

__global__ void vvmul(int len,double* A,double* B,double *C)
{
	int tid= blockIdx.x*blockDim.x*blockDim.y + threadIdx.x +threadIdx.y * blockDim.x;
	
	while(tid < len)
	{
		C[tid] = A[tid] * B[tid];
		tid += blockDim.x * gridDim.x;
	}	
}

/* Check for safe return of all calls to the devic */
void CUDA_SAFE_CALL(cudaError_t call)
{
        cudaError_t ret = call;
        //printf("RETURN FROM THE CUDA CALL:%d\t:",ret);
        switch(ret)
        {
                case cudaSuccess:
                //              printf("Success\n");
                                break;
        /*      case cudaErrorInvalidValue:
                                {
                                printf("ERROR: InvalidValue:%i.\n",__LINE__);
                                exit(-1);
                                break;
                                }
                case cudaErrorInvalidDevicePointer:
                                {
                                printf("ERROR:Invalid Device pointeri:%i.\n",__LINE__);
                                exit(-1);
                                break;
                                }
                case cudaErrorInvalidMemcpyDirection:
                                {
                                printf("ERROR:Invalid memcpy direction:%i.\n",__LINE__);
                                exit(-1);
                                break;
                                }                       */
                default:
                        {
                                printf(" ERROR at line :%i.%d' ' %s\n",__LINE__,ret,cudaGetErrorString(ret));
                                exit(-1);
                                break;
                        }
        }
}


/* Get the number of GPU devices present on the host */
int get_DeviceCount()
{
	int count;
	cudaGetDeviceCount(&count);	
	return count;	
}

/* Function for vector multiplication on host*/
void host_vvmul(double* A,double* B,int len,double &C)
{
        int i;
        for(i = 0;i <len;i++)
                C += A[i]*B[i];
}

/* Function to calulate Gflops */
double calculate_gflops(double &Tsec)
{
        //printf("time taken is %.8lf\n",Tsec);
        double gflops=(1.0e-9 * (( 2.0 * SIZE  )/Tsec));
        //printf("Gflops is \t%f\n",gflops);
        return gflops;

}

/* Function to display output */
void display(double* arr,int size)
{
	int i;
	for(i=0;i<size;i++)
		printf("%f ",arr[i]);
	printf("\t%d\n",i);
}	

/*Function doing device related computations */
void* routine(void* givendata)
{
	Data *data = (Data*)givendata;
	int len = data->size;
	double *a,*b,*part_c;
	double *d_a,*d_b,*d_part_c;
	double c;
	cudaEvent_t start,stop;
	float elapsedTime;

	a=data->a;
	b=data->b;
	part_c = (double*)malloc(len*sizeof(double));

		
	CUDA_SAFE_CALL(cudaSetDevice(data->deviceId));
	
	CUDA_SAFE_CALL(cudaEventCreate(&start));
	CUDA_SAFE_CALL(cudaEventCreate(&stop));
	
	CUDA_SAFE_CALL(cudaMalloc((void**)&d_a,len*sizeof(double)));
	CUDA_SAFE_CALL(cudaMalloc((void**)&d_b,len*sizeof(double)));
	CUDA_SAFE_CALL(cudaMalloc((void**)&d_part_c,len*sizeof(double)));
	

	CUDA_SAFE_CALL(cudaMemcpy(d_a,a,len*sizeof(double),cudaMemcpyHostToDevice));	
	CUDA_SAFE_CALL(cudaMemcpy(d_b,b,len*sizeof(double),cudaMemcpyHostToDevice));	


	dim3 threadsPerBlock(16,16);
	
	int numBlocks;
	if( len /256 == 0)
		numBlocks=1;
	else
		numBlocks = len/100;
	dim3 blocksPerGrid(numBlocks ,1);
	
	CUDA_SAFE_CALL(cudaEventRecord(start,0));
	
	vvmul<<<blocksPerGrid,threadsPerBlock>>>(len,d_a,d_b,d_part_c);

	if(cudaPeekAtLastError())
		printf("KERNEL ERROR: %s\t on device:%d\n",cudaGetErrorString(cudaPeekAtLastError()),data->deviceId);

	CUDA_SAFE_CALL(cudaEventRecord(stop,0));
	CUDA_SAFE_CALL(cudaEventSynchronize(stop));

	CUDA_SAFE_CALL(cudaMemcpy(part_c,d_part_c,len*sizeof(double),cudaMemcpyDeviceToHost));	
		
	
	int ind;
	for(ind=0;ind<len;ind++)
		c += part_c[ind];

	CUDA_SAFE_CALL(cudaEventElapsedTime(&elapsedTime,start,stop));
	data->Tsec=elapsedTime*(1.0e-3);	


	CUDA_SAFE_CALL(cudaFree(d_a));
	CUDA_SAFE_CALL(cudaFree(d_b));
	CUDA_SAFE_CALL(cudaFree(d_part_c));

	free(part_c);
	data->retVal=c;
	return 0;
}

void relError(double* dRes,double* hRes,int size)
{
        double relativeError=0.0,errorNorm=0.0;
	int flag=0;
	int i;

	for( i = 0; i < size; ++i) {
                if (fabs(hRes[i]) > fabs(dRes[i]))
                        relativeError = fabs((hRes[i] - dRes[i]) / hRes[i]);
                else
                        relativeError = fabs((dRes[i] - hRes[i]) / dRes[i]);

                if (relativeError > EPS && relativeError != 0.0e+00 )
		{
                        if(errorNorm < relativeError) 
			{
                        	errorNorm = relativeError;
                        	flag=1;
                        }
                }

        }
        if( flag == 1) 
	{
                printf(" \n Results verfication : Failed");
                printf(" \n Considered machine precision : %e", EPS);
                printf(" \n Relative Error                  : %e\n", errorNorm);

        }
        else 
                printf("\n Results verfication : Success\n");

}


/* prints the result in screen */
void print_on_screen(char * program_name,float tsec,double gflops,int size,int flag)//flag=1 if gflops has been calculated else flag =0
{
	printf("\n---------------%s----------------\n",program_name);
	printf("\tSIZE\t TIME_SEC\t Gflops\n");
	if(flag==1)
	printf("\t%d\t%f\t%lf\t",size,tsec,gflops);
	else
	printf("\t%d\t%lf\t%lf\t",size,"---","---");

}


int main(int argc,char** argv)
{
	
	int devCount;
	CUDA_SAFE_CALL(cudaGetDeviceCount(&devCount));
	if(devCount < 2)
	{
		printf("Atleast 2 GPU's are needed :%d\n",devCount);
		exit(0);
	}

	double *hVectA,*hVectB,hRes,host_hRes;
	int vlen=SIZE;
	int ind;

	hVectA=(double*)malloc(vlen*sizeof(double));
	hVectB=(double*)malloc(vlen*sizeof(double));

	for(ind=0;ind < vlen;ind++)
	{
		hVectA[ind]=2.00;
		hVectB[ind]=2.00;
	}

	Data vector[2];
	
	vector[0].deviceId 	= 0;
	vector[0].size		=vlen/2;
	vector[0].a		=hVectA;
	vector[0].b		=hVectB;	   
	
	vector[1].deviceId 	= 1;
	vector[1].size		=vlen/2;
	vector[1].a		=hVectA + vlen/2 ;
	vector[1].b		=hVectB + vlen/2 ;	   

	
	pthread_t thread;
	if(pthread_create(&thread,NULL,routine,(void*)&vector[0]) != 0)
		perror("Thread creation error\n");
	routine(&vector[1]);
	pthread_join(thread,NULL);

	hRes=vector[0].retVal + vector[1].retVal;

	/* ---------Check result with host CPU result ---------*/	
	
        host_vvmul(hVectA,hVectB,vlen,host_hRes);

	relError(&hRes,&host_hRes,1);                

	print_on_screen("MatMatMult_mGPU",vector[0].Tsec,calculate_gflops(vector[0].Tsec),vlen,1);
	print_on_screen("MatMatMult_mGPU",vector[1].Tsec,calculate_gflops(vector[1].Tsec),vlen,1);

	
	free(hVectA);
	free(hVectB);
	
	return 0;
}

	.file	"tmpxft_0035dbff_00000000-6_cuda-vector-vector-multiplication-mGPU.cudafe1.cpp"
	.text
#APP

#NO_APP
	.type	_ZL26__cudaUnregisterBinaryUtilv, @function
_ZL26__cudaUnregisterBinaryUtilv:
.LFB2049:
	.cfi_startproc
	endbr64
	movq	_ZL20__cudaFatCubinHandle(%rip), %rdi
	jmp	__cudaUnregisterFatBinary@PLT
	.cfi_endproc
.LFE2049:
	.size	_ZL26__cudaUnregisterBinaryUtilv, .-_ZL26__cudaUnregisterBinaryUtilv
	.section	.rodata.str1.1,"aMS",@progbits,1
.LC0:
	.string	" ERROR at line :%i.%d' ' %s\n"
	.text
	.globl	_Z14CUDA_SAFE_CALL9cudaError
	.type	_Z14CUDA_SAFE_CALL9cudaError, @function
_Z14CUDA_SAFE_CALL9cudaError:
.LFB2038:
	.cfi_startproc
	endbr64
	testl	%edi, %edi
	je	.L2
	pushq	%rbx
	.cfi_def_cfa_offset 16
	.cfi_offset 3, -16
	movl	%edi, %ebx
	call	cudaGetErrorString@PLT
	movl	$2, %edi
	movl	%ebx, %ecx
	movl	$86, %edx
	movq	%rax, %r8
	leaq	.LC0(%rip), %rsi
	xorl	%eax, %eax
	call	__printf_chk@PLT
	orl	$-1, %edi
	call	exit@PLT
.L2:
	.cfi_def_cfa_offset 8
	.cfi_restore 3
	ret
	.cfi_endproc
.LFE2038:
	.size	_Z14CUDA_SAFE_CALL9cudaError, .-_Z14CUDA_SAFE_CALL9cudaError
	.globl	_Z15get_DeviceCountv
	.type	_Z15get_DeviceCountv, @function
_Z15get_DeviceCountv:
.LFB2039:
	.cfi_startproc
	endbr64
	subq	$24, %rsp
	.cfi_def_cfa_offset 32
	movq	%fs:40, %rax
	movq	%rax, 8(%rsp)
	xorl	%eax, %eax
	leaq	4(%rsp), %rdi
	call	cudaGetDeviceCount@PLT
	movl	4(%rsp), %eax
	movq	8(%rsp), %rdx
	subq	%fs:40, %rdx
	je	.L9
	call	__stack_chk_fail@PLT
.L9:
	addq	$24, %rsp
	.cfi_def_cfa_offset 8
	ret
	.cfi_endproc
.LFE2039:
	.size	_Z15get_DeviceCountv, .-_Z15get_DeviceCountv
	.globl	_Z10host_vvmulPdS_iRd
	.type	_Z10host_vvmulPdS_iRd, @function
_Z10host_vvmulPdS_iRd:
.LFB2040:
	.cfi_startproc
	endbr64
	xorl	%eax, %eax
.L12:
	cmpl	%eax, %edx
	jle	.L14
	movsd	(%rdi,%rax,8), %xmm0
	mulsd	(%rsi,%rax,8), %xmm0
	incq	%rax
	addsd	(%rcx), %xmm0
	movsd	%xmm0, (%rcx)
	jmp	.L12
.L14:
	ret
	.cfi_endproc
.LFE2040:
	.size	_Z10host_vvmulPdS_iRd, .-_Z10host_vvmulPdS_iRd
	.globl	_Z16calculate_gflopsRd
	.type	_Z16calculate_gflopsRd, @function
_Z16calculate_gflopsRd:
.LFB2041:
	.cfi_startproc
	endbr64
	movsd	.LC1(%rip), %xmm0
	divsd	(%rdi), %xmm0
	mulsd	.LC2(%rip), %xmm0
	ret
	.cfi_endproc
.LFE2041:
	.size	_Z16calculate_gflopsRd, .-_Z16calculate_gflopsRd
	.section	.rodata.str1.1
.LC3:
	.string	"%f "
.LC4:
	.string	"\t%d\n"
	.text
	.globl	_Z7displayPdi
	.type	_Z7displayPdi, @function
_Z7displayPdi:
.LFB2042:
	.cfi_startproc
	endbr64
	pushq	%r13
	.cfi_def_cfa_offset 16
	.cfi_offset 13, -16
	leaq	.LC3(%rip), %r13
	pushq	%r12
	.cfi_def_cfa_offset 24
	.cfi_offset 12, -24
	movq	%rdi, %r12
	pushq	%rbp
	.cfi_def_cfa_offset 32
	.cfi_offset 6, -32
	xorl	%ebp, %ebp
	pushq	%rbx
	.cfi_def_cfa_offset 40
	.cfi_offset 3, -40
	movl	%esi, %ebx
	pushq	%r8
	.cfi_def_cfa_offset 48
.L17:
	cmpl	%ebp, %ebx
	jle	.L20
	movsd	(%r12,%rbp,8), %xmm0
	movq	%r13, %rsi
	movl	$2, %edi
	movb	$1, %al
	incq	%rbp
	call	__printf_chk@PLT
	jmp	.L17
.L20:
	xorl	%eax, %eax
	testl	%ebx, %ebx
	popq	%rcx
	.cfi_def_cfa_offset 40
	movl	$2, %edi
	cmovns	%ebx, %eax
	leaq	.LC4(%rip), %rsi
	popq	%rbx
	.cfi_def_cfa_offset 32
	popq	%rbp
	.cfi_def_cfa_offset 24
	popq	%r12
	.cfi_def_cfa_offset 16
	popq	%r13
	.cfi_def_cfa_offset 8
	movl	%eax, %edx
	xorl	%eax, %eax
	jmp	__printf_chk@PLT
	.cfi_endproc
.LFE2042:
	.size	_Z7displayPdi, .-_Z7displayPdi
	.section	.rodata.str1.1
.LC8:
	.string	" \n Results verfication : Failed"
.LC9:
	.string	" \n Considered machine precision : %e"
.LC10:
	.string	" \n Relative Error                  : %e\n"
.LC11:
	.string	"\n Results verfication : Success\n"
	.text
	.globl	_Z8relErrorPdS_i
	.type	_Z8relErrorPdS_i, @function
_Z8relErrorPdS_i:
.LFB2044:
	.cfi_startproc
	endbr64
	movq	.LC6(%rip), %xmm1
	xorl	%eax, %eax
	xorl	%ecx, %ecx
	xorps	%xmm4, %xmm4
	movsd	.LC7(%rip), %xmm5
.L22:
	cmpl	%eax, %edx
	jle	.L38
	movsd	(%rsi,%rax,8), %xmm3
	movsd	(%rdi,%rax,8), %xmm2
	movaps	%xmm3, %xmm6
	movaps	%xmm2, %xmm0
	andps	%xmm1, %xmm6
	andps	%xmm1, %xmm0
	comisd	%xmm0, %xmm6
	jbe	.L33
	movaps	%xmm3, %xmm0
	subsd	%xmm2, %xmm0
	divsd	%xmm3, %xmm0
	jmp	.L37
.L33:
	movaps	%xmm2, %xmm0
	subsd	%xmm3, %xmm0
	divsd	%xmm2, %xmm0
.L37:
	andps	%xmm1, %xmm0
	comisd	%xmm5, %xmm0
	jbe	.L26
	comisd	%xmm4, %xmm0
	jbe	.L26
	movaps	%xmm0, %xmm4
	movl	$1, %ecx
.L26:
	incq	%rax
	jmp	.L22
.L38:
	decl	%ecx
	jne	.L29
	subq	$24, %rsp
	.cfi_def_cfa_offset 32
	leaq	.LC8(%rip), %rsi
	movl	$2, %edi
	xorl	%eax, %eax
	movsd	%xmm4, 8(%rsp)
	call	__printf_chk@PLT
	movsd	.LC7(%rip), %xmm0
	leaq	.LC9(%rip), %rsi
	movb	$1, %al
	movl	$2, %edi
	call	__printf_chk@PLT
	movsd	8(%rsp), %xmm4
	leaq	.LC10(%rip), %rsi
	movb	$1, %al
	movl	$2, %edi
	addq	$24, %rsp
	.cfi_def_cfa_offset 8
	movaps	%xmm4, %xmm0
	jmp	__printf_chk@PLT
.L29:
	leaq	.LC11(%rip), %rsi
	movl	$2, %edi
	xorl	%eax, %eax
	jmp	__printf_chk@PLT
	.cfi_endproc
.LFE2044:
	.size	_Z8relErrorPdS_i, .-_Z8relErrorPdS_i
	.section	.rodata.str1.1
.LC12:
	.string	"\n---------------%s----------------\n"
.LC13:
	.string	"\tSIZE\t TIME_SEC\t Gflops\n"
.LC14:
	.string	"\t%d\t%f\t%lf\t"
.LC15:
	.string	"---"
.LC16:
	.string	"\t%d\t%lf\t%lf\t"
	.text
	.globl	_Z15print_on_screenPcfdii
	.type	_Z15print_on_screenPcfdii, @function
_Z15print_on_screenPcfdii:
.LFB2045:
	.cfi_startproc
	endbr64
	pushq	%rbp
	.cfi_def_cfa_offset 16
	.cfi_offset 6, -16
	xorl	%eax, %eax
	movl	%edx, %ebp
	movq	%rdi, %rdx
	pushq	%rbx
	.cfi_def_cfa_offset 24
	.cfi_offset 3, -24
	movl	$2, %edi
	movl	%esi, %ebx
	leaq	.LC12(%rip), %rsi
	subq	$24, %rsp
	.cfi_def_cfa_offset 48
	movss	%xmm0, 12(%rsp)
	movsd	%xmm1, (%rsp)
	call	__printf_chk@PLT
	xorl	%eax, %eax
	movl	$2, %edi
	leaq	.LC13(%rip), %rsi
	call	__printf_chk@PLT
	decl	%ebp
	movsd	(%rsp), %xmm1
	movss	12(%rsp), %xmm0
	jne	.L40
	addq	$24, %rsp
	.cfi_remember_state
	.cfi_def_cfa_offset 24
	movl	%ebx, %edx
	leaq	.LC14(%rip), %rsi
	movb	$2, %al
	popq	%rbx
	.cfi_def_cfa_offset 16
	movl	$2, %edi
	popq	%rbp
	.cfi_def_cfa_offset 8
	cvtss2sd	%xmm0, %xmm0
	jmp	__printf_chk@PLT
.L40:
	.cfi_restore_state
	addq	$24, %rsp
	.cfi_def_cfa_offset 24
	leaq	.LC15(%rip), %r8
	movl	%ebx, %edx
	xorl	%eax, %eax
	popq	%rbx
	.cfi_def_cfa_offset 16
	movq	%r8, %rcx
	leaq	.LC16(%rip), %rsi
	movl	$2, %edi
	popq	%rbp
	.cfi_def_cfa_offset 8
	jmp	__printf_chk@PLT
	.cfi_endproc
.LFE2045:
	.size	_Z15print_on_screenPcfdii, .-_Z15print_on_screenPcfdii
	.globl	_Z29__device_stub__Z5vvmuliPdS_S_iPdS_S_
	.type	_Z29__device_stub__Z5vvmuliPdS_S_iPdS_S_, @function
_Z29__device_stub__Z5vvmuliPdS_S_iPdS_S_:
.LFB2071:
	.cfi_startproc
	endbr64
	subq	$152, %rsp
	.cfi_def_cfa_offset 160
	movl	%edi, 28(%rsp)
	leaq	56(%rsp), %rdi
	movq	%rsi, 16(%rsp)
	leaq	68(%rsp), %rsi
	movq	%rdx, 8(%rsp)
	leaq	40(%rsp), %rdx
	movq	%rcx, (%rsp)
	leaq	48(%rsp), %rcx
	movq	%fs:40, %rax
	movq	%rax, 136(%rsp)
	xorl	%eax, %eax
	leaq	28(%rsp), %rax
	movl	$1, 64(%rsp)
	movq	%rax, 104(%rsp)
	leaq	16(%rsp), %rax
	movq	%rax, 112(%rsp)
	leaq	8(%rsp), %rax
	movq	%rax, 120(%rsp)
	movq	%rsp, %rax
	movq	%rax, 128(%rsp)
	movabsq	$4294967297, %rax
	movq	%rax, 56(%rsp)
	movq	%rax, 68(%rsp)
	movl	$1, 76(%rsp)
	call	__cudaPopCallConfiguration@PLT
	testl	%eax, %eax
	jne	.L42
	pushq	48(%rsp)
	.cfi_def_cfa_offset 168
	leaq	_Z5vvmuliPdS_S_(%rip), %rdi
	pushq	48(%rsp)
	.cfi_def_cfa_offset 176
	movq	84(%rsp), %rcx
	movl	92(%rsp), %r8d
	movq	72(%rsp), %rsi
	movl	80(%rsp), %edx
	leaq	120(%rsp), %r9
	call	cudaLaunchKernel@PLT
	popq	%rax
	.cfi_def_cfa_offset 168
	popq	%rdx
	.cfi_def_cfa_offset 160
.L42:
	movq	136(%rsp), %rax
	subq	%fs:40, %rax
	je	.L44
	call	__stack_chk_fail@PLT
.L44:
	addq	$152, %rsp
	.cfi_def_cfa_offset 8
	ret
	.cfi_endproc
.LFE2071:
	.size	_Z29__device_stub__Z5vvmuliPdS_S_iPdS_S_, .-_Z29__device_stub__Z5vvmuliPdS_S_iPdS_S_
	.globl	_Z5vvmuliPdS_S_
	.type	_Z5vvmuliPdS_S_, @function
_Z5vvmuliPdS_S_:
.LFB2072:
	.cfi_startproc
	endbr64
	jmp	_Z29__device_stub__Z5vvmuliPdS_S_iPdS_S_
	.cfi_endproc
.LFE2072:
	.size	_Z5vvmuliPdS_S_, .-_Z5vvmuliPdS_S_
	.section	.rodata.str1.1
.LC17:
	.string	"KERNEL ERROR: %s\t on device:%d\n"
	.text
	.globl	_Z7routinePv
	.type	_Z7routinePv, @function
_Z7routinePv:
.LFB2043:
	.cfi_startproc
	endbr64
	pushq	%r15
	.cfi_def_cfa_offset 16
	.cfi_offset 15, -16
	pushq	%r14
	.cfi_def_cfa_offset 24
	.cfi_offset 14, -24
	pushq	%r13
	.cfi_def_cfa_offset 32
	.cfi_offset 13, -32
	pushq	%r12
	.cfi_def_cfa_offset 40
	.cfi_offset 12, -40
	pushq	%rbp
	.cfi_def_cfa_offset 48
	.cfi_offset 6, -48
	pushq	%rbx
	.cfi_def_cfa_offset 56
	.cfi_offset 3, -56
	movq	%rdi, %rbx
	subq	$104, %rsp
	.cfi_def_cfa_offset 160
	movslq	4(%rdi), %rbp
	movq	8(%rdi), %r15
	movq	%fs:40, %rax
	movq	%rax, 88(%rsp)
	xorl	%eax, %eax
	movq	16(%rdi), %r14
	movq	%rbp, %r12
	salq	$3, %rbp
	movq	%rbp, %rdi
	call	malloc@PLT
	movl	(%rbx), %edi
	movq	%rax, %r13
	call	cudaSetDevice@PLT
	movl	%eax, %edi
	call	_Z14CUDA_SAFE_CALL9cudaError
	leaq	48(%rsp), %rdi
	call	cudaEventCreate@PLT
	movl	%eax, %edi
	call	_Z14CUDA_SAFE_CALL9cudaError
	leaq	56(%rsp), %rdi
	call	cudaEventCreate@PLT
	movl	%eax, %edi
	call	_Z14CUDA_SAFE_CALL9cudaError
	movq	%rbp, %rsi
	leaq	24(%rsp), %rdi
	call	cudaMalloc@PLT
	movl	%eax, %edi
	call	_Z14CUDA_SAFE_CALL9cudaError
	movq	%rbp, %rsi
	leaq	32(%rsp), %rdi
	call	cudaMalloc@PLT
	movl	%eax, %edi
	call	_Z14CUDA_SAFE_CALL9cudaError
	movq	%rbp, %rsi
	leaq	40(%rsp), %rdi
	call	cudaMalloc@PLT
	movl	%eax, %edi
	call	_Z14CUDA_SAFE_CALL9cudaError
	movq	24(%rsp), %rdi
	movq	%rbp, %rdx
	movq	%r15, %rsi
	movl	$1, %ecx
	call	cudaMemcpy@PLT
	movl	%eax, %edi
	call	_Z14CUDA_SAFE_CALL9cudaError
	movq	32(%rsp), %rdi
	movq	%rbp, %rdx
	movq	%r14, %rsi
	movl	$1, %ecx
	call	cudaMemcpy@PLT
	movl	%eax, %edi
	call	_Z14CUDA_SAFE_CALL9cudaError
	leal	255(%r12), %edx
	movabsq	$68719476752, %rax
	movl	$1, 72(%rsp)
	movq	%rax, 64(%rsp)
	movl	$1, %eax
	cmpl	$510, %edx
	jbe	.L48
	movl	%r12d, %eax
	movl	$100, %ecx
	cltd
	idivl	%ecx
.L48:
	movq	48(%rsp), %rdi
	movl	%eax, 76(%rsp)
	movabsq	$4294967297, %rax
	xorl	%esi, %esi
	movq	%rax, 80(%rsp)
	call	cudaEventRecord@PLT
	movl	%eax, %edi
	call	_Z14CUDA_SAFE_CALL9cudaError
	movl	72(%rsp), %ecx
	movl	84(%rsp), %esi
	xorl	%r9d, %r9d
	movq	64(%rsp), %rdx
	movq	76(%rsp), %rdi
	xorl	%r8d, %r8d
	call	__cudaPushCallConfiguration@PLT
	testl	%eax, %eax
	jne	.L49
	movq	40(%rsp), %rcx
	movq	32(%rsp), %rdx
	movl	%r12d, %edi
	movq	24(%rsp), %rsi
	call	_Z29__device_stub__Z5vvmuliPdS_S_iPdS_S_
.L49:
	call	cudaPeekAtLastError@PLT
	testl	%eax, %eax
	je	.L50
	movl	(%rbx), %r14d
	call	cudaPeekAtLastError@PLT
	movl	%eax, %edi
	call	cudaGetErrorString@PLT
	movl	%r14d, %ecx
	movl	$2, %edi
	leaq	.LC17(%rip), %rsi
	movq	%rax, %rdx
	xorl	%eax, %eax
	call	__printf_chk@PLT
.L50:
	movq	56(%rsp), %rdi
	xorl	%esi, %esi
	call	cudaEventRecord@PLT
	movl	%eax, %edi
	call	_Z14CUDA_SAFE_CALL9cudaError
	movq	56(%rsp), %rdi
	call	cudaEventSynchronize@PLT
	movl	%eax, %edi
	call	_Z14CUDA_SAFE_CALL9cudaError
	movq	40(%rsp), %rsi
	movq	%rbp, %rdx
	movq	%r13, %rdi
	movl	$2, %ecx
	call	cudaMemcpy@PLT
	movl	%eax, %edi
	call	_Z14CUDA_SAFE_CALL9cudaError
	xorl	%eax, %eax
.L51:
	cmpl	%eax, %r12d
	jle	.L60
	movsd	8(%rsp), %xmm1
	addsd	0(%r13,%rax,8), %xmm1
	incq	%rax
	movsd	%xmm1, 8(%rsp)
	jmp	.L51
.L60:
	movq	56(%rsp), %rdx
	movq	48(%rsp), %rsi
	leaq	20(%rsp), %rdi
	call	cudaEventElapsedTime@PLT
	movl	%eax, %edi
	call	_Z14CUDA_SAFE_CALL9cudaError
	cvtss2sd	20(%rsp), %xmm0
	movq	24(%rsp), %rdi
	mulsd	.LC18(%rip), %xmm0
	movsd	%xmm0, 32(%rbx)
	call	cudaFree@PLT
	movl	%eax, %edi
	call	_Z14CUDA_SAFE_CALL9cudaError
	movq	32(%rsp), %rdi
	call	cudaFree@PLT
	movl	%eax, %edi
	call	_Z14CUDA_SAFE_CALL9cudaError
	movq	40(%rsp), %rdi
	call	cudaFree@PLT
	movl	%eax, %edi
	call	_Z14CUDA_SAFE_CALL9cudaError
	movq	%r13, %rdi
	call	free@PLT
	movsd	8(%rsp), %xmm2
	movsd	%xmm2, 24(%rbx)
	movq	88(%rsp), %rax
	subq	%fs:40, %rax
	je	.L53
	call	__stack_chk_fail@PLT
.L53:
	addq	$104, %rsp
	.cfi_def_cfa_offset 56
	xorl	%eax, %eax
	popq	%rbx
	.cfi_def_cfa_offset 48
	popq	%rbp
	.cfi_def_cfa_offset 40
	popq	%r12
	.cfi_def_cfa_offset 32
	popq	%r13
	.cfi_def_cfa_offset 24
	popq	%r14
	.cfi_def_cfa_offset 16
	popq	%r15
	.cfi_def_cfa_offset 8
	ret
	.cfi_endproc
.LFE2043:
	.size	_Z7routinePv, .-_Z7routinePv
	.section	.rodata.str1.1
.LC19:
	.string	"Atleast 2 GPU's are needed :%d\n"
.LC21:
	.string	"Thread creation error\n"
.LC22:
	.string	"MatMatMult_mGPU"
	.section	.text.startup,"ax",@progbits
	.globl	main
	.type	main, @function
main:
.LFB2046:
	.cfi_startproc
	endbr64
	pushq	%r12
	.cfi_def_cfa_offset 16
	.cfi_offset 12, -16
	pushq	%rbp
	.cfi_def_cfa_offset 24
	.cfi_offset 6, -24
	pushq	%rbx
	.cfi_def_cfa_offset 32
	.cfi_offset 3, -32
	addq	$-128, %rsp
	.cfi_def_cfa_offset 160
	movq	%fs:40, %rax
	movq	%rax, 120(%rsp)
	xorl	%eax, %eax
	leaq	12(%rsp), %rdi
	call	cudaGetDeviceCount@PLT
	movl	%eax, %edi
	call	_Z14CUDA_SAFE_CALL9cudaError
	movl	12(%rsp), %edx
	cmpl	$1, %edx
	jg	.L62
	movl	$2, %edi
	leaq	.LC19(%rip), %rsi
	xorl	%eax, %eax
	call	__printf_chk@PLT
	xorl	%edi, %edi
	call	exit@PLT
.L62:
	movl	$1024, %edi
	call	malloc@PLT
	movl	$1024, %edi
	movq	%rax, %rbp
	call	malloc@PLT
	movsd	.LC20(%rip), %xmm0
	movq	%rax, %rbx
	xorl	%eax, %eax
.L63:
	movsd	%xmm0, 0(%rbp,%rax)
	movsd	%xmm0, (%rbx,%rax)
	addq	$8, %rax
	cmpq	$1024, %rax
	jne	.L63
	movl	$1, %eax
	xorl	%esi, %esi
	leaq	40(%rsp), %rcx
	movq	%rbp, 48(%rsp)
	salq	$38, %rax
	leaq	32(%rsp), %rdi
	movq	%rbx, 56(%rsp)
	leaq	_Z7routinePv(%rip), %rdx
	movq	%rax, 40(%rsp)
	incq	%rax
	movq	%rax, 80(%rsp)
	leaq	512(%rbp), %rax
	movq	%rax, 88(%rsp)
	leaq	512(%rbx), %rax
	movq	%rax, 96(%rsp)
	call	pthread_create@PLT
	testl	%eax, %eax
	je	.L64
	leaq	.LC21(%rip), %rdi
	call	perror@PLT
.L64:
	leaq	80(%rsp), %rdi
	leaq	.LC22(%rip), %r12
	call	_Z7routinePv
	movq	32(%rsp), %rdi
	xorl	%esi, %esi
	call	pthread_join@PLT
	leaq	24(%rsp), %rcx
	movq	%rbx, %rsi
	movq	%rbp, %rdi
	movsd	64(%rsp), %xmm0
	addsd	104(%rsp), %xmm0
	movl	$128, %edx
	movsd	%xmm0, 16(%rsp)
	call	_Z10host_vvmulPdS_iRd
	movq	%rcx, %rsi
	leaq	16(%rsp), %rdi
	movl	$1, %edx
	call	_Z8relErrorPdS_i
	movsd	72(%rsp), %xmm0
	movl	$1, %edx
	movq	%r12, %rdi
	movsd	.LC1(%rip), %xmm1
	movl	$128, %esi
	divsd	%xmm0, %xmm1
	cvtsd2ss	%xmm0, %xmm0
	mulsd	.LC2(%rip), %xmm1
	call	_Z15print_on_screenPcfdii
	movsd	112(%rsp), %xmm0
	movl	$1, %edx
	movq	%r12, %rdi
	movsd	.LC1(%rip), %xmm1
	movl	$128, %esi
	divsd	%xmm0, %xmm1
	cvtsd2ss	%xmm0, %xmm0
	mulsd	.LC2(%rip), %xmm1
	call	_Z15print_on_screenPcfdii
	movq	%rbp, %rdi
	call	free@PLT
	movq	%rbx, %rdi
	call	free@PLT
	movq	120(%rsp), %rax
	subq	%fs:40, %rax
	je	.L65
	call	__stack_chk_fail@PLT
.L65:
	subq	$-128, %rsp
	.cfi_def_cfa_offset 32
	xorl	%eax, %eax
	popq	%rbx
	.cfi_def_cfa_offset 24
	popq	%rbp
	.cfi_def_cfa_offset 16
	popq	%r12
	.cfi_def_cfa_offset 8
	ret
	.cfi_endproc
.LFE2046:
	.size	main, .-main
	.section	.rodata.str1.1
.LC23:
	.string	"_Z5vvmuliPdS_S_"
	.section	.text.startup
	.type	_ZL24__sti____cudaRegisterAllv, @function
_ZL24__sti____cudaRegisterAllv:
.LFB2074:
	.cfi_startproc
	endbr64
	pushq	%rax
	.cfi_def_cfa_offset 16
	leaq	_ZL15__fatDeviceText(%rip), %rdi
	call	__cudaRegisterFatBinary@PLT
	pushq	$0
	.cfi_def_cfa_offset 24
	xorl	%r9d, %r9d
	orl	$-1, %r8d
	pushq	$0
	.cfi_def_cfa_offset 32
	leaq	.LC23(%rip), %rdx
	movq	%rax, %rdi
	leaq	_Z5vvmuliPdS_S_(%rip), %rsi
	pushq	$0
	.cfi_def_cfa_offset 40
	movq	%rdx, %rcx
	pushq	$0
	.cfi_def_cfa_offset 48
	movq	%rax, _ZL20__cudaFatCubinHandle(%rip)
	call	__cudaRegisterFunction@PLT
	movq	_ZL20__cudaFatCubinHandle(%rip), %rdi
	addq	$32, %rsp
	.cfi_def_cfa_offset 16
	call	__cudaRegisterFatBinaryEnd@PLT
	leaq	_ZL26__cudaUnregisterBinaryUtilv(%rip), %rdi
	popq	%rdx
	.cfi_def_cfa_offset 8
	jmp	atexit@PLT
	.cfi_endproc
.LFE2074:
	.size	_ZL24__sti____cudaRegisterAllv, .-_ZL24__sti____cudaRegisterAllv
	.section	.init_array,"aw"
	.align 8
	.quad	_ZL24__sti____cudaRegisterAllv
	.section	.nvFatBinSegment,"aw"
	.align 8
	.type	_ZL15__fatDeviceText, @object
	.size	_ZL15__fatDeviceText, 24
_ZL15__fatDeviceText:
	.long	1180844977
	.long	1
	.quad	fatbinData
	.quad	0
	.local	_ZL20__cudaFatCubinHandle
	.comm	_ZL20__cudaFatCubinHandle,8,8
	.globl	gridsPerBlock
	.bss
	.align 4
	.type	gridsPerBlock, @object
	.size	gridsPerBlock, 4
gridsPerBlock:
	.zero	4
	.globl	blocksPerGrid
	.align 4
	.type	blocksPerGrid, @object
	.size	blocksPerGrid, 4
blocksPerGrid:
	.zero	4
	.section	.rodata.cst8,"aM",@progbits,8
	.align 8
.LC1:
	.long	0
	.long	1081081856
	.align 8
.LC2:
	.long	-400107883
	.long	1041313291
	.section	.rodata.cst16,"aM",@progbits,16
	.align 16
.LC6:
	.long	-1
	.long	2147483647
	.long	0
	.long	0
	.section	.rodata.cst8
	.align 8
.LC7:
	.long	-2127697391
	.long	1030854553
	.align 8
.LC18:
	.long	-755914244
	.long	1062232653
	.align 8
.LC20:
	.long	0
	.long	1073741824
	.ident	"GCC: (Ubuntu 13.3.0-6ubuntu2~24.04) 13.3.0"
	.section	.note.GNU-stack,"",@progbits
	.section	.note.gnu.property,"a"
	.align 8
	.long	1f - 0f
	.long	4f - 1f
	.long	5
0:
	.string	"GNU"
1:
	.align 8
	.long	0xc0000002
	.long	3f - 2f
2:
	.long	0x3
3:
	.align 8
4:


/*****************************************************************************


  Example     : cuda-vector-vector-multiplication-mGPU.cu

  Objective   : Write a CUDA Program to perform Vector Vector multiplication
                using global memory implementation to be executed on multiple GPUs.

  Input       : None

  Output      : Execution time in seconds , Gflops achieved

  Created     : Aug 2011

  E-mail      : RarchK

****************************************************************************/



#include<stdio.h>
#include<hip/hip_runtime.h>
#include<pthread.h>
#include<error.h>

#define EPS 1.0e-12
#define GRIDSIZE 10
#define BLOCKSIZE 16

#define SIZE 128

int blocksPerGrid;
int gridsPerBlock;
struct Data
{
	int deviceId;
	int size;
	double* a;
	double* b;
	double retVal;	
	double Tsec;
};

__global__ void vvmul(int len,double* A,double* B,double *C)
{
	int tid= blockIdx.x*blockDim.x*blockDim.y + threadIdx.x +threadIdx.y * blockDim.x;
	
	while(tid < len)
	{
		C[tid] = A[tid] * B[tid];
		tid += blockDim.x * gridDim.x;
	}	
}

/* Check for safe return of all calls to the devic */
void CUDA_SAFE_CALL(hipError_t call)
{
        hipError_t ret = call;
        //printf("RETURN FROM THE CUDA CALL:%d\t:",ret);
        switch(ret)
        {
                case hipSuccess:
                //              printf("Success\n");
                                break;
        /*      case cudaErrorInvalidValue:
                                {
                                printf("ERROR: InvalidValue:%i.\n",__LINE__);
                                exit(-1);
                                break;
                                }
                case cudaErrorInvalidDevicePointer:
                                {
                                printf("ERROR:Invalid Device pointeri:%i.\n",__LINE__);
                                exit(-1);
                                break;
                                }
                case cudaErrorInvalidMemcpyDirection:
                                {
                                printf("ERROR:Invalid memcpy direction:%i.\n",__LINE__);
                                exit(-1);
                                break;
                                }                       */
                default:
                        {
                                printf(" ERROR at line :%i.%d' ' %s\n",__LINE__,ret,hipGetErrorString(ret));
                                exit(-1);
                                break;
                        }
        }
}


/* Get the number of GPU devices present on the host */
int get_DeviceCount()
{
	int count;
	hipGetDeviceCount(&count);	
	return count;	
}

/* Function for vector multiplication on host*/
void host_vvmul(double* A,double* B,int len,double &C)
{
        int i;
        for(i = 0;i <len;i++)
                C += A[i]*B[i];
}

/* Function to calulate Gflops */
double calculate_gflops(double &Tsec)
{
        //printf("time taken is %.8lf\n",Tsec);
        double gflops=(1.0e-9 * (( 2.0 * SIZE  )/Tsec));
        //printf("Gflops is \t%f\n",gflops);
        return gflops;

}

/* Function to display output */
void display(double* arr,int size)
{
	int i;
	for(i=0;i<size;i++)
		printf("%f ",arr[i]);
	printf("\t%d\n",i);
}	

/*Function doing device related computations */
void* routine(void* givendata)
{
	Data *data = (Data*)givendata;
	int len = data->size;
	double *a,*b,*part_c;
	double *d_a,*d_b,*d_part_c;
	double c;
	hipEvent_t start,stop;
	float elapsedTime;

	a=data->a;
	b=data->b;
	part_c = (double*)malloc(len*sizeof(double));

		
	CUDA_SAFE_CALL(hipSetDevice(data->deviceId));
	
	CUDA_SAFE_CALL(hipEventCreate(&start));
	CUDA_SAFE_CALL(hipEventCreate(&stop));
	
	CUDA_SAFE_CALL(hipMalloc((void**)&d_a,len*sizeof(double)));
	CUDA_SAFE_CALL(hipMalloc((void**)&d_b,len*sizeof(double)));
	CUDA_SAFE_CALL(hipMalloc((void**)&d_part_c,len*sizeof(double)));
	

	CUDA_SAFE_CALL(hipMemcpy(d_a,a,len*sizeof(double),hipMemcpyHostToDevice));	
	CUDA_SAFE_CALL(hipMemcpy(d_b,b,len*sizeof(double),hipMemcpyHostToDevice));	


	dim3 threadsPerBlock(16,16);
	
	int numBlocks;
	if( len /256 == 0)
		numBlocks=1;
	else
		numBlocks = len/100;
	dim3 blocksPerGrid(numBlocks ,1);
	
	CUDA_SAFE_CALL(hipEventRecord(start,0));
	
	vvmul<<<blocksPerGrid,threadsPerBlock>>>(len,d_a,d_b,d_part_c);

	if(hipPeekAtLastError())
		printf("KERNEL ERROR: %s\t on device:%d\n",hipGetErrorString(hipPeekAtLastError()),data->deviceId);

	CUDA_SAFE_CALL(hipEventRecord(stop,0));
	CUDA_SAFE_CALL(hipEventSynchronize(stop));

	CUDA_SAFE_CALL(hipMemcpy(part_c,d_part_c,len*sizeof(double),hipMemcpyDeviceToHost));	
		
	
	int ind;
	for(ind=0;ind<len;ind++)
		c += part_c[ind];

	CUDA_SAFE_CALL(hipEventElapsedTime(&elapsedTime,start,stop));
	data->Tsec=elapsedTime*(1.0e-3);	


	CUDA_SAFE_CALL(hipFree(d_a));
	CUDA_SAFE_CALL(hipFree(d_b));
	CUDA_SAFE_CALL(hipFree(d_part_c));

	free(part_c);
	data->retVal=c;
	return 0;
}

void relError(double* dRes,double* hRes,int size)
{
        double relativeError=0.0,errorNorm=0.0;
	int flag=0;
	int i;

	for( i = 0; i < size; ++i) {
                if (fabs(hRes[i]) > fabs(dRes[i]))
                        relativeError = fabs((hRes[i] - dRes[i]) / hRes[i]);
                else
                        relativeError = fabs((dRes[i] - hRes[i]) / dRes[i]);

                if (relativeError > EPS && relativeError != 0.0e+00 )
		{
                        if(errorNorm < relativeError) 
			{
                        	errorNorm = relativeError;
                        	flag=1;
                        }
                }

        }
        if( flag == 1) 
	{
                printf(" \n Results verfication : Failed");
                printf(" \n Considered machine precision : %e", EPS);
                printf(" \n Relative Error                  : %e\n", errorNorm);

        }
        else 
                printf("\n Results verfication : Success\n");

}


/* prints the result in screen */
void print_on_screen(char * program_name,float tsec,double gflops,int size,int flag)//flag=1 if gflops has been calculated else flag =0
{
	printf("\n---------------%s----------------\n",program_name);
	printf("\tSIZE\t TIME_SEC\t Gflops\n");
	if(flag==1)
	printf("\t%d\t%f\t%lf\t",size,tsec,gflops);
	else
	printf("\t%d\t%lf\t%lf\t",size,"---","---");

}


int main(int argc,char** argv)
{
	
	int devCount;
	CUDA_SAFE_CALL(hipGetDeviceCount(&devCount));
	if(devCount < 2)
	{
		printf("Atleast 2 GPU's are needed :%d\n",devCount);
		exit(0);
	}

	double *hVectA,*hVectB,hRes,host_hRes;
	int vlen=SIZE;
	int ind;

	hVectA=(double*)malloc(vlen*sizeof(double));
	hVectB=(double*)malloc(vlen*sizeof(double));

	for(ind=0;ind < vlen;ind++)
	{
		hVectA[ind]=2.00;
		hVectB[ind]=2.00;
	}

	Data vector[2];
	
	vector[0].deviceId 	= 0;
	vector[0].size		=vlen/2;
	vector[0].a		=hVectA;
	vector[0].b		=hVectB;	   
	
	vector[1].deviceId 	= 1;
	vector[1].size		=vlen/2;
	vector[1].a		=hVectA + vlen/2 ;
	vector[1].b		=hVectB + vlen/2 ;	   

	
	pthread_t thread;
	if(pthread_create(&thread,NULL,routine,(void*)&vector[0]) != 0)
		perror("Thread creation error\n");
	routine(&vector[1]);
	pthread_join(thread,NULL);

	hRes=vector[0].retVal + vector[1].retVal;

	/* ---------Check result with host CPU result ---------*/	
	
        host_vvmul(hVectA,hVectB,vlen,host_hRes);

	relError(&hRes,&host_hRes,1);                

	print_on_screen("MatMatMult_mGPU",vector[0].Tsec,calculate_gflops(vector[0].Tsec),vlen,1);
	print_on_screen("MatMatMult_mGPU",vector[1].Tsec,calculate_gflops(vector[1].Tsec),vlen,1);

	
	free(hVectA);
	free(hVectB);
	
	return 0;
}

	.text
	.file	"cuda-vector-vector-multiplication-mGPU.hip"
	.globl	_Z20__device_stub__vvmuliPdS_S_ # -- Begin function _Z20__device_stub__vvmuliPdS_S_
	.type	_Z20__device_stub__vvmuliPdS_S_,@function
_Z20__device_stub__vvmuliPdS_S_:        # @_Z20__device_stub__vvmuliPdS_S_
	.cfi_startproc
# %bb.0:
	pushq	%r15
	.cfi_def_cfa_offset 16
	pushq	%r14
	.cfi_def_cfa_offset 24
	pushq	%r13
	.cfi_def_cfa_offset 32
	pushq	%r12
	.cfi_def_cfa_offset 40
	pushq	%rbx
	.cfi_def_cfa_offset 48
	subq	$112, %rsp
	.cfi_def_cfa_offset 160
	.cfi_offset %rbx, -48
	.cfi_offset %r12, -40
	.cfi_offset %r13, -32
	.cfi_offset %r14, -24
	.cfi_offset %r15, -16
	leaq	4(%rsp), %rax
	movl	%edi, (%rax)
	leaq	40(%rsp), %rdi
	movq	%rsi, (%rdi)
	leaq	32(%rsp), %rsi
	movq	%rdx, (%rsi)
	leaq	24(%rsp), %rdx
	movq	%rcx, (%rdx)
	leaq	80(%rsp), %rbx
	movq	%rax, (%rbx)
	movq	%rdi, 8(%rbx)
	movq	%rsi, 16(%rbx)
	movq	%rdx, 24(%rbx)
	leaq	64(%rsp), %r14
	leaq	48(%rsp), %r15
	leaq	16(%rsp), %r12
	leaq	8(%rsp), %r13
	movq	%r14, %rdi
	movq	%r15, %rsi
	movq	%r12, %rdx
	movq	%r13, %rcx
	callq	__hipPopCallConfiguration
	movq	(%r14), %rsi
	movl	8(%r14), %edx
	movq	(%r15), %rcx
	movl	8(%r15), %r8d
	movl	$_Z5vvmuliPdS_S_, %edi
	movq	%rbx, %r9
	pushq	(%r13)
	.cfi_adjust_cfa_offset 8
	pushq	(%r12)
	.cfi_adjust_cfa_offset 8
	callq	hipLaunchKernel
	addq	$128, %rsp
	.cfi_adjust_cfa_offset -128
	popq	%rbx
	.cfi_def_cfa_offset 40
	popq	%r12
	.cfi_def_cfa_offset 32
	popq	%r13
	.cfi_def_cfa_offset 24
	popq	%r14
	.cfi_def_cfa_offset 16
	popq	%r15
	.cfi_def_cfa_offset 8
	retq
.Lfunc_end0:
	.size	_Z20__device_stub__vvmuliPdS_S_, .Lfunc_end0-_Z20__device_stub__vvmuliPdS_S_
	.cfi_endproc
                                        # -- End function
	.globl	_Z14CUDA_SAFE_CALL10hipError_t  # -- Begin function _Z14CUDA_SAFE_CALL10hipError_t
	.type	_Z14CUDA_SAFE_CALL10hipError_t,@function
_Z14CUDA_SAFE_CALL10hipError_t:         # @_Z14CUDA_SAFE_CALL10hipError_t
	.cfi_startproc
# %bb.0:
	testl	%edi, %edi
	jne	.LBB1_2
# %bb.1:
	retq
.LBB1_2:
	pushq	%rbx
	.cfi_def_cfa_offset 16
	.cfi_offset %rbx, -16
	movl	%edi, %ebx
	callq	hipGetErrorString
	movl	$.L.str, %edi
	movl	$86, %esi
	movl	%ebx, %edx
	movq	%rax, %rcx
	xorl	%eax, %eax
	callq	printf
	movl	$-1, %edi
	callq	exit
.Lfunc_end1:
	.size	_Z14CUDA_SAFE_CALL10hipError_t, .Lfunc_end1-_Z14CUDA_SAFE_CALL10hipError_t
	.cfi_endproc
                                        # -- End function
	.globl	_Z15get_DeviceCountv            # -- Begin function _Z15get_DeviceCountv
	.type	_Z15get_DeviceCountv,@function
_Z15get_DeviceCountv:                   # @_Z15get_DeviceCountv
	.cfi_startproc
# %bb.0:
	pushq	%rbx
	.cfi_def_cfa_offset 16
	subq	$16, %rsp
	.cfi_def_cfa_offset 32
	.cfi_offset %rbx, -16
	leaq	12(%rsp), %rbx
	movq	%rbx, %rdi
	callq	hipGetDeviceCount
	movl	(%rbx), %eax
	addq	$16, %rsp
	.cfi_def_cfa_offset 16
	popq	%rbx
	.cfi_def_cfa_offset 8
	retq
.Lfunc_end2:
	.size	_Z15get_DeviceCountv, .Lfunc_end2-_Z15get_DeviceCountv
	.cfi_endproc
                                        # -- End function
	.globl	_Z10host_vvmulPdS_iRd           # -- Begin function _Z10host_vvmulPdS_iRd
	.type	_Z10host_vvmulPdS_iRd,@function
_Z10host_vvmulPdS_iRd:                  # @_Z10host_vvmulPdS_iRd
	.cfi_startproc
# %bb.0:
	testl	%edx, %edx
	jle	.LBB3_3
# %bb.1:                                # %.lr.ph.preheader
	movsd	(%rcx), %xmm0                   # xmm0 = mem[0],zero
	movl	%edx, %eax
	xorl	%edx, %edx
.LBB3_2:                                # %.lr.ph
                                        # =>This Inner Loop Header: Depth=1
	movsd	(%rdi,%rdx,8), %xmm1            # xmm1 = mem[0],zero
	mulsd	(%rsi,%rdx,8), %xmm1
	addsd	%xmm1, %xmm0
	movsd	%xmm0, (%rcx)
	incq	%rdx
	cmpq	%rdx, %rax
	jne	.LBB3_2
.LBB3_3:                                # %._crit_edge
	retq
.Lfunc_end3:
	.size	_Z10host_vvmulPdS_iRd, .Lfunc_end3-_Z10host_vvmulPdS_iRd
	.cfi_endproc
                                        # -- End function
	.section	.rodata.cst8,"aM",@progbits,8
	.p2align	3, 0x0                          # -- Begin function _Z16calculate_gflopsRd
.LCPI4_0:
	.quad	0x4070000000000000              # double 256
.LCPI4_1:
	.quad	0x3e112e0be826d695              # double 1.0000000000000001E-9
	.text
	.globl	_Z16calculate_gflopsRd
	.type	_Z16calculate_gflopsRd,@function
_Z16calculate_gflopsRd:                 # @_Z16calculate_gflopsRd
	.cfi_startproc
# %bb.0:
	movsd	.LCPI4_0(%rip), %xmm0           # xmm0 = mem[0],zero
	divsd	(%rdi), %xmm0
	mulsd	.LCPI4_1(%rip), %xmm0
	retq
.Lfunc_end4:
	.size	_Z16calculate_gflopsRd, .Lfunc_end4-_Z16calculate_gflopsRd
	.cfi_endproc
                                        # -- End function
	.globl	_Z7displayPdi                   # -- Begin function _Z7displayPdi
	.type	_Z7displayPdi,@function
_Z7displayPdi:                          # @_Z7displayPdi
	.cfi_startproc
# %bb.0:
	pushq	%r15
	.cfi_def_cfa_offset 16
	pushq	%r14
	.cfi_def_cfa_offset 24
	pushq	%r12
	.cfi_def_cfa_offset 32
	pushq	%rbx
	.cfi_def_cfa_offset 40
	pushq	%rax
	.cfi_def_cfa_offset 48
	.cfi_offset %rbx, -40
	.cfi_offset %r12, -32
	.cfi_offset %r14, -24
	.cfi_offset %r15, -16
	testl	%esi, %esi
	jle	.LBB5_1
# %bb.2:                                # %.lr.ph.preheader
	movl	%esi, %ebx
	movq	%rdi, %r14
	movl	%esi, %r15d
	xorl	%r12d, %r12d
.LBB5_3:                                # %.lr.ph
                                        # =>This Inner Loop Header: Depth=1
	movsd	(%r14,%r12,8), %xmm0            # xmm0 = mem[0],zero
	movl	$.L.str.1, %edi
	movb	$1, %al
	callq	printf
	incq	%r12
	cmpq	%r12, %r15
	jne	.LBB5_3
	jmp	.LBB5_4
.LBB5_1:
	xorl	%ebx, %ebx
.LBB5_4:                                # %._crit_edge
	movl	$.L.str.2, %edi
	movl	%ebx, %esi
	xorl	%eax, %eax
	addq	$8, %rsp
	.cfi_def_cfa_offset 40
	popq	%rbx
	.cfi_def_cfa_offset 32
	popq	%r12
	.cfi_def_cfa_offset 24
	popq	%r14
	.cfi_def_cfa_offset 16
	popq	%r15
	.cfi_def_cfa_offset 8
	jmp	printf                          # TAILCALL
.Lfunc_end5:
	.size	_Z7displayPdi, .Lfunc_end5-_Z7displayPdi
	.cfi_endproc
                                        # -- End function
	.section	.rodata.cst8,"aM",@progbits,8
	.p2align	3, 0x0                          # -- Begin function _Z7routinePv
.LCPI6_0:
	.quad	0x3f50624dd2f1a9fc              # double 0.001
	.text
	.globl	_Z7routinePv
	.type	_Z7routinePv,@function
_Z7routinePv:                           # @_Z7routinePv
	.cfi_startproc
# %bb.0:
	pushq	%rbp
	.cfi_def_cfa_offset 16
	pushq	%r15
	.cfi_def_cfa_offset 24
	pushq	%r14
	.cfi_def_cfa_offset 32
	pushq	%r13
	.cfi_def_cfa_offset 40
	pushq	%r12
	.cfi_def_cfa_offset 48
	pushq	%rbx
	.cfi_def_cfa_offset 56
	subq	$72, %rsp
	.cfi_def_cfa_offset 128
	.cfi_offset %rbx, -56
	.cfi_offset %r12, -48
	.cfi_offset %r13, -40
	.cfi_offset %r14, -32
	.cfi_offset %r15, -24
	.cfi_offset %rbp, -16
	movq	%rdi, %r13
	movslq	4(%rdi), %rbx
	movl	%ebx, %r15d
	movq	8(%rdi), %rax
	movq	%rax, 48(%rsp)                  # 8-byte Spill
	movq	16(%rdi), %rax
	movq	%rax, (%rsp)                    # 8-byte Spill
	leaq	(,%rbx,8), %r12
	movq	%r12, %rdi
	callq	malloc
	movq	%rax, %r14
	movq	%r13, 56(%rsp)                  # 8-byte Spill
	movl	(%r13), %edi
	callq	hipSetDevice
	movl	%eax, %edi
	callq	_Z14CUDA_SAFE_CALL10hipError_t
	leaq	24(%rsp), %rdi
	callq	hipEventCreate
	movl	%eax, %edi
	callq	_Z14CUDA_SAFE_CALL10hipError_t
	leaq	8(%rsp), %rdi
	callq	hipEventCreate
	movl	%eax, %edi
	callq	_Z14CUDA_SAFE_CALL10hipError_t
	leaq	40(%rsp), %r13
	movq	%r13, %rdi
	movq	%r12, %rsi
	callq	hipMalloc
	movl	%eax, %edi
	callq	_Z14CUDA_SAFE_CALL10hipError_t
	leaq	32(%rsp), %rbp
	movq	%rbp, %rdi
	movq	%r12, %rsi
	callq	hipMalloc
	movl	%eax, %edi
	callq	_Z14CUDA_SAFE_CALL10hipError_t
	leaq	16(%rsp), %rdi
	movq	%r12, %rsi
	callq	hipMalloc
	movl	%eax, %edi
	callq	_Z14CUDA_SAFE_CALL10hipError_t
	movq	(%r13), %rdi
	movq	48(%rsp), %rsi                  # 8-byte Reload
	movq	%r12, %rdx
	movl	$1, %ecx
	callq	hipMemcpy
	movl	%eax, %edi
	callq	_Z14CUDA_SAFE_CALL10hipError_t
	movq	(%rbp), %rdi
	movq	(%rsp), %rsi                    # 8-byte Reload
	movq	%r12, %rdx
	movl	$1, %ecx
	callq	hipMemcpy
	movl	%eax, %edi
	callq	_Z14CUDA_SAFE_CALL10hipError_t
	leal	255(%rbx), %eax
	imulq	$1374389535, %rbx, %r13         # imm = 0x51EB851F
	movq	%r13, %rcx
	shrq	$63, %rcx
	sarq	$37, %r13
	addl	%ecx, %r13d
	cmpl	$511, %eax                      # imm = 0x1FF
	movl	$1, %eax
	cmovbl	%eax, %r13d
	btsq	$32, %r13
	leaq	24(%rsp), %rax
	movq	(%rax), %rdi
	xorl	%esi, %esi
	callq	hipEventRecord
	movl	%eax, %edi
	callq	_Z14CUDA_SAFE_CALL10hipError_t
	movabsq	$68719476752, %rdx              # imm = 0x1000000010
	movq	%r13, %rdi
	movl	$1, %esi
	movl	$1, %ecx
	xorl	%r8d, %r8d
	xorl	%r9d, %r9d
	callq	__hipPushCallConfiguration
	testl	%eax, %eax
	jne	.LBB6_2
# %bb.1:
	movq	40(%rsp), %rsi
	movq	32(%rsp), %rdx
	movq	16(%rsp), %rcx
	movl	%r15d, %edi
	callq	_Z20__device_stub__vvmuliPdS_S_
.LBB6_2:
	callq	hipPeekAtLastError
	testl	%eax, %eax
	movq	56(%rsp), %rbx                  # 8-byte Reload
	je	.LBB6_4
# %bb.3:
	callq	hipPeekAtLastError
	movl	%eax, %edi
	callq	hipGetErrorString
	movl	(%rbx), %edx
	movl	$.L.str.3, %edi
	movq	%rax, %rsi
	xorl	%eax, %eax
	callq	printf
.LBB6_4:
	movq	8(%rsp), %rdi
	xorl	%esi, %esi
	callq	hipEventRecord
	movl	%eax, %edi
	callq	_Z14CUDA_SAFE_CALL10hipError_t
	movq	8(%rsp), %rdi
	callq	hipEventSynchronize
	movl	%eax, %edi
	callq	_Z14CUDA_SAFE_CALL10hipError_t
	movq	16(%rsp), %rsi
	movq	%r14, %rdi
	movq	%r12, %rdx
	movl	$2, %ecx
	callq	hipMemcpy
	movl	%eax, %edi
	callq	_Z14CUDA_SAFE_CALL10hipError_t
	testl	%r15d, %r15d
	jle	.LBB6_5
# %bb.6:                                # %.lr.ph.preheader
	xorl	%eax, %eax
                                        # implicit-def: $xmm0
.LBB6_7:                                # %.lr.ph
                                        # =>This Inner Loop Header: Depth=1
	addsd	(%r14,%rax,8), %xmm0
	incq	%rax
	cmpq	%rax, %r15
	jne	.LBB6_7
	jmp	.LBB6_8
.LBB6_5:
                                        # implicit-def: $xmm0
.LBB6_8:                                # %._crit_edge
	movsd	%xmm0, (%rsp)                   # 8-byte Spill
	movq	24(%rsp), %rsi
	movq	8(%rsp), %rdx
	leaq	68(%rsp), %r15
	movq	%r15, %rdi
	callq	hipEventElapsedTime
	movl	%eax, %edi
	callq	_Z14CUDA_SAFE_CALL10hipError_t
	xorps	%xmm0, %xmm0
	cvtss2sd	(%r15), %xmm0
	mulsd	.LCPI6_0(%rip), %xmm0
	movsd	%xmm0, 32(%rbx)
	movq	40(%rsp), %rdi
	callq	hipFree
	movl	%eax, %edi
	callq	_Z14CUDA_SAFE_CALL10hipError_t
	movq	32(%rsp), %rdi
	callq	hipFree
	movl	%eax, %edi
	callq	_Z14CUDA_SAFE_CALL10hipError_t
	movq	16(%rsp), %rdi
	callq	hipFree
	movl	%eax, %edi
	callq	_Z14CUDA_SAFE_CALL10hipError_t
	movq	%r14, %rdi
	callq	free
	movsd	(%rsp), %xmm0                   # 8-byte Reload
                                        # xmm0 = mem[0],zero
	movsd	%xmm0, 24(%rbx)
	xorl	%eax, %eax
	addq	$72, %rsp
	.cfi_def_cfa_offset 56
	popq	%rbx
	.cfi_def_cfa_offset 48
	popq	%r12
	.cfi_def_cfa_offset 40
	popq	%r13
	.cfi_def_cfa_offset 32
	popq	%r14
	.cfi_def_cfa_offset 24
	popq	%r15
	.cfi_def_cfa_offset 16
	popq	%rbp
	.cfi_def_cfa_offset 8
	retq
.Lfunc_end6:
	.size	_Z7routinePv, .Lfunc_end6-_Z7routinePv
	.cfi_endproc
                                        # -- End function
	.section	.rodata.cst16,"aM",@progbits,16
	.p2align	4, 0x0                          # -- Begin function _Z8relErrorPdS_i
.LCPI7_0:
	.quad	0x7fffffffffffffff              # double NaN
	.quad	0x7fffffffffffffff              # double NaN
	.section	.rodata.cst8,"aM",@progbits,8
	.p2align	3, 0x0
.LCPI7_1:
	.quad	0x3d719799812dea11              # double 9.9999999999999998E-13
	.text
	.globl	_Z8relErrorPdS_i
	.type	_Z8relErrorPdS_i,@function
_Z8relErrorPdS_i:                       # @_Z8relErrorPdS_i
	.cfi_startproc
# %bb.0:
	testl	%edx, %edx
	jle	.LBB7_7
# %bb.1:                                # %.lr.ph.preheader
	movl	%edx, %eax
	xorpd	%xmm0, %xmm0
	xorl	%ecx, %ecx
	movapd	.LCPI7_0(%rip), %xmm1           # xmm1 = [NaN,NaN]
	movsd	.LCPI7_1(%rip), %xmm2           # xmm2 = mem[0],zero
	movl	$1, %edx
	xorl	%r8d, %r8d
	xorpd	%xmm3, %xmm3
.LBB7_2:                                # %.lr.ph
                                        # =>This Inner Loop Header: Depth=1
	movsd	(%rsi,%rcx,8), %xmm4            # xmm4 = mem[0],zero
	movsd	(%rdi,%rcx,8), %xmm5            # xmm5 = mem[0],zero
	movapd	%xmm5, %xmm6
	subsd	%xmm4, %xmm6
	movapd	%xmm4, %xmm7
	subsd	%xmm5, %xmm7
	divsd	%xmm4, %xmm7
	andpd	%xmm1, %xmm4
	divsd	%xmm5, %xmm6
	andpd	%xmm1, %xmm5
	cmpltsd	%xmm4, %xmm5
	andpd	%xmm5, %xmm7
	andnpd	%xmm6, %xmm5
	orpd	%xmm7, %xmm5
	movapd	%xmm5, %xmm6
	andpd	%xmm1, %xmm6
	cmpneqpd	%xmm0, %xmm5
	movapd	%xmm2, %xmm4
	cmpltpd	%xmm6, %xmm4
	andpd	%xmm5, %xmm4
	ucomisd	%xmm3, %xmm6
	movd	%xmm4, %r9d
	seta	%r10b
	testb	%r10b, %r9b
	jne	.LBB7_4
# %bb.3:                                # %.lr.ph
                                        #   in Loop: Header=BB7_2 Depth=1
	movapd	%xmm3, %xmm6
.LBB7_4:                                # %.lr.ph
                                        #   in Loop: Header=BB7_2 Depth=1
	cmovnel	%edx, %r8d
	incq	%rcx
	movapd	%xmm6, %xmm3
	cmpq	%rcx, %rax
	jne	.LBB7_2
# %bb.5:                                # %._crit_edge
	cmpl	$1, %r8d
	jne	.LBB7_7
# %bb.6:
	subq	$24, %rsp
	.cfi_def_cfa_offset 32
	movl	$.L.str.4, %edi
	xorl	%eax, %eax
	movapd	%xmm6, (%rsp)                   # 16-byte Spill
	callq	printf
	movsd	.LCPI7_1(%rip), %xmm0           # xmm0 = mem[0],zero
	movl	$.L.str.5, %edi
	movb	$1, %al
	callq	printf
	movl	$.L.str.6, %edi
	movaps	(%rsp), %xmm0                   # 16-byte Reload
	movb	$1, %al
	addq	$24, %rsp
	.cfi_def_cfa_offset 8
	jmp	printf                          # TAILCALL
.LBB7_7:                                # %._crit_edge.thread
	movl	$.Lstr, %edi
	jmp	puts@PLT                        # TAILCALL
.Lfunc_end7:
	.size	_Z8relErrorPdS_i, .Lfunc_end7-_Z8relErrorPdS_i
	.cfi_endproc
                                        # -- End function
	.globl	_Z15print_on_screenPcfdii       # -- Begin function _Z15print_on_screenPcfdii
	.type	_Z15print_on_screenPcfdii,@function
_Z15print_on_screenPcfdii:              # @_Z15print_on_screenPcfdii
	.cfi_startproc
# %bb.0:
	pushq	%rbp
	.cfi_def_cfa_offset 16
	pushq	%rbx
	.cfi_def_cfa_offset 24
	subq	$24, %rsp
	.cfi_def_cfa_offset 48
	.cfi_offset %rbx, -24
	.cfi_offset %rbp, -16
	movl	%edx, %ebp
	movl	%esi, %ebx
	movsd	%xmm1, 16(%rsp)                 # 8-byte Spill
	movss	%xmm0, 12(%rsp)                 # 4-byte Spill
	movq	%rdi, %rsi
	movl	$.L.str.8, %edi
	xorl	%eax, %eax
	callq	printf
	movl	$.Lstr.1, %edi
	callq	puts@PLT
	cmpl	$1, %ebp
	jne	.LBB8_2
# %bb.1:
	xorps	%xmm0, %xmm0
	cvtss2sd	12(%rsp), %xmm0                 # 4-byte Folded Reload
	movl	$.L.str.10, %edi
	movl	%ebx, %esi
	movsd	16(%rsp), %xmm1                 # 8-byte Reload
                                        # xmm1 = mem[0],zero
	movb	$2, %al
	addq	$24, %rsp
	.cfi_def_cfa_offset 24
	popq	%rbx
	.cfi_def_cfa_offset 16
	popq	%rbp
	.cfi_def_cfa_offset 8
	jmp	printf                          # TAILCALL
.LBB8_2:
	.cfi_def_cfa_offset 48
	movl	$.L.str.11, %edi
	movl	$.L.str.12, %edx
	movl	$.L.str.12, %ecx
	movl	%ebx, %esi
	xorl	%eax, %eax
	addq	$24, %rsp
	.cfi_def_cfa_offset 24
	popq	%rbx
	.cfi_def_cfa_offset 16
	popq	%rbp
	.cfi_def_cfa_offset 8
	jmp	printf                          # TAILCALL
.Lfunc_end8:
	.size	_Z15print_on_screenPcfdii, .Lfunc_end8-_Z15print_on_screenPcfdii
	.cfi_endproc
                                        # -- End function
	.section	.rodata.cst8,"aM",@progbits,8
	.p2align	3, 0x0                          # -- Begin function main
.LCPI9_0:
	.quad	0x4070000000000000              # double 256
.LCPI9_1:
	.quad	0x3e112e0be826d695              # double 1.0000000000000001E-9
	.text
	.globl	main
	.type	main,@function
main:                                   # @main
	.cfi_startproc
# %bb.0:
	pushq	%r15
	.cfi_def_cfa_offset 16
	pushq	%r14
	.cfi_def_cfa_offset 24
	pushq	%rbx
	.cfi_def_cfa_offset 32
	subq	$112, %rsp
	.cfi_def_cfa_offset 144
	.cfi_offset %rbx, -32
	.cfi_offset %r14, -24
	.cfi_offset %r15, -16
	leaq	20(%rsp), %rbx
	movq	%rbx, %rdi
	callq	hipGetDeviceCount
	movl	%eax, %edi
	callq	_Z14CUDA_SAFE_CALL10hipError_t
	movl	(%rbx), %esi
	cmpl	$1, %esi
	jle	.LBB9_8
# %bb.1:
	movl	$1024, %edi                     # imm = 0x400
	callq	malloc
	movq	%rax, %rbx
	movl	$1024, %edi                     # imm = 0x400
	callq	malloc
	movq	%rax, %r14
	xorl	%eax, %eax
	movabsq	$4611686018427387904, %rcx      # imm = 0x4000000000000000
.LBB9_2:                                # =>This Inner Loop Header: Depth=1
	movq	%rcx, (%rbx,%rax,8)
	movq	%rcx, (%r14,%rax,8)
	incq	%rax
	cmpq	$128, %rax
	jne	.LBB9_2
# %bb.3:
	movabsq	$274877906944, %rax             # imm = 0x4000000000
	leaq	72(%rsp), %r15
	movq	%rax, -40(%r15)
	movq	%rbx, -32(%r15)
	movq	%r14, -24(%r15)
	movabsq	$274877906945, %rax             # imm = 0x4000000001
	movq	%rax, (%r15)
	movl	$512, %eax                      # imm = 0x200
	movq	%rbx, %rcx
	addq	%rax, %rcx
	movq	%rcx, 8(%r15)
	movq	%r14, %rcx
	addq	%rax, %rcx
	movq	%rcx, 16(%r15)
	movq	%rsp, %rdi
	leaq	32(%rsp), %rcx
	movl	$_Z7routinePv, %edx
	xorl	%esi, %esi
	callq	pthread_create
	testl	%eax, %eax
	jne	.LBB9_4
.LBB9_5:
	movq	%r15, %rdi
	callq	_Z7routinePv
	movq	(%rsp), %rdi
	xorl	%r15d, %r15d
	xorl	%esi, %esi
	callq	pthread_join
	movsd	56(%rsp), %xmm0                 # xmm0 = mem[0],zero
	addsd	96(%rsp), %xmm0
	movsd	%xmm0, 8(%rsp)
                                        # implicit-def: $xmm0
.LBB9_6:                                # %.lr.ph.i
                                        # =>This Inner Loop Header: Depth=1
	movsd	(%rbx,%r15,8), %xmm1            # xmm1 = mem[0],zero
	mulsd	(%r14,%r15,8), %xmm1
	addsd	%xmm1, %xmm0
	incq	%r15
	cmpq	$128, %r15
	jne	.LBB9_6
# %bb.7:                                # %_Z10host_vvmulPdS_iRd.exit
	leaq	24(%rsp), %rsi
	movsd	%xmm0, (%rsi)
	leaq	8(%rsp), %rdi
	movl	$1, %edx
	callq	_Z8relErrorPdS_i
	movsd	64(%rsp), %xmm2                 # xmm2 = mem[0],zero
	xorps	%xmm0, %xmm0
	cvtsd2ss	%xmm2, %xmm0
	movsd	.LCPI9_0(%rip), %xmm1           # xmm1 = mem[0],zero
	divsd	%xmm2, %xmm1
	mulsd	.LCPI9_1(%rip), %xmm1
	movl	$.L.str.15, %edi
	movl	$128, %esi
	movl	$1, %edx
	callq	_Z15print_on_screenPcfdii
	movsd	104(%rsp), %xmm1                # xmm1 = mem[0],zero
	xorps	%xmm0, %xmm0
	cvtsd2ss	%xmm1, %xmm0
	movsd	.LCPI9_0(%rip), %xmm2           # xmm2 = mem[0],zero
	divsd	%xmm1, %xmm2
	mulsd	.LCPI9_1(%rip), %xmm2
	movapd	%xmm2, %xmm1
	movl	$.L.str.15, %edi
	movl	$128, %esi
	movl	$1, %edx
	callq	_Z15print_on_screenPcfdii
	movq	%rbx, %rdi
	callq	free
	movq	%r14, %rdi
	callq	free
	xorl	%eax, %eax
	addq	$112, %rsp
	.cfi_def_cfa_offset 32
	popq	%rbx
	.cfi_def_cfa_offset 24
	popq	%r14
	.cfi_def_cfa_offset 16
	popq	%r15
	.cfi_def_cfa_offset 8
	retq
.LBB9_4:
	.cfi_def_cfa_offset 144
	movl	$.L.str.14, %edi
	callq	perror
	jmp	.LBB9_5
.LBB9_8:
	movl	$.L.str.13, %edi
	xorl	%eax, %eax
	callq	printf
	xorl	%edi, %edi
	callq	exit
.Lfunc_end9:
	.size	main, .Lfunc_end9-main
	.cfi_endproc
                                        # -- End function
	.p2align	4, 0x90                         # -- Begin function __hip_module_ctor
	.type	__hip_module_ctor,@function
__hip_module_ctor:                      # @__hip_module_ctor
	.cfi_startproc
# %bb.0:
	subq	$40, %rsp
	.cfi_def_cfa_offset 48
	movq	__hip_gpubin_handle(%rip), %rdi
	testq	%rdi, %rdi
	jne	.LBB10_2
# %bb.1:
	movl	$__hip_fatbin_wrapper, %edi
	callq	__hipRegisterFatBinary
	movq	%rax, %rdi
	movq	%rax, __hip_gpubin_handle(%rip)
.LBB10_2:
	xorps	%xmm0, %xmm0
	movups	%xmm0, 16(%rsp)
	movups	%xmm0, (%rsp)
	movl	$_Z5vvmuliPdS_S_, %esi
	movl	$.L__unnamed_1, %edx
	movl	$.L__unnamed_1, %ecx
	movl	$-1, %r8d
	xorl	%r9d, %r9d
	callq	__hipRegisterFunction
	movl	$__hip_module_dtor, %edi
	addq	$40, %rsp
	.cfi_def_cfa_offset 8
	jmp	atexit                          # TAILCALL
.Lfunc_end10:
	.size	__hip_module_ctor, .Lfunc_end10-__hip_module_ctor
	.cfi_endproc
                                        # -- End function
	.p2align	4, 0x90                         # -- Begin function __hip_module_dtor
	.type	__hip_module_dtor,@function
__hip_module_dtor:                      # @__hip_module_dtor
	.cfi_startproc
# %bb.0:
	movq	__hip_gpubin_handle(%rip), %rdi
	testq	%rdi, %rdi
	je	.LBB11_2
# %bb.1:
	pushq	%rax
	.cfi_def_cfa_offset 16
	callq	__hipUnregisterFatBinary
	movq	$0, __hip_gpubin_handle(%rip)
	addq	$8, %rsp
	.cfi_def_cfa_offset 8
.LBB11_2:
	retq
.Lfunc_end11:
	.size	__hip_module_dtor, .Lfunc_end11-__hip_module_dtor
	.cfi_endproc
                                        # -- End function
	.type	blocksPerGrid,@object           # @blocksPerGrid
	.bss
	.globl	blocksPerGrid
	.p2align	2, 0x0
blocksPerGrid:
	.long	0                               # 0x0
	.size	blocksPerGrid, 4

	.type	gridsPerBlock,@object           # @gridsPerBlock
	.globl	gridsPerBlock
	.p2align	2, 0x0
gridsPerBlock:
	.long	0                               # 0x0
	.size	gridsPerBlock, 4

	.type	_Z5vvmuliPdS_S_,@object         # @_Z5vvmuliPdS_S_
	.section	.rodata,"a",@progbits
	.globl	_Z5vvmuliPdS_S_
	.p2align	3, 0x0
_Z5vvmuliPdS_S_:
	.quad	_Z20__device_stub__vvmuliPdS_S_
	.size	_Z5vvmuliPdS_S_, 8

	.type	.L.str,@object                  # @.str
	.section	.rodata.str1.1,"aMS",@progbits,1
.L.str:
	.asciz	" ERROR at line :%i.%d' ' %s\n"
	.size	.L.str, 29

	.type	.L.str.1,@object                # @.str.1
.L.str.1:
	.asciz	"%f "
	.size	.L.str.1, 4

	.type	.L.str.2,@object                # @.str.2
.L.str.2:
	.asciz	"\t%d\n"
	.size	.L.str.2, 5

	.type	.L.str.3,@object                # @.str.3
.L.str.3:
	.asciz	"KERNEL ERROR: %s\t on device:%d\n"
	.size	.L.str.3, 32

	.type	.L.str.4,@object                # @.str.4
.L.str.4:
	.asciz	" \n Results verfication : Failed"
	.size	.L.str.4, 32

	.type	.L.str.5,@object                # @.str.5
.L.str.5:
	.asciz	" \n Considered machine precision : %e"
	.size	.L.str.5, 37

	.type	.L.str.6,@object                # @.str.6
.L.str.6:
	.asciz	" \n Relative Error                  : %e\n"
	.size	.L.str.6, 41

	.type	.L.str.8,@object                # @.str.8
.L.str.8:
	.asciz	"\n---------------%s----------------\n"
	.size	.L.str.8, 36

	.type	.L.str.10,@object               # @.str.10
.L.str.10:
	.asciz	"\t%d\t%f\t%lf\t"
	.size	.L.str.10, 12

	.type	.L.str.11,@object               # @.str.11
.L.str.11:
	.asciz	"\t%d\t%lf\t%lf\t"
	.size	.L.str.11, 13

	.type	.L.str.12,@object               # @.str.12
.L.str.12:
	.asciz	"---"
	.size	.L.str.12, 4

	.type	.L.str.13,@object               # @.str.13
.L.str.13:
	.asciz	"Atleast 2 GPU's are needed :%d\n"
	.size	.L.str.13, 32

	.type	.L.str.14,@object               # @.str.14
.L.str.14:
	.asciz	"Thread creation error\n"
	.size	.L.str.14, 23

	.type	.L.str.15,@object               # @.str.15
.L.str.15:
	.asciz	"MatMatMult_mGPU"
	.size	.L.str.15, 16

	.type	.L__unnamed_1,@object           # @0
.L__unnamed_1:
	.asciz	"_Z5vvmuliPdS_S_"
	.size	.L__unnamed_1, 16

	.type	__hip_fatbin_wrapper,@object    # @__hip_fatbin_wrapper
	.section	.hipFatBinSegment,"a",@progbits
	.p2align	3, 0x0
__hip_fatbin_wrapper:
	.long	1212764230                      # 0x48495046
	.long	1                               # 0x1
	.quad	__hip_fatbin
	.quad	0
	.size	__hip_fatbin_wrapper, 24

	.type	__hip_gpubin_handle,@object     # @__hip_gpubin_handle
	.local	__hip_gpubin_handle
	.comm	__hip_gpubin_handle,8,8
	.section	.init_array,"aw",@init_array
	.p2align	3, 0x0
	.quad	__hip_module_ctor
	.type	__hip_cuid_,@object             # @__hip_cuid_
	.bss
	.globl	__hip_cuid_
__hip_cuid_:
	.byte	0                               # 0x0
	.size	__hip_cuid_, 1

	.type	.Lstr,@object                   # @str
	.section	.rodata.str1.1,"aMS",@progbits,1
.Lstr:
	.asciz	"\n Results verfication : Success"
	.size	.Lstr, 32

	.type	.Lstr.1,@object                 # @str.1
.Lstr.1:
	.asciz	"\tSIZE\t TIME_SEC\t Gflops"
	.size	.Lstr.1, 24

	.ident	"AMD clang version 18.0.0git (https://github.com/RadeonOpenCompute/llvm-project roc-6.3.2 25012 e5bf7e55c91490b07c49d8960fa7983d864936c4)"
	.section	".note.GNU-stack","",@progbits
	.addrsig
	.addrsig_sym _Z20__device_stub__vvmuliPdS_S_
	.addrsig_sym _Z7routinePv
	.addrsig_sym __hip_module_ctor
	.addrsig_sym __hip_module_dtor
	.addrsig_sym _Z5vvmuliPdS_S_
	.addrsig_sym __hip_fatbin
	.addrsig_sym __hip_fatbin_wrapper
	.addrsig_sym __hip_cuid_

	.text
	.amdgcn_target "amdgcn-amd-amdhsa--gfx1100"
	.protected	_Z5vvmuliPdS_S_         ; -- Begin function _Z5vvmuliPdS_S_
	.globl	_Z5vvmuliPdS_S_
	.p2align	8
	.type	_Z5vvmuliPdS_S_,@function
_Z5vvmuliPdS_S_:                        ; @_Z5vvmuliPdS_S_
; %bb.0:
	s_clause 0x1
	s_load_b32 s4, s[0:1], 0x2c
	s_load_b32 s8, s[0:1], 0x0
	v_bfe_u32 v1, v0, 10, 10
	s_add_u32 s2, s0, 32
	s_addc_u32 s3, s1, 0
	s_waitcnt lgkmcnt(0)
	s_lshr_b32 s5, s4, 16
	s_and_b32 s9, s4, 0xffff
	v_mad_u64_u32 v[2:3], null, s15, s5, v[1:2]
	v_and_b32_e32 v3, 0x3ff, v0
	s_mov_b32 s4, exec_lo
	s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
	v_mad_u64_u32 v[0:1], null, v2, s9, v[3:4]
	v_cmpx_gt_i32_e64 s8, v0
	s_cbranch_execz .LBB0_3
; %bb.1:                                ; %.lr.ph
	s_load_b32 s10, s[2:3], 0x0
	s_clause 0x1
	s_load_b128 s[4:7], s[0:1], 0x8
	s_load_b64 s[2:3], s[0:1], 0x18
	s_waitcnt lgkmcnt(0)
	s_mul_i32 s1, s10, s9
	s_mov_b32 s9, 0
.LBB0_2:                                ; =>This Inner Loop Header: Depth=1
	v_ashrrev_i32_e32 v1, 31, v0
	s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
	v_lshlrev_b64 v[1:2], 3, v[0:1]
	v_add_nc_u32_e32 v0, s1, v0
	v_add_co_u32 v3, vcc_lo, s4, v1
	s_delay_alu instid0(VALU_DEP_3)
	v_add_co_ci_u32_e32 v4, vcc_lo, s5, v2, vcc_lo
	v_add_co_u32 v5, vcc_lo, s6, v1
	v_add_co_ci_u32_e32 v6, vcc_lo, s7, v2, vcc_lo
	v_cmp_le_i32_e32 vcc_lo, s8, v0
	global_load_b64 v[3:4], v[3:4], off
	global_load_b64 v[5:6], v[5:6], off
	v_add_co_u32 v1, s0, s2, v1
	s_delay_alu instid0(VALU_DEP_1)
	v_add_co_ci_u32_e64 v2, s0, s3, v2, s0
	s_or_b32 s9, vcc_lo, s9
	s_waitcnt vmcnt(0)
	v_mul_f64 v[3:4], v[3:4], v[5:6]
	global_store_b64 v[1:2], v[3:4], off
	s_and_not1_b32 exec_lo, exec_lo, s9
	s_cbranch_execnz .LBB0_2
.LBB0_3:                                ; %._crit_edge
	s_nop 0
	s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
	s_endpgm
	.section	.rodata,"a",@progbits
	.p2align	6, 0x0
	.amdhsa_kernel _Z5vvmuliPdS_S_
		.amdhsa_group_segment_fixed_size 0
		.amdhsa_private_segment_fixed_size 0
		.amdhsa_kernarg_size 288
		.amdhsa_user_sgpr_count 15
		.amdhsa_user_sgpr_dispatch_ptr 0
		.amdhsa_user_sgpr_queue_ptr 0
		.amdhsa_user_sgpr_kernarg_segment_ptr 1
		.amdhsa_user_sgpr_dispatch_id 0
		.amdhsa_user_sgpr_private_segment_size 0
		.amdhsa_wavefront_size32 1
		.amdhsa_uses_dynamic_stack 0
		.amdhsa_enable_private_segment 0
		.amdhsa_system_sgpr_workgroup_id_x 1
		.amdhsa_system_sgpr_workgroup_id_y 0
		.amdhsa_system_sgpr_workgroup_id_z 0
		.amdhsa_system_sgpr_workgroup_info 0
		.amdhsa_system_vgpr_workitem_id 1
		.amdhsa_next_free_vgpr 7
		.amdhsa_next_free_sgpr 16
		.amdhsa_float_round_mode_32 0
		.amdhsa_float_round_mode_16_64 0
		.amdhsa_float_denorm_mode_32 3
		.amdhsa_float_denorm_mode_16_64 3
		.amdhsa_dx10_clamp 1
		.amdhsa_ieee_mode 1
		.amdhsa_fp16_overflow 0
		.amdhsa_workgroup_processor_mode 1
		.amdhsa_memory_ordered 1
		.amdhsa_forward_progress 0
		.amdhsa_shared_vgpr_count 0
		.amdhsa_exception_fp_ieee_invalid_op 0
		.amdhsa_exception_fp_denorm_src 0
		.amdhsa_exception_fp_ieee_div_zero 0
		.amdhsa_exception_fp_ieee_overflow 0
		.amdhsa_exception_fp_ieee_underflow 0
		.amdhsa_exception_fp_ieee_inexact 0
		.amdhsa_exception_int_div_zero 0
	.end_amdhsa_kernel
	.text
.Lfunc_end0:
	.size	_Z5vvmuliPdS_S_, .Lfunc_end0-_Z5vvmuliPdS_S_
                                        ; -- End function
	.section	.AMDGPU.csdata,"",@progbits
; Kernel info:
; codeLenInByte = 268
; NumSgprs: 18
; NumVgprs: 7
; ScratchSize: 0
; MemoryBound: 0
; FloatMode: 240
; IeeeMode: 1
; LDSByteSize: 0 bytes/workgroup (compile time only)
; SGPRBlocks: 2
; VGPRBlocks: 0
; NumSGPRsForWavesPerEU: 18
; NumVGPRsForWavesPerEU: 7
; Occupancy: 16
; WaveLimiterHint : 0
; COMPUTE_PGM_RSRC2:SCRATCH_EN: 0
; COMPUTE_PGM_RSRC2:USER_SGPR: 15
; COMPUTE_PGM_RSRC2:TRAP_HANDLER: 0
; COMPUTE_PGM_RSRC2:TGID_X_EN: 1
; COMPUTE_PGM_RSRC2:TGID_Y_EN: 0
; COMPUTE_PGM_RSRC2:TGID_Z_EN: 0
; COMPUTE_PGM_RSRC2:TIDIG_COMP_CNT: 1
	.text
	.p2alignl 7, 3214868480
	.fill 96, 4, 3214868480
	.type	__hip_cuid_,@object             ; @__hip_cuid_
	.section	.bss,"aw",@nobits
	.globl	__hip_cuid_
__hip_cuid_:
	.byte	0                               ; 0x0
	.size	__hip_cuid_, 1

	.ident	"AMD clang version 18.0.0git (https://github.com/RadeonOpenCompute/llvm-project roc-6.3.2 25012 e5bf7e55c91490b07c49d8960fa7983d864936c4)"
	.section	".note.GNU-stack","",@progbits
	.addrsig
	.addrsig_sym __hip_cuid_
	.amdgpu_metadata
---
amdhsa.kernels:
  - .args:
      - .offset:         0
        .size:           4
        .value_kind:     by_value
      - .address_space:  global
        .offset:         8
        .size:           8
        .value_kind:     global_buffer
      - .address_space:  global
        .offset:         16
        .size:           8
        .value_kind:     global_buffer
      - .address_space:  global
        .offset:         24
        .size:           8
        .value_kind:     global_buffer
      - .offset:         32
        .size:           4
        .value_kind:     hidden_block_count_x
      - .offset:         36
        .size:           4
        .value_kind:     hidden_block_count_y
      - .offset:         40
        .size:           4
        .value_kind:     hidden_block_count_z
      - .offset:         44
        .size:           2
        .value_kind:     hidden_group_size_x
      - .offset:         46
        .size:           2
        .value_kind:     hidden_group_size_y
      - .offset:         48
        .size:           2
        .value_kind:     hidden_group_size_z
      - .offset:         50
        .size:           2
        .value_kind:     hidden_remainder_x
      - .offset:         52
        .size:           2
        .value_kind:     hidden_remainder_y
      - .offset:         54
        .size:           2
        .value_kind:     hidden_remainder_z
      - .offset:         72
        .size:           8
        .value_kind:     hidden_global_offset_x
      - .offset:         80
        .size:           8
        .value_kind:     hidden_global_offset_y
      - .offset:         88
        .size:           8
        .value_kind:     hidden_global_offset_z
      - .offset:         96
        .size:           2
        .value_kind:     hidden_grid_dims
    .group_segment_fixed_size: 0
    .kernarg_segment_align: 8
    .kernarg_segment_size: 288
    .language:       OpenCL C
    .language_version:
      - 2
      - 0
    .max_flat_workgroup_size: 1024
    .name:           _Z5vvmuliPdS_S_
    .private_segment_fixed_size: 0
    .sgpr_count:     18
    .sgpr_spill_count: 0
    .symbol:         _Z5vvmuliPdS_S_.kd
    .uniform_work_group_size: 1
    .uses_dynamic_stack: false
    .vgpr_count:     7
    .vgpr_spill_count: 0
    .wavefront_size: 32
    .workgroup_processor_mode: 1
amdhsa.target:   amdgcn-amd-amdhsa--gfx1100
amdhsa.version:
  - 1
  - 2
...

	.end_amdgpu_metadata
