/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include <stdio.h>
#include <stdlib.h>
#include <pthread.h>

struct GPUState {
	int deviceCount; // number of GPUs to use
	int deviceToUse; // GPU to use (round-robin)
	pthread_mutex_t initLock;
} gpustate = {-1,-1,PTHREAD_MUTEX_INITIALIZER};

// returns which GPU to run on, or -1 if no GPUs are available
int get_gpu() {
	if (gpustate.deviceCount == 1)
		return 0; // return immediately for the common case of 1 GPU
	else if (gpustate.deviceCount > 1) { // multiple GPUs
		int newval, oldval;
		do {
			oldval = gpustate.deviceToUse;
			if (oldval == gpustate.deviceCount-1)
				newval = 0;
			else
				newval = oldval+1;
		} while (!__sync_bool_compare_and_swap(&gpustate.deviceToUse, oldval, newval));
	}
	else if (gpustate.deviceCount == -1) { // not yet initialized... run initialization
		pthread_mutex_lock(&gpustate.initLock);
		// check if another thread already completed initialization
		if (gpustate.deviceCount != -1) {
			pthread_mutex_unlock(&gpustate.initLock);
			return get_gpu();
		}
		// continue with initialization
		if (cudaGetDeviceCount(&gpustate.deviceCount)) {
			fprintf(stderr, "Cuda Error in GetDeviceCount: %s\n", cudaGetErrorString(cudaGetLastError()));
			gpustate.deviceCount = 0;
		}
		else if (gpustate.deviceCount <= 0)
			gpustate.deviceCount = 0;
		else
			gpustate.deviceToUse = 0;

		for (int deviceID=0; deviceID<gpustate.deviceCount; deviceID++) {
			cudaSetDevice(deviceID);
			cudaDeviceReset();
		}
		pthread_mutex_unlock(&gpustate.initLock);
	}

	return gpustate.deviceToUse;
}

	.file	"tmpxft_0024b92c_00000000-6_utilities.cudafe1.cpp"
	.text
#APP

#NO_APP
	.type	_ZL26__cudaUnregisterBinaryUtilv, @function
_ZL26__cudaUnregisterBinaryUtilv:
.LFB2039:
	.cfi_startproc
	endbr64
	movq	_ZL20__cudaFatCubinHandle(%rip), %rdi
	jmp	__cudaUnregisterFatBinary@PLT
	.cfi_endproc
.LFE2039:
	.size	_ZL26__cudaUnregisterBinaryUtilv, .-_ZL26__cudaUnregisterBinaryUtilv
	.section	.rodata.str1.1,"aMS",@progbits,1
.LC0:
	.string	"Cuda Error in GetDeviceCount: %s\n"
	.text
	.globl	_Z7get_gpuv
	.type	_Z7get_gpuv, @function
_Z7get_gpuv:
.LFB2036:
	.cfi_startproc
	endbr64
	pushq	%r12
	.cfi_def_cfa_offset 16
	.cfi_offset 12, -16
	pushq	%rbp
	.cfi_def_cfa_offset 24
	.cfi_offset 6, -24
	pushq	%rbx
	.cfi_def_cfa_offset 32
	.cfi_offset 3, -32
	leaq	gpustate(%rip), %rbx
.L10:
	movl	(%rbx), %eax
	cmpl	$1, %eax
	je	.L16
	jle	.L4
.L6:
	movl	(%rbx), %esi
	movl	4+gpustate(%rip), %eax
	xorl	%ecx, %ecx
	leal	-1(%rsi), %edx
	cmpl	%eax, %edx
	je	.L5
	leal	1(%rax), %ecx
.L5:
	lock cmpxchgl	%ecx, 4+gpustate(%rip)
	je	.L7
	jmp	.L6
.L4:
	incl	%eax
	jne	.L7
	leaq	8+gpustate(%rip), %r12
	movq	%r12, %rdi
	call	pthread_mutex_lock@PLT
	cmpl	$-1, (%rbx)
	je	.L9
	movq	%r12, %rdi
	call	pthread_mutex_unlock@PLT
	jmp	.L10
.L9:
	movq	%rbx, %rdi
	call	cudaGetDeviceCount@PLT
	testl	%eax, %eax
	je	.L11
	call	cudaGetLastError@PLT
	movl	%eax, %edi
	call	cudaGetErrorString@PLT
	movq	stderr(%rip), %rdi
	movl	$2, %esi
	leaq	.LC0(%rip), %rdx
	movq	%rax, %rcx
	xorl	%eax, %eax
	call	__fprintf_chk@PLT
	jmp	.L12
.L11:
	cmpl	$0, gpustate(%rip)
	jle	.L12
	xorl	%edx, %edx
	xorl	%ebp, %ebp
	movl	%edx, 4+gpustate(%rip)
.L13:
	cmpl	%ebp, (%rbx)
	jle	.L15
	movl	%ebp, %edi
	incl	%ebp
	call	cudaSetDevice@PLT
	call	cudaDeviceReset@PLT
	jmp	.L13
.L12:
	xorl	%eax, %eax
	movl	%eax, gpustate(%rip)
.L15:
	movq	%r12, %rdi
	call	pthread_mutex_unlock@PLT
.L7:
	movl	4+gpustate(%rip), %eax
	jmp	.L2
.L16:
	xorl	%eax, %eax
.L2:
	popq	%rbx
	.cfi_def_cfa_offset 24
	popq	%rbp
	.cfi_def_cfa_offset 16
	popq	%r12
	.cfi_def_cfa_offset 8
	ret
	.cfi_endproc
.LFE2036:
	.size	_Z7get_gpuv, .-_Z7get_gpuv
	.section	.text.startup,"ax",@progbits
	.type	_ZL24__sti____cudaRegisterAllv, @function
_ZL24__sti____cudaRegisterAllv:
.LFB2062:
	.cfi_startproc
	endbr64
	pushq	%rax
	.cfi_def_cfa_offset 16
	leaq	_ZL15__fatDeviceText(%rip), %rdi
	call	__cudaRegisterFatBinary@PLT
	movq	%rax, %rdi
	movq	%rax, _ZL20__cudaFatCubinHandle(%rip)
	call	__cudaRegisterFatBinaryEnd@PLT
	leaq	_ZL26__cudaUnregisterBinaryUtilv(%rip), %rdi
	popq	%rdx
	.cfi_def_cfa_offset 8
	jmp	atexit@PLT
	.cfi_endproc
.LFE2062:
	.size	_ZL24__sti____cudaRegisterAllv, .-_ZL24__sti____cudaRegisterAllv
	.section	.init_array,"aw"
	.align 8
	.quad	_ZL24__sti____cudaRegisterAllv
	.section	.nvFatBinSegment,"aw"
	.align 8
	.type	_ZL15__fatDeviceText, @object
	.size	_ZL15__fatDeviceText, 24
_ZL15__fatDeviceText:
	.long	1180844977
	.long	1
	.quad	fatbinData
	.quad	0
	.local	_ZL20__cudaFatCubinHandle
	.comm	_ZL20__cudaFatCubinHandle,8,8
	.globl	gpustate
	.data
	.align 32
	.type	gpustate, @object
	.size	gpustate, 48
gpustate:
	.long	-1
	.long	-1
	.long	0
	.long	0
	.long	0
	.long	0
	.long	0
	.value	0
	.value	0
	.quad	0
	.quad	0
	.ident	"GCC: (Ubuntu 13.3.0-6ubuntu2~24.04) 13.3.0"
	.section	.note.GNU-stack,"",@progbits
	.section	.note.gnu.property,"a"
	.align 8
	.long	1f - 0f
	.long	4f - 1f
	.long	5
0:
	.string	"GNU"
1:
	.align 8
	.long	0xc0000002
	.long	3f - 2f
2:
	.long	0x3
3:
	.align 8
4:

/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */


#include <hip/hip_runtime.h>
#include <stdio.h>
#include <stdlib.h>
#include <pthread.h>

struct GPUState {
	int deviceCount; // number of GPUs to use
	int deviceToUse; // GPU to use (round-robin)
	pthread_mutex_t initLock;
} gpustate = {-1,-1,PTHREAD_MUTEX_INITIALIZER};

// returns which GPU to run on, or -1 if no GPUs are available
int get_gpu() {
	if (gpustate.deviceCount == 1)
		return 0; // return immediately for the common case of 1 GPU
	else if (gpustate.deviceCount > 1) { // multiple GPUs
		int newval, oldval;
		do {
			oldval = gpustate.deviceToUse;
			if (oldval == gpustate.deviceCount-1)
				newval = 0;
			else
				newval = oldval+1;
		} while (!__sync_bool_compare_and_swap(&gpustate.deviceToUse, oldval, newval));
	}
	else if (gpustate.deviceCount == -1) { // not yet initialized... run initialization
		pthread_mutex_lock(&gpustate.initLock);
		// check if another thread already completed initialization
		if (gpustate.deviceCount != -1) {
			pthread_mutex_unlock(&gpustate.initLock);
			return get_gpu();
		}
		// continue with initialization
		if (hipGetDeviceCount(&gpustate.deviceCount)) {
			fprintf(stderr, "Cuda Error in GetDeviceCount: %s\n", hipGetErrorString(hipGetLastError()));
			gpustate.deviceCount = 0;
		}
		else if (gpustate.deviceCount <= 0)
			gpustate.deviceCount = 0;
		else
			gpustate.deviceToUse = 0;

		for (int deviceID=0; deviceID<gpustate.deviceCount; deviceID++) {
			hipSetDevice(deviceID);
			hipDeviceReset();
		}
		pthread_mutex_unlock(&gpustate.initLock);
	}

	return gpustate.deviceToUse;
}

	.text
	.file	"utilities.hip"
	.globl	_Z7get_gpuv                     # -- Begin function _Z7get_gpuv
	.type	_Z7get_gpuv,@function
_Z7get_gpuv:                            # @_Z7get_gpuv
	.cfi_startproc
# %bb.0:
	pushq	%rbx
	.cfi_def_cfa_offset 16
	.cfi_offset %rbx, -16
	movl	gpustate(%rip), %eax
	xorl	%ebx, %ebx
.LBB0_1:                                # =>This Inner Loop Header: Depth=1
	cmpl	$1, %eax
	je	.LBB0_16
# %bb.2:                                # %.lr.ph
                                        #   in Loop: Header=BB0_1 Depth=1
	cmpl	$2, %eax
	jge	.LBB0_3
# %bb.5:                                #   in Loop: Header=BB0_1 Depth=1
	cmpl	$-1, %eax
	jne	.LBB0_15
# %bb.6:                                #   in Loop: Header=BB0_1 Depth=1
	movl	$gpustate+8, %edi
	callq	pthread_mutex_lock
	cmpl	$-1, gpustate(%rip)
	je	.LBB0_8
# %bb.7:                                # %tailrecurse
                                        #   in Loop: Header=BB0_1 Depth=1
	movl	$gpustate+8, %edi
	callq	pthread_mutex_unlock
	movl	gpustate(%rip), %eax
	jmp	.LBB0_1
.LBB0_3:                                # %.preheader.preheader
	xorl	%ecx, %ecx
.LBB0_4:                                # %.preheader
                                        # =>This Inner Loop Header: Depth=1
	movl	gpustate+4(%rip), %eax
	movl	gpustate(%rip), %edx
	decl	%edx
	leal	1(%rax), %esi
	cmpl	%edx, %eax
	cmovel	%ecx, %esi
                                        # kill: def $eax killed $eax killed $rax
	lock		cmpxchgl	%esi, gpustate+4(%rip)
	jne	.LBB0_4
	jmp	.LBB0_15
.LBB0_8:
	movl	$gpustate, %edi
	callq	hipGetDeviceCount
	testl	%eax, %eax
	jne	.LBB0_9
# %bb.11:
	cmpl	$0, gpustate(%rip)
	jle	.LBB0_10
# %bb.12:                               # %.lr.ph16.preheader
	movl	$0, gpustate+4(%rip)
	xorl	%ebx, %ebx
.LBB0_13:                               # %.lr.ph16
                                        # =>This Inner Loop Header: Depth=1
	movl	%ebx, %edi
	callq	hipSetDevice
	callq	hipDeviceReset
	incl	%ebx
	cmpl	gpustate(%rip), %ebx
	jl	.LBB0_13
	jmp	.LBB0_14
.LBB0_9:
	movq	stderr(%rip), %rbx
	callq	hipGetLastError
	movl	%eax, %edi
	callq	hipGetErrorString
	movl	$.L.str, %esi
	movq	%rbx, %rdi
	movq	%rax, %rdx
	xorl	%eax, %eax
	callq	fprintf
.LBB0_10:                               # %._crit_edge.sink.split
	movl	$0, gpustate(%rip)
.LBB0_14:                               # %._crit_edge
	movl	$gpustate+8, %edi
	callq	pthread_mutex_unlock
.LBB0_15:                               # %.loopexit
	movl	gpustate+4(%rip), %ebx
.LBB0_16:                               # %.loopexit13
	movl	%ebx, %eax
	popq	%rbx
	.cfi_def_cfa_offset 8
	retq
.Lfunc_end0:
	.size	_Z7get_gpuv, .Lfunc_end0-_Z7get_gpuv
	.cfi_endproc
                                        # -- End function
	.type	gpustate,@object                # @gpustate
	.data
	.globl	gpustate
	.p2align	3, 0x0
gpustate:
	.long	4294967295                      # 0xffffffff
	.long	4294967295                      # 0xffffffff
	.zero	40
	.size	gpustate, 48

	.type	.L.str,@object                  # @.str
	.section	.rodata.str1.1,"aMS",@progbits,1
.L.str:
	.asciz	"Cuda Error in GetDeviceCount: %s\n"
	.size	.L.str, 34

	.type	__hip_cuid_,@object             # @__hip_cuid_
	.bss
	.globl	__hip_cuid_
__hip_cuid_:
	.byte	0                               # 0x0
	.size	__hip_cuid_, 1

	.ident	"AMD clang version 18.0.0git (https://github.com/RadeonOpenCompute/llvm-project roc-6.3.2 25012 e5bf7e55c91490b07c49d8960fa7983d864936c4)"
	.section	".note.GNU-stack","",@progbits
	.addrsig
	.addrsig_sym gpustate
	.addrsig_sym __hip_cuid_

	.text
	.p2alignl 7, 3214868480
	.fill 96, 4, 3214868480
	.type	__hip_cuid_,@object             ; @__hip_cuid_
	.section	.bss,"aw",@nobits
	.globl	__hip_cuid_
__hip_cuid_:
	.byte	0                               ; 0x0
	.size	__hip_cuid_, 1

	.ident	"AMD clang version 18.0.0git (https://github.com/RadeonOpenCompute/llvm-project roc-6.3.2 25012 e5bf7e55c91490b07c49d8960fa7983d864936c4)"
	.section	".note.GNU-stack","",@progbits
	.addrsig
	.addrsig_sym __hip_cuid_
	.amdgcn_target "amdgcn-amd-amdhsa--gfx1100"
	.amdgpu_metadata
---
amdhsa.kernels:  []
amdhsa.target:   amdgcn-amd-amdhsa--gfx1100
amdhsa.version:
  - 1
  - 2
...

	.end_amdgpu_metadata
