/*
 * MSU CUDA Course Examples and Exercises.
 *
 * Copyright (c) 2011 Dmitry Mikushin
 *
 * This software is provided 'as-is', without any express or implied warranty.
 * In no event will the authors be held liable for any damages arising 
 * from the use of this software.
 * Permission is granted to anyone to use this software for any purpose, 
 * including commercial applications, and to alter it and redistribute it freely,
 * without any restrictons.
 *
 * This sample demonstates parallel execution using process forking.
 * Each process works on own private data.
 *
 */

#include <cuda_runtime.h>
#include <errno.h>
#include <malloc.h>
#include <stdio.h>
#include <unistd.h>

// Perform some dummy 2D field processing on GPU and CPU,
// and compare results.
int pattern2d(int nx, int ny, float* in, float* out, int pid, int step);

int nticks = 10;

// The size of memory region.
int nx = 512, ny = 256;
size_t size = nx * ny * sizeof(float);

int main(int argc, char* argv[])
{
	// Allocate input & output arrays.
	float* input = (float*)malloc(size);
	float* output = (float*)malloc(size);

	// Generate input data array of the
	// specified size.
	long np = nx * ny;
	float invdrandmax = 1.0 / RAND_MAX;
	for (long i = 0; i < np; i++)
		input[i] = rand() * invdrandmax;

	// Call fork to create another process.
	// Standard: "Memory mappings created in the parent
	// shall be retained in the child process."
	pid_t fork_status = fork();

	// From this point two processes are running the same code, if no errors.
	if (fork_status == -1)
	{
		fprintf(stderr, "Cannot fork process, errno = %d\n", errno);
		return errno;
	}

	// By fork return value we can determine the process role:
	// master or child (worker).
	int master = fork_status ? 1 : 0;

	int ndevices = 0;
	cudaError_t cuda_status = cudaGetDeviceCount(&ndevices);
	if (cuda_status != cudaSuccess)
	{
		fprintf(stderr, "Cannot get the cuda device count, status = %d\n",
			cuda_status);
		return cuda_status;
	}
	
	// Return if no cuda devices present.
	if (master)
		printf("%d CUDA device(s) found\n", ndevices);
	if (!ndevices) return 0;

	// Get the process ID.
	int pid = (int)getpid();
	
	// Use different devices, if more than one present.
	if (ndevices > 1)
	{
		int idevice = 1;
		if (master) idevice = 0;
		
		cuda_status = cudaSetDevice(idevice);
		if (cuda_status != cudaSuccess)
		{
			fprintf(stderr, "Cannot set CUDA device by process %d, status = %d\n",
				pid, cuda_status);
			return cuda_status;
		}
		printf("Process %d uses device #%d\n", pid, idevice);
	}
	
	// Perform some "iterations" on data array private to each process.
	for (int i = 0; i < nticks; i++)
	{
		// Execute function with CUDA kernel.
		int status = pattern2d(nx, ny, input, output, pid, i);
		if (status)
		{
			fprintf(stderr, "Pattern 2D failed by process %d, status = %d\n",
				pid, status);
			return status;
		}
	}
	
	free(input);
	free(output);
	
	return 0;
}

	.file	"tmpxft_0031ebf6_00000000-6_process_fork_cuda.cudafe1.cpp"
	.text
#APP

#NO_APP
	.type	_ZL26__cudaUnregisterBinaryUtilv, @function
_ZL26__cudaUnregisterBinaryUtilv:
.LFB2043:
	.cfi_startproc
	endbr64
	movq	_ZL20__cudaFatCubinHandle(%rip), %rdi
	jmp	__cudaUnregisterFatBinary@PLT
	.cfi_endproc
.LFE2043:
	.size	_ZL26__cudaUnregisterBinaryUtilv, .-_ZL26__cudaUnregisterBinaryUtilv
	.section	.rodata.str1.1,"aMS",@progbits,1
.LC1:
	.string	"Cannot fork process, errno = %d\n"
.LC2:
	.string	"Cannot get the cuda device count, status = %d\n"
.LC3:
	.string	"%d CUDA device(s) found\n"
.LC4:
	.string	"Cannot set CUDA device by process %d, status = %d\n"
.LC5:
	.string	"Process %d uses device #%d\n"
.LC6:
	.string	"Pattern 2D failed by process %d, status = %d\n"
	.section	.text.startup,"ax",@progbits
	.globl	main
	.type	main, @function
main:
.LFB2040:
	.cfi_startproc
	endbr64
	pushq	%r14
	.cfi_def_cfa_offset 16
	.cfi_offset 14, -16
	pushq	%r13
	.cfi_def_cfa_offset 24
	.cfi_offset 13, -24
	xorl	%r13d, %r13d
	pushq	%r12
	.cfi_def_cfa_offset 32
	.cfi_offset 12, -32
	pushq	%rbp
	.cfi_def_cfa_offset 40
	.cfi_offset 6, -40
	pushq	%rbx
	.cfi_def_cfa_offset 48
	.cfi_offset 3, -48
	subq	$16, %rsp
	.cfi_def_cfa_offset 64
	movq	size(%rip), %rbx
	movq	%fs:40, %rax
	movq	%rax, 8(%rsp)
	xorl	%eax, %eax
	movq	%rbx, %rdi
	call	malloc@PLT
	movq	%rbx, %rdi
	movq	%rax, %rbp
	call	malloc@PLT
	movl	nx(%rip), %ebx
	imull	ny(%rip), %ebx
	movq	%rax, %r12
	movslq	%ebx, %rbx
.L3:
	cmpq	%rbx, %r13
	jge	.L22
	call	rand@PLT
	cvtsi2ssl	%eax, %xmm0
	mulss	.LC0(%rip), %xmm0
	movss	%xmm0, 0(%rbp,%r13,4)
	incq	%r13
	jmp	.L3
.L22:
	call	fork@PLT
	movl	%eax, %r14d
	cmpl	$-1, %eax
	jne	.L5
	call	__errno_location@PLT
	movq	stderr(%rip), %rdi
	movl	$2, %esi
	leaq	.LC1(%rip), %rdx
	movl	(%rax), %ecx
	movq	%rax, %rbx
	xorl	%eax, %eax
	call	__fprintf_chk@PLT
	movl	(%rbx), %ebx
	jmp	.L2
.L5:
	xorl	%esi, %esi
	leaq	4(%rsp), %rdi
	movl	%esi, 4(%rsp)
	call	cudaGetDeviceCount@PLT
	movl	%eax, %ebx
	testl	%eax, %eax
	je	.L7
	movq	stderr(%rip), %rdi
	movl	%eax, %ecx
	movl	$2, %esi
	xorl	%eax, %eax
	leaq	.LC2(%rip), %rdx
	call	__fprintf_chk@PLT
	jmp	.L2
.L7:
	testl	%r14d, %r14d
	je	.L8
	movl	4(%rsp), %edx
	leaq	.LC3(%rip), %rsi
	movl	$2, %edi
	xorl	%eax, %eax
	call	__printf_chk@PLT
.L8:
	cmpl	$0, 4(%rsp)
	je	.L9
	call	getpid@PLT
	cmpl	$1, 4(%rsp)
	movl	%eax, %r13d
	jle	.L10
	testl	%r14d, %r14d
	sete	%r14b
	movzbl	%r14b, %r14d
	movl	%r14d, %edi
	call	cudaSetDevice@PLT
	movl	%eax, %ebx
	testl	%eax, %eax
	je	.L11
	movl	%eax, %r8d
	movl	%r13d, %ecx
	leaq	.LC4(%rip), %rdx
	jmp	.L20
.L11:
	movl	%r14d, %ecx
	movl	%r13d, %edx
	leaq	.LC5(%rip), %rsi
	xorl	%eax, %eax
	movl	$2, %edi
	call	__printf_chk@PLT
	jmp	.L10
.L12:
	incl	%r14d
.L13:
	cmpl	%r14d, nticks(%rip)
	jle	.L23
	movl	ny(%rip), %esi
	movl	%r14d, %r9d
	movl	%r13d, %r8d
	movq	%r12, %rcx
	movl	nx(%rip), %edi
	movq	%rbp, %rdx
	call	_Z9pattern2diiPfS_ii@PLT
	movl	%eax, %ebx
	testl	%eax, %eax
	je	.L12
	movl	%eax, %r8d
	movl	%r13d, %ecx
	leaq	.LC6(%rip), %rdx
.L20:
	movq	stderr(%rip), %rdi
	movl	$2, %esi
	xorl	%eax, %eax
	call	__fprintf_chk@PLT
	jmp	.L2
.L10:
	xorl	%r14d, %r14d
	jmp	.L13
.L23:
	movq	%rbp, %rdi
	call	free@PLT
	movq	%r12, %rdi
	call	free@PLT
.L9:
	xorl	%ebx, %ebx
.L2:
	movq	8(%rsp), %rax
	subq	%fs:40, %rax
	je	.L15
	call	__stack_chk_fail@PLT
.L15:
	addq	$16, %rsp
	.cfi_def_cfa_offset 48
	movl	%ebx, %eax
	popq	%rbx
	.cfi_def_cfa_offset 40
	popq	%rbp
	.cfi_def_cfa_offset 32
	popq	%r12
	.cfi_def_cfa_offset 24
	popq	%r13
	.cfi_def_cfa_offset 16
	popq	%r14
	.cfi_def_cfa_offset 8
	ret
	.cfi_endproc
.LFE2040:
	.size	main, .-main
	.type	_ZL24__sti____cudaRegisterAllv, @function
_ZL24__sti____cudaRegisterAllv:
.LFB2066:
	.cfi_startproc
	endbr64
	pushq	%rax
	.cfi_def_cfa_offset 16
	leaq	_ZL15__fatDeviceText(%rip), %rdi
	call	__cudaRegisterFatBinary@PLT
	movq	%rax, %rdi
	movq	%rax, _ZL20__cudaFatCubinHandle(%rip)
	call	__cudaRegisterFatBinaryEnd@PLT
	leaq	_ZL26__cudaUnregisterBinaryUtilv(%rip), %rdi
	popq	%rdx
	.cfi_def_cfa_offset 8
	jmp	atexit@PLT
	.cfi_endproc
.LFE2066:
	.size	_ZL24__sti____cudaRegisterAllv, .-_ZL24__sti____cudaRegisterAllv
	.section	.init_array,"aw"
	.align 8
	.quad	_ZL24__sti____cudaRegisterAllv
	.section	.text.startup
	.type	_GLOBAL__sub_I_nticks, @function
_GLOBAL__sub_I_nticks:
.LFB2182:
	.cfi_startproc
	endbr64
	movl	nx(%rip), %eax
	imull	ny(%rip), %eax
	cltq
	salq	$2, %rax
	movq	%rax, size(%rip)
	ret
	.cfi_endproc
.LFE2182:
	.size	_GLOBAL__sub_I_nticks, .-_GLOBAL__sub_I_nticks
	.section	.init_array
	.align 8
	.quad	_GLOBAL__sub_I_nticks
	.section	.nvFatBinSegment,"aw"
	.align 8
	.type	_ZL15__fatDeviceText, @object
	.size	_ZL15__fatDeviceText, 24
_ZL15__fatDeviceText:
	.long	1180844977
	.long	1
	.quad	fatbinData
	.quad	0
	.local	_ZL20__cudaFatCubinHandle
	.comm	_ZL20__cudaFatCubinHandle,8,8
	.globl	size
	.bss
	.align 8
	.type	size, @object
	.size	size, 8
size:
	.zero	8
	.globl	ny
	.data
	.align 4
	.type	ny, @object
	.size	ny, 4
ny:
	.long	256
	.globl	nx
	.align 4
	.type	nx, @object
	.size	nx, 4
nx:
	.long	512
	.globl	nticks
	.align 4
	.type	nticks, @object
	.size	nticks, 4
nticks:
	.long	10
	.section	.rodata.cst4,"aM",@progbits,4
	.align 4
.LC0:
	.long	805306368
	.ident	"GCC: (Ubuntu 13.3.0-6ubuntu2~24.04) 13.3.0"
	.section	.note.GNU-stack,"",@progbits
	.section	.note.gnu.property,"a"
	.align 8
	.long	1f - 0f
	.long	4f - 1f
	.long	5
0:
	.string	"GNU"
1:
	.align 8
	.long	0xc0000002
	.long	3f - 2f
2:
	.long	0x3
3:
	.align 8
4:

/*
 * MSU CUDA Course Examples and Exercises.
 *
 * Copyright (c) 2011 Dmitry Mikushin
 *
 * This software is provided 'as-is', without any express or implied warranty.
 * In no event will the authors be held liable for any damages arising 
 * from the use of this software.
 * Permission is granted to anyone to use this software for any purpose, 
 * including commercial applications, and to alter it and redistribute it freely,
 * without any restrictons.
 *
 * This sample demonstates parallel execution using process forking.
 * Each process works on own private data.
 *
 */

#include <hip/hip_runtime.h>
#include <errno.h>
#include <malloc.h>
#include <stdio.h>
#include <unistd.h>

// Perform some dummy 2D field processing on GPU and CPU,
// and compare results.
int pattern2d(int nx, int ny, float* in, float* out, int pid, int step);

int nticks = 10;

// The size of memory region.
int nx = 512, ny = 256;
size_t size = nx * ny * sizeof(float);

int main(int argc, char* argv[])
{
	// Allocate input & output arrays.
	float* input = (float*)malloc(size);
	float* output = (float*)malloc(size);

	// Generate input data array of the
	// specified size.
	long np = nx * ny;
	float invdrandmax = 1.0 / RAND_MAX;
	for (long i = 0; i < np; i++)
		input[i] = rand() * invdrandmax;

	// Call fork to create another process.
	// Standard: "Memory mappings created in the parent
	// shall be retained in the child process."
	pid_t fork_status = fork();

	// From this point two processes are running the same code, if no errors.
	if (fork_status == -1)
	{
		fprintf(stderr, "Cannot fork process, errno = %d\n", errno);
		return errno;
	}

	// By fork return value we can determine the process role:
	// master or child (worker).
	int master = fork_status ? 1 : 0;

	int ndevices = 0;
	hipError_t cuda_status = hipGetDeviceCount(&ndevices);
	if (cuda_status != hipSuccess)
	{
		fprintf(stderr, "Cannot get the cuda device count, status = %d\n",
			cuda_status);
		return cuda_status;
	}
	
	// Return if no cuda devices present.
	if (master)
		printf("%d CUDA device(s) found\n", ndevices);
	if (!ndevices) return 0;

	// Get the process ID.
	int pid = (int)getpid();
	
	// Use different devices, if more than one present.
	if (ndevices > 1)
	{
		int idevice = 1;
		if (master) idevice = 0;
		
		cuda_status = hipSetDevice(idevice);
		if (cuda_status != hipSuccess)
		{
			fprintf(stderr, "Cannot set CUDA device by process %d, status = %d\n",
				pid, cuda_status);
			return cuda_status;
		}
		printf("Process %d uses device #%d\n", pid, idevice);
	}
	
	// Perform some "iterations" on data array private to each process.
	for (int i = 0; i < nticks; i++)
	{
		// Execute function with CUDA kernel.
		int status = pattern2d(nx, ny, input, output, pid, i);
		if (status)
		{
			fprintf(stderr, "Pattern 2D failed by process %d, status = %d\n",
				pid, status);
			return status;
		}
	}
	
	free(input);
	free(output);
	
	return 0;
}

	.text
	.file	"process_fork_cuda.hip"
	.section	.rodata.cst4,"aM",@progbits,4
	.p2align	2, 0x0                          # -- Begin function main
.LCPI0_0:
	.long	0x30000000                      # float 4.65661287E-10
	.text
	.globl	main
	.type	main,@function
main:                                   # @main
	.cfi_startproc
# %bb.0:
	pushq	%rbp
	.cfi_def_cfa_offset 16
	pushq	%r15
	.cfi_def_cfa_offset 24
	pushq	%r14
	.cfi_def_cfa_offset 32
	pushq	%r13
	.cfi_def_cfa_offset 40
	pushq	%r12
	.cfi_def_cfa_offset 48
	pushq	%rbx
	.cfi_def_cfa_offset 56
	pushq	%rax
	.cfi_def_cfa_offset 64
	.cfi_offset %rbx, -56
	.cfi_offset %r12, -48
	.cfi_offset %r13, -40
	.cfi_offset %r14, -32
	.cfi_offset %r15, -24
	.cfi_offset %rbp, -16
	movq	size(%rip), %r14
	movq	%r14, %rdi
	callq	malloc
	movq	%rax, %rbx
	movq	%r14, %rdi
	callq	malloc
	movq	%rax, %r14
	movslq	nx(%rip), %rax
	movslq	ny(%rip), %r15
	imulq	%rax, %r15
	testl	%r15d, %r15d
	jle	.LBB0_3
# %bb.1:                                # %.lr.ph.preheader
	xorl	%r12d, %r12d
.LBB0_2:                                # %.lr.ph
                                        # =>This Inner Loop Header: Depth=1
	callq	rand
	xorps	%xmm0, %xmm0
	cvtsi2ss	%eax, %xmm0
	mulss	.LCPI0_0(%rip), %xmm0
	movss	%xmm0, (%rbx,%r12,4)
	incq	%r12
	cmpq	%r12, %r15
	jne	.LBB0_2
.LBB0_3:                                # %._crit_edge
	callq	fork
	cmpl	$-1, %eax
	je	.LBB0_4
# %bb.5:
	movl	%eax, %r12d
	leaq	4(%rsp), %rdi
	movl	$0, (%rdi)
	callq	hipGetDeviceCount
	testl	%eax, %eax
	jne	.LBB0_6
# %bb.7:
	testl	%r12d, %r12d
	je	.LBB0_9
# %bb.8:
	movl	4(%rsp), %esi
	movl	$.L.str.2, %edi
	xorl	%eax, %eax
	callq	printf
.LBB0_9:
	xorl	%ebp, %ebp
	cmpl	$0, 4(%rsp)
	je	.LBB0_19
# %bb.10:
	callq	getpid
	movl	%eax, %r15d
	cmpl	$2, 4(%rsp)
	jl	.LBB0_13
# %bb.11:
	xorl	%r13d, %r13d
	testl	%r12d, %r12d
	sete	%r13b
	movl	%r13d, %edi
	callq	hipSetDevice
	testl	%eax, %eax
	jne	.LBB0_20
# %bb.12:                               # %.thread
	movl	$.L.str.4, %edi
	movl	%r15d, %esi
	movl	%r13d, %edx
	xorl	%eax, %eax
	callq	printf
.LBB0_13:
	cmpl	$0, nticks(%rip)
	jle	.LBB0_18
# %bb.14:                               # %.lr.ph61.preheader
	xorl	%r12d, %r12d
.LBB0_16:                               # %.lr.ph61
                                        # =>This Inner Loop Header: Depth=1
	movl	nx(%rip), %edi
	movl	ny(%rip), %esi
	movq	%rbx, %rdx
	movq	%r14, %rcx
	movl	%r15d, %r8d
	movl	%r12d, %r9d
	callq	_Z9pattern2diiPfS_ii
	testl	%eax, %eax
	jne	.LBB0_17
# %bb.15:                               #   in Loop: Header=BB0_16 Depth=1
	incl	%r12d
	cmpl	nticks(%rip), %r12d
	jl	.LBB0_16
.LBB0_18:                               # %._crit_edge62
	movq	%rbx, %rdi
	callq	free
	movq	%r14, %rdi
	callq	free
.LBB0_19:
	movl	%ebp, %eax
	addq	$8, %rsp
	.cfi_def_cfa_offset 56
	popq	%rbx
	.cfi_def_cfa_offset 48
	popq	%r12
	.cfi_def_cfa_offset 40
	popq	%r13
	.cfi_def_cfa_offset 32
	popq	%r14
	.cfi_def_cfa_offset 24
	popq	%r15
	.cfi_def_cfa_offset 16
	popq	%rbp
	.cfi_def_cfa_offset 8
	retq
.LBB0_4:
	.cfi_def_cfa_offset 64
	movq	stderr(%rip), %rbx
	callq	__errno_location
	movq	%rax, %r14
	movl	(%rax), %edx
	movl	$.L.str, %esi
	movq	%rbx, %rdi
	xorl	%eax, %eax
	callq	fprintf
	movl	(%r14), %ebp
	jmp	.LBB0_19
.LBB0_6:
	movl	%eax, %ebp
	movq	stderr(%rip), %rdi
	movl	$.L.str.1, %esi
	movl	%eax, %edx
	xorl	%eax, %eax
	callq	fprintf
	jmp	.LBB0_19
.LBB0_17:                               # %.thread55
	movl	%eax, %r13d
	movq	stderr(%rip), %rdi
	movl	$.L.str.5, %esi
	movl	%r15d, %edx
	movl	%eax, %ecx
	xorl	%eax, %eax
	callq	fprintf
	movl	%r13d, %ebp
	jmp	.LBB0_19
.LBB0_20:
	movl	%eax, %r12d
	movq	stderr(%rip), %rdi
	movl	$.L.str.3, %esi
	movl	%r15d, %edx
	movl	%eax, %ecx
	xorl	%eax, %eax
	callq	fprintf
	movl	%r12d, %ebp
	jmp	.LBB0_19
.Lfunc_end0:
	.size	main, .Lfunc_end0-main
	.cfi_endproc
                                        # -- End function
	.type	nticks,@object                  # @nticks
	.data
	.globl	nticks
	.p2align	2, 0x0
nticks:
	.long	10                              # 0xa
	.size	nticks, 4

	.type	nx,@object                      # @nx
	.globl	nx
	.p2align	2, 0x0
nx:
	.long	512                             # 0x200
	.size	nx, 4

	.type	ny,@object                      # @ny
	.globl	ny
	.p2align	2, 0x0
ny:
	.long	256                             # 0x100
	.size	ny, 4

	.type	size,@object                    # @size
	.globl	size
	.p2align	3, 0x0
size:
	.quad	524288                          # 0x80000
	.size	size, 8

	.type	.L.str,@object                  # @.str
	.section	.rodata.str1.1,"aMS",@progbits,1
.L.str:
	.asciz	"Cannot fork process, errno = %d\n"
	.size	.L.str, 33

	.type	.L.str.1,@object                # @.str.1
.L.str.1:
	.asciz	"Cannot get the cuda device count, status = %d\n"
	.size	.L.str.1, 47

	.type	.L.str.2,@object                # @.str.2
.L.str.2:
	.asciz	"%d CUDA device(s) found\n"
	.size	.L.str.2, 25

	.type	.L.str.3,@object                # @.str.3
.L.str.3:
	.asciz	"Cannot set CUDA device by process %d, status = %d\n"
	.size	.L.str.3, 51

	.type	.L.str.4,@object                # @.str.4
.L.str.4:
	.asciz	"Process %d uses device #%d\n"
	.size	.L.str.4, 28

	.type	.L.str.5,@object                # @.str.5
.L.str.5:
	.asciz	"Pattern 2D failed by process %d, status = %d\n"
	.size	.L.str.5, 46

	.type	__hip_cuid_,@object             # @__hip_cuid_
	.bss
	.globl	__hip_cuid_
__hip_cuid_:
	.byte	0                               # 0x0
	.size	__hip_cuid_, 1

	.ident	"AMD clang version 18.0.0git (https://github.com/RadeonOpenCompute/llvm-project roc-6.3.2 25012 e5bf7e55c91490b07c49d8960fa7983d864936c4)"
	.section	".note.GNU-stack","",@progbits
	.addrsig
	.addrsig_sym __hip_cuid_

	.text
	.p2alignl 7, 3214868480
	.fill 96, 4, 3214868480
	.type	__hip_cuid_,@object             ; @__hip_cuid_
	.section	.bss,"aw",@nobits
	.globl	__hip_cuid_
__hip_cuid_:
	.byte	0                               ; 0x0
	.size	__hip_cuid_, 1

	.ident	"AMD clang version 18.0.0git (https://github.com/RadeonOpenCompute/llvm-project roc-6.3.2 25012 e5bf7e55c91490b07c49d8960fa7983d864936c4)"
	.section	".note.GNU-stack","",@progbits
	.addrsig
	.addrsig_sym __hip_cuid_
	.amdgcn_target "amdgcn-amd-amdhsa--gfx1100"
	.amdgpu_metadata
---
amdhsa.kernels:  []
amdhsa.target:   amdgcn-amd-amdhsa--gfx1100
amdhsa.version:
  - 1
  - 2
...

	.end_amdgpu_metadata
