Switch back to Intel toolchain as the default option

Seems that Intel MPI also supports CUDA-aware by setting I_MPI_OFFLOAD to 1. Besides, I_MPI_OFFLOAD_IPC=0 is needed to avoid segfaults.

AMSS_NCKU_source/makefile.inc

@@ -1,7 +1,7 @@
 ## Toolchain selection
 ##   nvhpc : NVIDIA HPC SDK + CUDA-aware MPI (default)
 ##   intel : Intel oneAPI toolchain (legacy path)
-TOOLCHAIN ?= nvhpc
+TOOLCHAIN ?= intel
 
 ## PGO build mode switch (ABE only; TwoPunctureABE always uses opt flags)
 ##   opt        : (default) maximum performance with PGO profile-guided optimization
@@ -88,4 +88,4 @@ endif
 
 Cu = $(NVHPC_ROOT)/compilers/bin/nvcc
 CUDA_LIB_PATH = -L$(CUDA_HOME)/lib64 -I$(CUDA_HOME)/include
-CUDA_APP_FLAGS = -c -g -O3 --ptxas-options=-v -Dfortran3 -Dnewc -arch=$(CUDA_ARCH)
+CUDA_APP_FLAGS = -c -g -O3 --ptxas-options=-v -Dfortran3 -Dnewc -arch=$(CUDA_ARCH)

makefile_and_run.py

@@ -266,7 +266,7 @@ def run_ABE():
         #mpi_command         = " mpirun -np " + str(input_data.MPI_processes) + " ./ABE"
         mpi_command_outfile = "ABE_out.log"
     elif (input_data.GPU_Calculation == "yes"):
-        mpi_command         = NUMACTL_CPU_BIND + " mpirun -np " + str(input_data.MPI_processes) + " ./ABE_CUDA"
+        mpi_command         = NUMACTL_CPU_BIND + " I_MPI_OFFLOAD=1 I_MPI_OFFLOAD_IPC=0 mpirun -np " + str(input_data.MPI_processes) + " ./ABE_CUDA"
         mpi_command_outfile = "ABEGPU_out.log"
         mpi_env = _gpu_runtime_env()
         started_mps = _start_cuda_mps_if_requested(mpi_env)