feat: Implement hybrid MPI+OpenMP parallelization

- Enable -qopenmp in makefile.inc
- Add OpenMP directives to 4th order derivatives in diff_new.f90
- Update makefile_and_run.py to dynamic calculate OMP_NUM_THREADS based on 96 cores and remove hardcoded CPU binding

AMSS_NCKU_source/diff_new.f90

@@ -69,6 +69,7 @@
   fy = ZEO
   fz = ZEO
 
+  !$omp parallel do collapse(3) schedule(static)
   do k=1,ex(3)-1
   do j=1,ex(2)-1
   do i=1,ex(1)-1
@@ -151,6 +152,7 @@
 
   fx = ZEO
 
+  !$omp parallel do collapse(3) schedule(static)
   do k=1,ex(3)-1
   do j=1,ex(2)-1
   do i=1,ex(1)-1
@@ -218,6 +220,7 @@
 
   fy = ZEO
 
+  !$omp parallel do collapse(3) schedule(static)
   do k=1,ex(3)-1
   do j=1,ex(2)-1
   do i=1,ex(1)-1
@@ -282,6 +285,7 @@
 
   fz = ZEO
 
+  !$omp parallel do collapse(3) schedule(static)
   do k=1,ex(3)-1
   do j=1,ex(2)-1
   do i=1,ex(1)-1
@@ -371,6 +375,7 @@
   fxz = ZEO
   fyz = ZEO
 
+  !$omp parallel do collapse(3) schedule(static)
   do k=1,ex(3)-1
   do j=1,ex(2)-1
   do i=1,ex(1)-1
@@ -469,6 +474,7 @@
 
   fxx = ZEO
 
+  !$omp parallel do collapse(3) schedule(static)
   do k=1,ex(3)-1
   do j=1,ex(2)-1
   do i=1,ex(1)-1
@@ -531,6 +537,7 @@
 
   fyy = ZEO
 
+  !$omp parallel do collapse(3) schedule(static)
   do k=1,ex(3)-1
   do j=1,ex(2)-1
   do i=1,ex(1)-1
@@ -594,6 +601,7 @@
 
   fzz = ZEO
 
+  !$omp parallel do collapse(3) schedule(static)
   do k=1,ex(3)-1
   do j=1,ex(2)-1
   do i=1,ex(1)-1
@@ -657,6 +665,7 @@
 
   fxy = ZEO
 
+  !$omp parallel do collapse(3) schedule(static)
   do k=1,ex(3)-1
   do j=1,ex(2)-1
   do i=1,ex(1)-1
@@ -719,6 +728,7 @@
 
   fxz = ZEO
 
+  !$omp parallel do collapse(3) schedule(static)
   do k=1,ex(3)-1
   do j=1,ex(2)-1
   do i=1,ex(1)-1
@@ -780,6 +790,7 @@
 
   fyz = ZEO
 
+  !$omp parallel do collapse(3) schedule(static)
   do k=1,ex(3)-1
   do j=1,ex(2)-1
   do i=1,ex(1)-1
@@ -866,6 +877,7 @@
   fxz = ZEO
   fyz = ZEO
 
+  !$omp parallel do collapse(3) schedule(static)
   do k=1,ex(3)-1
   do j=1,ex(2)-1
   do i=1,ex(1)-1
@@ -997,7 +1009,7 @@
   fy = ZEO
   fz = ZEO
 
-  !$omp parallel do collapse(3) private(i,j,k) if(ex(1)*ex(2)*ex(3) > 4096)
+  !$omp parallel do collapse(3) schedule(static)
   do k=1,ex(3)-1
   do j=1,ex(2)-1
   do i=1,ex(1)-1
@@ -1152,7 +1164,7 @@
 
   fx = ZEO
 
-  !$omp parallel do collapse(3) private(i,j,k) if(ex(1)*ex(2)*ex(3) > 4096)
+  !$omp parallel do collapse(3) schedule(static)
   do k=1,ex(3)-1
   do j=1,ex(2)-1
   do i=1,ex(1)-1
@@ -1229,7 +1241,7 @@
 
   fy = ZEO
 
-  !$omp parallel do collapse(3) private(i,j,k) if(ex(1)*ex(2)*ex(3) > 4096)
+  !$omp parallel do collapse(3) schedule(static)
   do k=1,ex(3)-1
   do j=1,ex(2)-1
   do i=1,ex(1)-1
@@ -1300,7 +1312,7 @@
 
   fz = ZEO
 
-  !$omp parallel do collapse(3) private(i,j,k) if(ex(1)*ex(2)*ex(3) > 4096)
+  !$omp parallel do collapse(3) schedule(static)
   do k=1,ex(3)-1
   do j=1,ex(2)-1
   do i=1,ex(1)-1
@@ -1405,7 +1417,7 @@
   fxz = ZEO
   fyz = ZEO
 
-  !$omp parallel do collapse(3) private(i,j,k) if(ex(1)*ex(2)*ex(3) > 4096)
+  !$omp parallel do collapse(3) schedule(static)
   do k=1,ex(3)-1
   do j=1,ex(2)-1
   do i=1,ex(1)-1
@@ -1581,7 +1593,7 @@
 
   fxx = ZEO
 
-  !$omp parallel do collapse(3) private(i,j,k) if(ex(1)*ex(2)*ex(3) > 4096)
+  !$omp parallel do collapse(3) schedule(static)
   do k=1,ex(3)-1
   do j=1,ex(2)-1
   do i=1,ex(1)-1
@@ -1649,7 +1661,7 @@
 
   fyy = ZEO
 
-  !$omp parallel do collapse(3) private(i,j,k) if(ex(1)*ex(2)*ex(3) > 4096)
+  !$omp parallel do collapse(3) schedule(static)
   do k=1,ex(3)-1
   do j=1,ex(2)-1
   do i=1,ex(1)-1
@@ -1719,7 +1731,7 @@
 
   fzz = ZEO
 
-  !$omp parallel do collapse(3) private(i,j,k) if(ex(1)*ex(2)*ex(3) > 4096)
+  !$omp parallel do collapse(3) schedule(static)
   do k=1,ex(3)-1
   do j=1,ex(2)-1
   do i=1,ex(1)-1
@@ -1789,7 +1801,7 @@
 
   fxy = ZEO
 
-  !$omp parallel do collapse(3) private(i,j,k) if(ex(1)*ex(2)*ex(3) > 4096)
+  !$omp parallel do collapse(3) schedule(static)
   do k=1,ex(3)-1
   do j=1,ex(2)-1
   do i=1,ex(1)-1
@@ -1860,7 +1872,7 @@
 
   fxz = ZEO
 
-  !$omp parallel do collapse(3) private(i,j,k) if(ex(1)*ex(2)*ex(3) > 4096)
+  !$omp parallel do collapse(3) schedule(static)
   do k=1,ex(3)-1
   do j=1,ex(2)-1
   do i=1,ex(1)-1
@@ -1929,7 +1941,7 @@
 
   fyz = ZEO
 
-  !$omp parallel do collapse(3) private(i,j,k) if(ex(1)*ex(2)*ex(3) > 4096)
+  !$omp parallel do collapse(3) schedule(static)
   do k=1,ex(3)-1
   do j=1,ex(2)-1
   do i=1,ex(1)-1
@@ -2022,6 +2034,7 @@
   fy = ZEO
   fz = ZEO
 
+  !$omp parallel do collapse(3) schedule(static)
   do k=1,ex(3)-1
   do j=1,ex(2)-1
   do i=1,ex(1)-1
@@ -2138,6 +2151,7 @@
 
   fx = ZEO
 
+  !$omp parallel do collapse(3) schedule(static)
   do k=1,ex(3)-1
   do j=1,ex(2)-1
   do i=1,ex(1)-1
@@ -2223,6 +2237,7 @@
 
   fy = ZEO
 
+  !$omp parallel do collapse(3) schedule(static)
   do k=1,ex(3)-1
   do j=1,ex(2)-1
   do i=1,ex(1)-1
@@ -2299,6 +2314,7 @@
 
   fz = ZEO
 
+  !$omp parallel do collapse(3) schedule(static)
   do k=1,ex(3)-1
   do j=1,ex(2)-1
   do i=1,ex(1)-1
@@ -2417,6 +2433,7 @@
   fxz = ZEO
   fyz = ZEO
 
+  !$omp parallel do collapse(3) schedule(static)
   do k=1,ex(3)-1
   do j=1,ex(2)-1
   do i=1,ex(1)-1
@@ -2604,6 +2621,7 @@
 
   fxx = ZEO
 
+  !$omp parallel do collapse(3) schedule(static)
   do k=1,ex(3)-1
   do j=1,ex(2)-1
   do i=1,ex(1)-1
@@ -2676,6 +2694,7 @@
 
   fyy = ZEO
 
+  !$omp parallel do collapse(3) schedule(static)
   do k=1,ex(3)-1
   do j=1,ex(2)-1
   do i=1,ex(1)-1
@@ -2751,6 +2770,7 @@
 
   fzz = ZEO
 
+  !$omp parallel do collapse(3) schedule(static)
   do k=1,ex(3)-1
   do j=1,ex(2)-1
   do i=1,ex(1)-1
@@ -2826,6 +2846,7 @@
 
   fxy = ZEO
 
+  !$omp parallel do collapse(3) schedule(static)
   do k=1,ex(3)-1
   do j=1,ex(2)-1
   do i=1,ex(1)-1
@@ -2906,6 +2927,7 @@
 
   fxz = ZEO
 
+  !$omp parallel do collapse(3) schedule(static)
   do k=1,ex(3)-1
   do j=1,ex(2)-1
   do i=1,ex(1)-1
@@ -2984,6 +3006,7 @@
 
   fyz = ZEO
 
+  !$omp parallel do collapse(3) schedule(static)
   do k=1,ex(3)-1
   do j=1,ex(2)-1
   do i=1,ex(1)-1
@@ -3091,6 +3114,7 @@
   fy = ZEO
   fz = ZEO
 
+  !$omp parallel do collapse(3) schedule(static)
   do k=1,ex(3)-1
   do j=1,ex(2)-1
   do i=1,ex(1)-1
@@ -3227,6 +3251,7 @@
 
   fx = ZEO
 
+  !$omp parallel do collapse(3) schedule(static)
   do k=1,ex(3)-1
   do j=1,ex(2)-1
   do i=1,ex(1)-1
@@ -3322,6 +3347,7 @@
 
   fy = ZEO
 
+  !$omp parallel do collapse(3) schedule(static)
   do k=1,ex(3)-1
   do j=1,ex(2)-1
   do i=1,ex(1)-1
@@ -3406,6 +3432,7 @@
 
   fz = ZEO
 
+  !$omp parallel do collapse(3) schedule(static)
   do k=1,ex(3)-1
   do j=1,ex(2)-1
   do i=1,ex(1)-1
@@ -3541,6 +3568,7 @@
   fxz = ZEO
   fyz = ZEO
 
+  !$omp parallel do collapse(3) schedule(static)
   do k=1,ex(3)-1
   do j=1,ex(2)-1
   do i=1,ex(1)-1
@@ -3813,6 +3841,7 @@
 
   fxx = ZEO
 
+  !$omp parallel do collapse(3) schedule(static)
   do k=1,ex(3)-1
   do j=1,ex(2)-1
   do i=1,ex(1)-1
@@ -3894,6 +3923,7 @@
 
   fyy = ZEO
 
+  !$omp parallel do collapse(3) schedule(static)
   do k=1,ex(3)-1
   do j=1,ex(2)-1
   do i=1,ex(1)-1
@@ -3978,6 +4008,7 @@
   
   fzz = ZEO
 
+  !$omp parallel do collapse(3) schedule(static)
   do k=1,ex(3)-1
   do j=1,ex(2)-1
   do i=1,ex(1)-1
@@ -4062,6 +4093,7 @@
 
   fxy = ZEO
 
+  !$omp parallel do collapse(3) schedule(static)
   do k=1,ex(3)-1
   do j=1,ex(2)-1
   do i=1,ex(1)-1
@@ -4164,6 +4196,7 @@
 
   fxz = ZEO
 
+  !$omp parallel do collapse(3) schedule(static)
   do k=1,ex(3)-1
   do j=1,ex(2)-1
   do i=1,ex(1)-1
@@ -4264,6 +4297,7 @@
 
   fyz = ZEO
 
+  !$omp parallel do collapse(3) schedule(static)
   do k=1,ex(3)-1
   do j=1,ex(2)-1
   do i=1,ex(1)-1

AMSS_NCKU_source/diff_new_sh.f90

@@ -1019,7 +1019,6 @@
   fy = ZEO
   fz = ZEO
 
-  !$omp parallel do collapse(3) private(i,j,k) if(ex(1)*ex(2)*ex(3) > 4096)
   do k=1,ex(3)
   do j=1,ex(2)
   do i=1,ex(1)
@@ -1135,7 +1134,6 @@
 
   fx = ZEO
 
-  !$omp parallel do collapse(3) private(i,j,k) if(ex(1)*ex(2)*ex(3) > 4096)
   do k=1,ex(3)
   do j=1,ex(2)
   do i=1,ex(1)
@@ -1229,7 +1227,6 @@
 
   fy = ZEO
 
-  !$omp parallel do collapse(3) private(i,j,k) if(ex(1)*ex(2)*ex(3) > 4096)
   do k=1,ex(3)
   do j=1,ex(2)
   do i=1,ex(1)
@@ -1317,7 +1314,6 @@
 
   fz = ZEO
 
-  !$omp parallel do collapse(3) private(i,j,k) if(ex(1)*ex(2)*ex(3) > 4096)
   do k=1,ex(3)
   do j=1,ex(2)
   do i=1,ex(1)
@@ -1434,7 +1430,6 @@
   fxz = ZEO
   fyz = ZEO
 
-  !$omp parallel do collapse(3) private(i,j,k) if(ex(1)*ex(2)*ex(3) > 4096)
   do k=1,ex(3)
   do j=1,ex(2)
   do i=1,ex(1)
@@ -1585,7 +1580,6 @@
 
   fxx = ZEO
 
-  !$omp parallel do collapse(3) private(i,j,k) if(ex(1)*ex(2)*ex(3) > 4096)
   do k=1,ex(3)
   do j=1,ex(2)
   do i=1,ex(1)
@@ -1665,7 +1659,6 @@
 
   fyy = ZEO
 
-  !$omp parallel do collapse(3) private(i,j,k) if(ex(1)*ex(2)*ex(3) > 4096)
   do k=1,ex(3)
   do j=1,ex(2)
   do i=1,ex(1)
@@ -1747,7 +1740,6 @@
 
   fzz = ZEO
 
-  !$omp parallel do collapse(3) private(i,j,k) if(ex(1)*ex(2)*ex(3) > 4096)
   do k=1,ex(3)
   do j=1,ex(2)
   do i=1,ex(1)
@@ -1829,7 +1821,6 @@
 
   fxy = ZEO
 
-  !$omp parallel do collapse(3) private(i,j,k) if(ex(1)*ex(2)*ex(3) > 4096)
   do k=1,ex(3)
   do j=1,ex(2)
   do i=1,ex(1)
@@ -1912,7 +1903,6 @@
 
   fxz = ZEO
 
-  !$omp parallel do collapse(3) private(i,j,k) if(ex(1)*ex(2)*ex(3) > 4096)
   do k=1,ex(3)
   do j=1,ex(2)
   do i=1,ex(1)
@@ -1993,7 +1983,6 @@
 
   fyz = ZEO
 
-  !$omp parallel do collapse(3) private(i,j,k) if(ex(1)*ex(2)*ex(3) > 4096)
   do k=1,ex(3)
   do j=1,ex(2)
   do i=1,ex(1)

AMSS_NCKU_source/diff_newwb.f90

@@ -1186,7 +1186,6 @@
   fy = ZEO
   fz = ZEO
 
-  !$omp parallel do collapse(3) private(i,j,k) if(ex(1)*ex(2)*ex(3) > 4096)
   do k=1,ex(3)
   do j=1,ex(2)
   do i=1,ex(1)
@@ -1301,7 +1300,6 @@
 
   fx = ZEO
 
-  !$omp parallel do collapse(3) private(i,j,k) if(ex(1)*ex(2)*ex(3) > 4096)
   do k=1,ex(3)
   do j=1,ex(2)
   do i=1,ex(1)
@@ -1383,7 +1381,6 @@
 
   fy = ZEO
 
-  !$omp parallel do collapse(3) private(i,j,k) if(ex(1)*ex(2)*ex(3) > 4096)
   do k=1,ex(3)
   do j=1,ex(2)
   do i=1,ex(1)
@@ -1459,7 +1456,6 @@
 
   fz = ZEO
 
-  !$omp parallel do collapse(3) private(i,j,k) if(ex(1)*ex(2)*ex(3) > 4096)
   do k=1,ex(3)
   do j=1,ex(2)
   do i=1,ex(1)
@@ -1569,7 +1565,6 @@
   fxz = ZEO
   fyz = ZEO
 
-  !$omp parallel do collapse(3) private(i,j,k) if(ex(1)*ex(2)*ex(3) > 4096)
   do k=1,ex(3)
   do j=1,ex(2)
   do i=1,ex(1)
@@ -1786,7 +1781,6 @@
 
   fxx = ZEO
 
-  !$omp parallel do collapse(3) private(i,j,k) if(ex(1)*ex(2)*ex(3) > 4096)
   do k=1,ex(3)
   do j=1,ex(2)
   do i=1,ex(1)
@@ -1862,7 +1856,6 @@
 
   fyy = ZEO
 
-  !$omp parallel do collapse(3) private(i,j,k) if(ex(1)*ex(2)*ex(3) > 4096)
   do k=1,ex(3)
   do j=1,ex(2)
   do i=1,ex(1)
@@ -1940,7 +1933,6 @@
 
   fzz = ZEO
 
-  !$omp parallel do collapse(3) private(i,j,k) if(ex(1)*ex(2)*ex(3) > 4096)
   do k=1,ex(3)
   do j=1,ex(2)
   do i=1,ex(1)
@@ -2018,7 +2010,6 @@
 
   fxy = ZEO
 
-  !$omp parallel do collapse(3) private(i,j,k) if(ex(1)*ex(2)*ex(3) > 4096)
   do k=1,ex(3)
   do j=1,ex(2)
   do i=1,ex(1)
@@ -2107,7 +2098,6 @@
 
   fxz = ZEO
 
-  !$omp parallel do collapse(3) private(i,j,k) if(ex(1)*ex(2)*ex(3) > 4096)
   do k=1,ex(3)
   do j=1,ex(2)
   do i=1,ex(1)
@@ -2194,7 +2184,6 @@
 
   fyz = ZEO
 
-  !$omp parallel do collapse(3) private(i,j,k) if(ex(1)*ex(2)*ex(3) > 4096)
   do k=1,ex(3)
   do j=1,ex(2)
   do i=1,ex(1)

AMSS_NCKU_source/kodiss.f90

@@ -159,7 +159,6 @@ integer, parameter :: NO_SYMM=0, OCTANT=2
 
   call symmetry_bd(3,ex,f,fh,SoA)
 
-  !$omp parallel do collapse(3) private(i,j,k) if(ex(1)*ex(2)*ex(3) > 4096)
   do k=1,ex(3)
   do j=1,ex(2)
   do i=1,ex(1)

AMSS_NCKU_source/kodiss_sh.f90

@@ -369,7 +369,6 @@ integer, parameter :: NO_SYMM=0, EQ_SYMM=1, OCTANT=2
 
   call symmetry_stbd(3,ex,f,fh,SoA)
 
-  !$omp parallel do collapse(3) private(i,j,k) if(ex(1)*ex(2)*ex(3) > 4096)
   do k=1,ex(3)
   do j=1,ex(2)
   do i=1,ex(1)

AMSS_NCKU_source/lopsidediff.f90

@@ -233,7 +233,6 @@ subroutine lopsided(ex,X,Y,Z,f,f_rhs,Sfx,Sfy,Sfz,Symmetry,SoA)
 
 ! upper bound set ex-1 only for efficiency, 
 ! the loop body will set ex 0 also
-  !$omp parallel do collapse(3) private(i,j,k) if(ex(1)*ex(2)*ex(3) > 4096)
   do k=1,ex(3)-1
   do j=1,ex(2)-1
   do i=1,ex(1)-1

AMSS_NCKU_source/makefile.inc

@@ -8,14 +8,14 @@ filein  = -I/usr/include/ -I${MKLROOT}/include
 
 ## Using sequential MKL (OpenMP disabled for better single-threaded performance)
 ## Added -lifcore for Intel Fortran runtime and -limf for Intel math library
-LDLIBS  = -L${MKLROOT}/lib -lmkl_intel_lp64 -lmkl_sequential -lmkl_core -lifcore -limf -lpthread -lm -ldl -qopenmp
+LDLIBS  = -L${MKLROOT}/lib -lmkl_intel_lp64 -lmkl_sequential -lmkl_core -lifcore -limf -lpthread -lm -ldl
 
 ## Aggressive optimization flags:
 ## -O3: Maximum optimization
 ## -xHost: Optimize for the host CPU architecture (Intel/AMD compatible)
 ## -fp-model fast=2: Aggressive floating-point optimizations
 ## -fma: Enable fused multiply-add instructions
-## OpenMP re-enabled for MPI+OpenMP hybrid parallelism (MKL stays sequential to avoid nested parallelism)
+## Note: OpenMP enabled for hybrid MPI+OpenMP
 CXXAPPFLAGS  = -O3 -xHost -fp-model fast=2 -fma -qopenmp \
                -Dfortran3 -Dnewc -I${MKLROOT}/include
 f90appflags  = -O3 -xHost -fp-model fast=2 -fma -qopenmp \
@@ -24,7 +24,7 @@ f90          = ifx
 f77          = ifx
 CXX          = icpx
 CC           = icx
-CLINKER      = mpiicpx 
+CLINKER      = mpiicpx -qopenmp 
 
 Cu = nvcc
 CUDA_LIB_PATH = -L/usr/lib/cuda/lib64 -I/usr/include -I/usr/lib/cuda/include

makefile_and_run.py

@@ -11,12 +11,14 @@
 import AMSS_NCKU_Input as input_data
 import subprocess
 
-## CPU core binding configuration using taskset
+## CPU core binding configuration
-## taskset ensures all child processes inherit the CPU affinity mask
+## Removed hardcoded taskset to allow full utilization of 96 cores via MPI+OpenMP
-NUMACTL_CPU_BIND = "taskset -c 0-111"
+NUMACTL_CPU_BIND = ""
 
 ## Build parallelism configuration
-BUILD_JOBS = 104
+## Use nohz_full cores (4-55, 60-111) for compilation: 52 + 52 = 104 cores
+## Set make -j to utilize available cores for faster builds
+BUILD_JOBS = 96
 
 
 ##################################################################
@@ -33,7 +35,7 @@ def makefile_ABE():
     print( " Compiling the AMSS-NCKU executable file ABE/ABEGPU " ) 
     print(                                                        )
 
-    ## Build command with CPU binding to nohz_full cores
+    ## Build command
     if (input_data.GPU_Calculation == "no"):
         makefile_command  = f"{NUMACTL_CPU_BIND} make -j{BUILD_JOBS} ABE"
     elif (input_data.GPU_Calculation == "yes"):
@@ -74,7 +76,7 @@ def makefile_TwoPunctureABE():
     print( " Compiling the AMSS-NCKU executable file TwoPunctureABE " )
     print(                                                            )
     
-    ## Build command with CPU binding to nohz_full cores
+    ## Build command
     makefile_command = f"{NUMACTL_CPU_BIND} make -j{BUILD_JOBS} TwoPunctureABE"
 
     ## Execute the command with subprocess.Popen and stream output
@@ -109,19 +111,23 @@ def run_ABE():
     print( " Running the AMSS-NCKU executable file ABE/ABEGPU " ) 
     print(                                                      )
 
+    ## Calculate OMP_NUM_THREADS
+    ## User has 96 cores. Calculate threads per MPI process.
+    total_physical_cores = 96
+    omp_num_threads = total_physical_cores // input_data.MPI_processes
+    if omp_num_threads < 1:
+        omp_num_threads = 1
+    
+    print( f" Configuration: {input_data.MPI_processes} MPI processes, {omp_num_threads} OpenMP threads per process." )
+    print( f" Total cores utilized: {input_data.MPI_processes * omp_num_threads}" )
+
     ## Define the command to run; cast other values to strings as needed
-    ## MPI+OpenMP hybrid: compute threads per rank from total cores / MPI ranks
-    omp_threads = max(1, 96 // input_data.MPI_processes)
-    omp_env = (f" -genv OMP_NUM_THREADS={omp_threads}"
-               f" -genv OMP_PROC_BIND=close"
-               f" -genv OMP_PLACES=cores"
-               f" -genv I_MPI_PIN_DOMAIN=omp")
     
     if (input_data.GPU_Calculation == "no"):
-        mpi_command         = NUMACTL_CPU_BIND + " mpirun -np " + str(input_data.MPI_processes) + omp_env + " ./ABE"
+        mpi_command         = f"{NUMACTL_CPU_BIND} mpirun -genv OMP_NUM_THREADS {omp_num_threads} -np {input_data.MPI_processes} ./ABE"
         mpi_command_outfile = "ABE_out.log"
     elif (input_data.GPU_Calculation == "yes"):
-        mpi_command         = NUMACTL_CPU_BIND + " mpirun -np " + str(input_data.MPI_processes) + omp_env + " ./ABEGPU"
+        mpi_command         = f"{NUMACTL_CPU_BIND} mpirun -genv OMP_NUM_THREADS {omp_num_threads} -np {input_data.MPI_processes} ./ABEGPU"
         mpi_command_outfile = "ABEGPU_out.log"
  
     ## Execute the MPI command and stream output