Stabilize EScalar CUDA fallback path

Rewritten done by Codex.
This still has errors, do not pick this one now.

AMSS_NCKU_Input.py

@@ -16,9 +16,9 @@ import numpy
 File_directory   = "GW150914"                    ## output file directory
 Output_directory = "binary_output"               ## binary data file directory
                                                  ## The file directory name should not be too long
-MPI_processes    = 64                             ## number of mpi processes used in the simulation
+MPI_processes    = 2                             ## number of mpi processes used in the simulation
 
-GPU_Calculation  = "no"                          ## Use GPU or not 
+GPU_Calculation  = "yes"                          ## Use GPU or not 
                                                  ## (prefer "no" in the current version, because the GPU part may have bugs when integrated in this Python interface)
 CPU_Part         = 1.0
 GPU_Part         = 0.0
@@ -31,7 +31,7 @@ GPU_Part         = 0.0
 ## Setting the physical system and numerical method
 
 Symmetry                 = "equatorial-symmetry"   ## Symmetry of System: choose equatorial-symmetry、no-symmetry、octant-symmetry
-Equation_Class           = "BSSN"                  ## Evolution Equation: choose "BSSN", "BSSN-EScalar", "BSSN-EM", "Z4C" 
+Equation_Class           = "BSSN-EScalar"                  ## Evolution Equation: choose "BSSN", "BSSN-EScalar", "BSSN-EM", "Z4C"
                                                    ## If "BSSN-EScalar" is chosen, it is necessary to set other parameters below
 Initial_Data_Method      = "Ansorg-TwoPuncture"    ## initial data method: choose "Ansorg-TwoPuncture", "Lousto-Analytical", "Cao-Analytical", "KerrSchild-Analytical"
 Time_Evolution_Method    = "runge-kutta-45"        ## time evolution method: choose "runge-kutta-45"

AMSS_NCKU_Program.py

@@ -58,31 +58,36 @@ File_directory = os.path.join(input_data.File_directory)
 
 ## If the specified output directory exists, ask the user whether to continue
 if os.path.exists(File_directory):
-    print( " Output dictionary has been existed !!!  "                                                              )
-    print( " If you want to overwrite the existing file directory, please input 'continue' in the terminal !! "     ) 
-    print( " If you want to retain the existing file directory, please input 'stop' in the terminal to stop the "   ) 
-    print( " simulation. Then you can reset the output dictionary in the input script file AMSS_NCKU_Input.py !!! " )
-    print(                                                                                                          )
-    ## Prompt whether to overwrite the existing directory
-    while True:
-        try:
-            inputvalue = input()
-            ## If the user agrees to overwrite, proceed and remove the existing directory
-            if ( inputvalue == "continue" ):
-                print( " Continue the calculation !!! " )
-                print(                                  )
-                break  
-            ## If the user chooses not to overwrite, exit and keep the existing directory
-            elif ( inputvalue == "stop" ):
-                print( " Stop the calculation !!! "    )
-                sys.exit() 
-            ## If the user input is invalid, prompt again
-            else:
+    auto_overwrite = str(getattr(input_data, "Auto_Overwrite_Output", "yes")).strip().lower()
+    if auto_overwrite in ("1", "yes", "y", "true", "on", "continue"):
+        print( " Output dictionary has been existed; Auto_Overwrite_Output=yes, continue the calculation. " )
+        print(                                                                                              )
+    else:
+        print( " Output dictionary has been existed !!!  "                                                              )
+        print( " If you want to overwrite the existing file directory, please input 'continue' in the terminal !! "     ) 
+        print( " If you want to retain the existing file directory, please input 'stop' in the terminal to stop the "   ) 
+        print( " simulation. Then you can reset the output dictionary in the input script file AMSS_NCKU_Input.py !!! " )
+        print(                                                                                                          )
+        ## Prompt whether to overwrite the existing directory
+        while True:
+            try:
+                inputvalue = input()
+                ## If the user agrees to overwrite, proceed and remove the existing directory
+                if ( inputvalue == "continue" ):
+                    print( " Continue the calculation !!! " )
+                    print(                                  )
+                    break  
+                ## If the user chooses not to overwrite, exit and keep the existing directory
+                elif ( inputvalue == "stop" ):
+                    print( " Stop the calculation !!! "    )
+                    sys.exit() 
+                ## If the user input is invalid, prompt again
+                else:
+                    print( " Please input your choice !!! "                   )
+                    print( " Input 'continue' or 'stop' in the terminal !!! " )
+            except ValueError:
                 print( " Please input your choice !!! "                   )
                 print( " Input 'continue' or 'stop' in the terminal !!! " )
-        except ValueError:
-            print( " Please input your choice !!! "                   )
-            print( " Input 'continue' or 'stop' in the terminal !!! " )
         
 ## Remove the existing output directory if present
 shutil.rmtree(File_directory, ignore_errors=True)
@@ -174,14 +179,11 @@ import generate_macrodef
 generate_macrodef.generate_macrodef_h()
 print( " AMSS-NCKU macro file macrodef.h has been generated. " )
      
-generate_macrodef.generate_macrodef_fh()
-print( " AMSS-NCKU macro file macrodef.fh has been generated. " )
-
-generate_macrodef.generate_build_config()
-print( " AMSS-NCKU build config AMSS_NCKU_build.mk has been generated. " )
-
-
-##################################################################
+generate_macrodef.generate_macrodef_fh()
+print( " AMSS-NCKU macro file macrodef.fh has been generated. " )
+
+
+##################################################################
 
 # Compile the AMSS-NCKU program according to user requirements
 
@@ -220,13 +222,11 @@ shutil.copytree(AMSS_NCKU_source_path, AMSS_NCKU_source_copy)
 
 # Copy the generated macro files into the AMSS_NCKU source folder
 
-macrodef_h_path  = os.path.join(File_directory, "macrodef.h")
-macrodef_fh_path = os.path.join(File_directory, "macrodef.fh")
-build_config_path = os.path.join(File_directory, "AMSS_NCKU_build.mk")
-
-shutil.copy2(macrodef_h_path,  AMSS_NCKU_source_copy)
-shutil.copy2(macrodef_fh_path, AMSS_NCKU_source_copy)
-shutil.copy2(build_config_path, AMSS_NCKU_source_copy)
+macrodef_h_path  = os.path.join(File_directory, "macrodef.h") 
+macrodef_fh_path = os.path.join(File_directory, "macrodef.fh") 
+
+shutil.copy2(macrodef_h_path,  AMSS_NCKU_source_copy)
+shutil.copy2(macrodef_fh_path, AMSS_NCKU_source_copy)
 
 # Notes on copying files:
 # shutil.copy2 preserves file metadata such as modification time.
@@ -263,7 +263,7 @@ print()
 if (input_data.GPU_Calculation == "no"):
     ABE_file = os.path.join(AMSS_NCKU_source_copy, "ABE")
 elif (input_data.GPU_Calculation == "yes"):
-    ABE_file = os.path.join(AMSS_NCKU_source_copy, "ABEGPU")
+    ABE_file = os.path.join(AMSS_NCKU_source_copy, "ABE_CUDA")
 
 if not os.path.exists( ABE_file ):
     print(                                                                                                  )

AMSS_NCKU_Program_Plot.py

@@ -1,100 +0,0 @@
-##################################################################
-##
-## AMSS-NCKU Plot-Only Restart Script
-## Author: Xiaoqu / Claude
-## 2026/05/12
-##
-## This script checks for existing output data from AMSS_NCKU_Program.py.
-## If data exists, it skips all computation and goes directly to plotting,
-## saving time when plotting was interrupted.
-## If no data is found, it exits with a message.
-##
-##################################################################
-
-## Guard against re-execution by multiprocessing child processes.
-if __name__ != '__main__':
-    import sys as _sys
-    _sys.exit(0)
-
-
-import os
-import sys
-
-import AMSS_NCKU_Input as input_data
-
-##################################################################
-
-## Construct paths from input configuration
-File_directory         = os.path.join(input_data.File_directory)
-output_directory       = os.path.join(File_directory, "AMSS_NCKU_output")
-binary_results_directory = os.path.join(output_directory, input_data.Output_directory)
-figure_directory       = os.path.join(File_directory, "figure")
-
-##################################################################
-
-## Check whether the required output data files exist
-
-required_files = [
-    os.path.join(binary_results_directory, "bssn_BH.dat"),
-    os.path.join(binary_results_directory, "bssn_ADMQs.dat"),
-    os.path.join(binary_results_directory, "bssn_psi4.dat"),
-    os.path.join(binary_results_directory, "bssn_constraint.dat"),
-]
-
-missing_files = [f for f in required_files if not os.path.exists(f)]
-
-if missing_files:
-    print(" No existing AMSS_NCKU_Program.py output data found. ")
-    print(" The following required files are missing: ")
-    for f in missing_files:
-        print(f"   {f}")
-    print()
-    print(" Please run AMSS_NCKU_Program.py first to generate the simulation data. ")
-    print(" Exiting. ")
-    sys.exit(1)
-
-print(" Found existing AMSS_NCKU_Program.py output data. "          )
-print(" Skipping all computation and going directly to plotting. " )
-print()
-
-## Ensure the figure directory exists (it should, but be safe)
-os.makedirs(figure_directory, exist_ok=True)
-
-##################################################################
-
-## Plot the AMSS-NCKU program results
-
-import plot_xiaoqu
-import plot_GW_strain_amplitude_xiaoqu
-from parallel_plot_helper import run_plot_tasks_parallel
-
-plot_tasks = []
-
-## Plot black hole trajectory
-plot_tasks.append((plot_xiaoqu.generate_puncture_orbit_plot,   (binary_results_directory, figure_directory)))
-plot_tasks.append((plot_xiaoqu.generate_puncture_orbit_plot3D, (binary_results_directory, figure_directory)))
-
-## Plot black hole separation vs. time
-plot_tasks.append((plot_xiaoqu.generate_puncture_distence_plot, (binary_results_directory, figure_directory)))
-
-## Plot gravitational waveforms (psi4 and strain amplitude)
-for i in range(input_data.Detector_Number):
-    plot_tasks.append((plot_xiaoqu.generate_gravitational_wave_psi4_plot, (binary_results_directory, figure_directory, i)))
-    plot_tasks.append((plot_GW_strain_amplitude_xiaoqu.generate_gravitational_wave_amplitude_plot, (binary_results_directory, figure_directory, i)))
-
-## Plot ADM mass evolution
-for i in range(input_data.Detector_Number):
-    plot_tasks.append((plot_xiaoqu.generate_ADMmass_plot, (binary_results_directory, figure_directory, i)))
-
-## Plot Hamiltonian constraint violation over time
-for i in range(input_data.grid_level):
-    plot_tasks.append((plot_xiaoqu.generate_constraint_check_plot, (binary_results_directory, figure_directory, i)))
-
-run_plot_tasks_parallel(plot_tasks)
-
-## Plot stored binary data (runs serially, not in the parallel pool)
-plot_xiaoqu.generate_binary_data_plot(binary_results_directory, figure_directory)
-
-print()
-print(" Plotting completed successfully. ")
-print()

AMSS_NCKU_source/Block.C

@@ -6,14 +6,68 @@
 #include <cstdio>
 #include <string>
 #include <cmath>
-#include <new>
-using namespace std;
-
-#include "Block.h"
-#include "misc.h"
-
-Block::Block(int DIM, int *shapei, double *bboxi, int ranki, int ingfsi, int fngfsi, int levi, const int cgpui) : rank(ranki), ingfs(ingfsi), fngfs(fngfsi), lev(levi), cgpu(cgpui)
-{
+#include <new>
+using namespace std;
+
+#include "Block.h"
+#include "misc.h"
+
+#if USE_CUDA_BSSN || USE_CUDA_Z4C
+#include <cuda_runtime_api.h>
+#endif
+
+namespace {
+
+bool cuda_pin_gridfuncs_enabled()
+{
+  static int enabled = -1;
+  if (enabled < 0)
+  {
+    const char *env = getenv("AMSS_CUDA_PIN_GRIDFUNCS");
+    enabled = (env && atoi(env) != 0) ? 1 : 0;
+  }
+  return enabled != 0;
+}
+
+double *alloc_gridfunc(size_t count, unsigned char &pinned)
+{
+  pinned = 0;
+#if USE_CUDA_BSSN || USE_CUDA_Z4C
+  if (cuda_pin_gridfuncs_enabled())
+  {
+    double *ptr = 0;
+    cudaError_t err = cudaMallocHost((void **)&ptr, count * sizeof(double));
+    if (err == cudaSuccess)
+    {
+      pinned = 1;
+      return ptr;
+    }
+    cudaGetLastError();
+  }
+#endif
+  return (double *)malloc(sizeof(double) * count);
+}
+
+void free_gridfunc(double *ptr, unsigned char pinned)
+{
+  if (!ptr)
+    return;
+#if USE_CUDA_BSSN || USE_CUDA_Z4C
+  if (pinned)
+  {
+    cudaFreeHost(ptr);
+    return;
+  }
+#else
+  (void)pinned;
+#endif
+  free(ptr);
+}
+
+}
+
+Block::Block(int DIM, int *shapei, double *bboxi, int ranki, int ingfsi, int fngfsi, int levi, const int cgpui) : rank(ranki), lev(levi), cgpu(cgpui), ingfs(ingfsi), fngfs(fngfsi), igfs(0), fgfs(0), fgfs_pinned(0)
+{
   for (int i = 0; i < dim; i++)
     X[i] = 0;
 
@@ -68,14 +122,15 @@ Block::Block(int DIM, int *shapei, double *bboxi, int ranki, int ingfsi, int fng
 #endif
     }
 
-    int nn = shape[0] * shape[1] * shape[2];
-    fgfs = new double *[fngfs];
-    for (int i = 0; i < fngfs; i++)
-    {
-      fgfs[i] = (double *)malloc(sizeof(double) * nn);
-      if (!(fgfs[i]))
-      {
-        cout << "on node#" << rank << ", out of memory when constructing Block." << endl;
+    int nn = shape[0] * shape[1] * shape[2];
+    fgfs = new double *[fngfs];
+    fgfs_pinned = new unsigned char[fngfs];
+    for (int i = 0; i < fngfs; i++)
+    {
+      fgfs[i] = alloc_gridfunc((size_t)nn, fgfs_pinned[i]);
+      if (!(fgfs[i]))
+      {
+        cout << "on node#" << rank << ", out of memory when constructing Block." << endl;
         MPI_Abort(MPI_COMM_WORLD, 1);
       }
       memset(fgfs[i], 0, sizeof(double) * nn);
@@ -103,17 +158,19 @@ Block::~Block()
   {
     for (int i = 0; i < dim; i++)
       delete[] X[i];
-    for (int i = 0; i < ingfs; i++)
-      free(igfs[i]);
-    delete[] igfs;
-    for (int i = 0; i < fngfs; i++)
-      free(fgfs[i]);
-    delete[] fgfs;
-    X[0] = X[1] = X[2] = 0;
-    igfs = 0;
-    fgfs = 0;
-  }
-}
+    for (int i = 0; i < ingfs; i++)
+      free(igfs[i]);
+    delete[] igfs;
+    for (int i = 0; i < fngfs; i++)
+      free_gridfunc(fgfs[i], fgfs_pinned ? fgfs_pinned[i] : 0);
+    delete[] fgfs;
+    delete[] fgfs_pinned;
+    X[0] = X[1] = X[2] = 0;
+    igfs = 0;
+    fgfs = 0;
+    fgfs_pinned = 0;
+  }
+}
 void Block::checkBlock()
 {
   int myrank;
@@ -184,12 +241,14 @@ void Block::swapList(MyList<var> *VarList1, MyList<var> *VarList2, int myrank)
   if (rank == myrank)
   {
     MyList<var> *varl1 = VarList1, *varl2 = VarList2;
-    while (varl1 && varl2)
-    {
-      misc::swap<double *>(fgfs[varl1->data->sgfn], fgfs[varl2->data->sgfn]);
-      varl1 = varl1->next;
-      varl2 = varl2->next;
-    }
+    while (varl1 && varl2)
+    {
+      misc::swap<double *>(fgfs[varl1->data->sgfn], fgfs[varl2->data->sgfn]);
+      if (fgfs_pinned)
+        misc::swap<unsigned char>(fgfs_pinned[varl1->data->sgfn], fgfs_pinned[varl2->data->sgfn]);
+      varl1 = varl1->next;
+      varl2 = varl2->next;
+    }
     if (varl1 || varl2)
     {
       cout << "error in Block::swaplist, var lists does not match." << endl;

AMSS_NCKU_source/Block.h

@@ -13,14 +13,15 @@ public:
    int shape[dim];
    double bbox[2 * dim];
    double *X[dim];
-   int rank; // where the real data locate in
-   int lev, cgpu;
-   int ingfs, fngfs;
-   int *(*igfs);
-   double *(*fgfs);
+   int rank; // where the real data locate in
+   int lev, cgpu;
+   int ingfs, fngfs;
+   int *(*igfs);
+   double *(*fgfs);
+   unsigned char *fgfs_pinned;
 
 public:
-   Block() {};
+   Block() : rank(0), lev(0), cgpu(0), ingfs(0), fngfs(0), igfs(0), fgfs(0), fgfs_pinned(0) {};
    Block(int DIM, int *shapei, double *bboxi, int ranki, int ingfsi, int fngfs, int levi, const int cgpui = 0);
 
    ~Block();

AMSS_NCKU_source/MPatch.C

@@ -11,12 +11,15 @@
 using namespace std;
 
 #include "misc.h"
-#include "MPatch.h"
-#include "Parallel.h"
-#include "fmisc.h"
-#ifdef INTERP_LB_PROFILE
-#include "interp_lb_profile.h"
-#endif
+#include "MPatch.h"
+#include "Parallel.h"
+#include "fmisc.h"
+#if USE_CUDA_BSSN
+#include "bssn_rhs_cuda.h"
+#endif
+#ifdef INTERP_LB_PROFILE
+#include "interp_lb_profile.h"
+#endif
 
 namespace
 {
@@ -154,8 +157,8 @@ void build_block_bin_index(Patch *patch, const double *DH, BlockBinIndex &index)
   index.valid = true;
 }
 
-int find_block_index_for_point(const BlockBinIndex &index, const double *pox, const double *DH)
-{
+int find_block_index_for_point(const BlockBinIndex &index, const double *pox, const double *DH)
+{
   if (!index.valid)
     return -1;
 
@@ -175,10 +178,448 @@ int find_block_index_for_point(const BlockBinIndex &index, const double *pox, co
   for (size_t bi = 0; bi < index.views.size(); bi++)
     if (point_in_block_view(index.views[bi], pox, DH))
       return int(bi);
-
-  return -1;
-}
-} // namespace
+
+  return -1;
+}
+
+inline int fortran_idint_local(double x)
+{
+  return int(x);
+}
+
+bool interp_fast_enabled()
+{
+  static int enabled = -1;
+  if (enabled < 0)
+  {
+    const char *env = getenv("AMSS_INTERP_FAST");
+    enabled = (!env || atoi(env) != 0) ? 1 : 0;
+  }
+  return enabled != 0;
+}
+
+bool interp_gpu_enabled()
+{
+  static int enabled = -1;
+  if (enabled < 0)
+  {
+    const char *env = getenv("AMSS_INTERP_GPU");
+    enabled = (env && atoi(env) != 0) ? 1 : 0;
+  }
+  return enabled != 0;
+}
+
+bool interp_fast_compare_enabled()
+{
+  static int enabled = -1;
+  if (enabled < 0)
+  {
+    const char *env = getenv("AMSS_INTERP_FAST_COMPARE");
+    enabled = (env && atoi(env) != 0) ? 1 : 0;
+  }
+  return enabled != 0;
+}
+
+double interp_fast_compare_tol()
+{
+  static double tol = -1.0;
+  if (tol < 0.0)
+  {
+    const char *env = getenv("AMSS_INTERP_FAST_COMPARE_TOL");
+    tol = (env && atof(env) > 0.0) ? atof(env) : 1.0e-11;
+  }
+  return tol;
+}
+
+long long interp_fast_compare_limit()
+{
+  static long long limit = -1;
+  if (limit < 0)
+  {
+    const char *env = getenv("AMSS_INTERP_FAST_COMPARE_LIMIT");
+    limit = (env && atoll(env) > 0) ? atoll(env) : 4096;
+  }
+  return limit;
+}
+
+struct FastInterpStencil
+{
+  int cxB[dim];
+  double cx[dim];
+  double wx[8];
+  double wy[8];
+  double wz[8];
+  int nsamples;
+  int loc[512];
+  unsigned char sign_mask[512];
+  double weight[512];
+};
+
+inline void lagrange_unit_weights(double x, int ordn, double *w)
+{
+  for (int i = 0; i < ordn; i++)
+  {
+    double num = 1.0;
+    double den = 1.0;
+    for (int j = 0; j < ordn; j++)
+    {
+      if (j == i)
+        continue;
+      num *= (x - double(j));
+      den *= double(i - j);
+    }
+    w[i] = num / den;
+  }
+}
+
+inline void z_unit_weights(double x, int ordn, double *w)
+{
+  if (ordn == 6)
+  {
+    static const double c_uniform[6] = {-1.0, 5.0, -10.0, 10.0, -5.0, 1.0};
+    for (int i = 0; i < 6; i++)
+    {
+      if (x == double(i))
+      {
+        for (int j = 0; j < 6; j++)
+          w[j] = (j == i) ? 1.0 : 0.0;
+        return;
+      }
+    }
+    double den = 0.0;
+    for (int i = 0; i < 6; i++)
+    {
+      w[i] = c_uniform[i] / (x - double(i));
+      den += w[i];
+    }
+    for (int i = 0; i < 6; i++)
+      w[i] /= den;
+    return;
+  }
+  lagrange_unit_weights(x, ordn, w);
+}
+
+inline bool fast_interp_map_index(int idx, int extent, int d,
+                                  int &mapped, unsigned char &mask)
+{
+  if (idx > 0)
+    mapped = idx;
+  else
+  {
+    mask |= (unsigned char)(1u << d);
+#ifdef Vertex
+#ifdef Cell
+#error Both Cell and Vertex are defined
+#endif
+    mapped = 2 - idx;
+#else
+#ifdef Cell
+    mapped = 1 - idx;
+#else
+#error Not define Vertex nor Cell
+#endif
+#endif
+  }
+  return mapped >= 1 && mapped <= extent;
+}
+
+bool prepare_fast_interp_stencil(Block *BP, const double *pox, int ordn,
+                                 int Symmetry, FastInterpStencil &st)
+{
+  if (!BP || ordn <= 0 || ordn > 8)
+    return false;
+
+  st.nsamples = 0;
+
+  const int NO_SYMM = 0;
+  const int OCTANT = 2;
+  int cmin[dim], cmax[dim], cxT[dim];
+  for (int d = 0; d < dim; d++)
+  {
+    const double *X = BP->X[d];
+    const double dX = X[1] - X[0];
+    const int cxI = fortran_idint_local((pox[d] - X[0]) / dX + 0.4) + 1;
+    st.cxB[d] = cxI - ordn / 2 + 1;
+    cxT[d] = st.cxB[d] + ordn - 1;
+    cmin[d] = 1;
+    cmax[d] = BP->shape[d];
+
+#ifdef Vertex
+#ifdef Cell
+#error Both Cell and Vertex are defined
+#endif
+    if (Symmetry == OCTANT && d < 2 && fabs(X[0]) < dX)
+      cmin[d] = -ordn / 2 + 2;
+    if (Symmetry != NO_SYMM && d == 2 && fabs(X[0]) < dX)
+      cmin[d] = -ordn / 2 + 2;
+#else
+#ifdef Cell
+    if (Symmetry == OCTANT && d < 2 && fabs(X[0]) < dX)
+      cmin[d] = -ordn / 2 + 1;
+    if (Symmetry != NO_SYMM && d == 2 && fabs(X[0]) < dX)
+      cmin[d] = -ordn / 2 + 1;
+#else
+#error Not define Vertex nor Cell
+#endif
+#endif
+
+    if (st.cxB[d] < cmin[d])
+    {
+      st.cxB[d] = cmin[d];
+      cxT[d] = st.cxB[d] + ordn - 1;
+    }
+    if (cxT[d] > cmax[d])
+    {
+      cxT[d] = cmax[d];
+      st.cxB[d] = cxT[d] + 1 - ordn;
+    }
+
+    if (st.cxB[d] > 0)
+      st.cx[d] = (pox[d] - X[st.cxB[d] - 1]) / dX;
+    else
+    {
+#ifdef Vertex
+#ifdef Cell
+#error Both Cell and Vertex are defined
+#endif
+      st.cx[d] = (pox[d] + X[1 - st.cxB[d]]) / dX;
+#else
+#ifdef Cell
+      st.cx[d] = (pox[d] + X[-st.cxB[d]]) / dX;
+#else
+#error Not define Vertex nor Cell
+#endif
+#endif
+    }
+  }
+
+  lagrange_unit_weights(st.cx[0], ordn, st.wx);
+  lagrange_unit_weights(st.cx[1], ordn, st.wy);
+  z_unit_weights(st.cx[2], ordn, st.wz);
+
+  for (int kk = 0; kk < ordn; kk++)
+  {
+    for (int jj = 0; jj < ordn; jj++)
+    {
+      for (int ii = 0; ii < ordn; ii++)
+      {
+        unsigned char mask = 0;
+        int ix, iy, iz;
+        if (!fast_interp_map_index(st.cxB[0] + ii, BP->shape[0], 0, ix, mask) ||
+            !fast_interp_map_index(st.cxB[1] + jj, BP->shape[1], 1, iy, mask) ||
+            !fast_interp_map_index(st.cxB[2] + kk, BP->shape[2], 2, iz, mask))
+          return false;
+        const int s = st.nsamples++;
+        st.loc[s] = (ix - 1) + (iy - 1) * BP->shape[0] +
+                    (iz - 1) * BP->shape[0] * BP->shape[1];
+        st.sign_mask[s] = mask;
+        st.weight[s] = st.wx[ii] * st.wy[jj] * st.wz[kk];
+      }
+    }
+  }
+  return true;
+}
+
+bool interpolate_var_list_with_stencil(Block *BP, MyList<var> *VarList,
+                                       int num_var, const double *pox,
+                                       int ordn, int Symmetry,
+                                       const FastInterpStencil &st,
+                                       double *out)
+{
+  if (num_var <= 0 || num_var > 128)
+    return false;
+
+  double *data_ptrs[128];
+  double *soa_ptrs[128];
+  var *vars[128];
+  MyList<var> *varl = VarList;
+  int k = 0;
+  while (varl)
+  {
+    if (k >= num_var)
+      return false;
+    vars[k] = varl->data;
+    data_ptrs[k] = BP->fgfs[vars[k]->sgfn];
+    soa_ptrs[k] = vars[k]->SoA;
+    out[k] = 0.0;
+    varl = varl->next;
+    k++;
+  }
+
+  if (k != num_var)
+    return false;
+
+  for (int s = 0; s < st.nsamples; s++)
+  {
+    const int loc = st.loc[s];
+    const double w = st.weight[s];
+    const unsigned char mask = st.sign_mask[s];
+    if (mask == 0)
+    {
+      for (int v = 0; v < num_var; v++)
+        out[v] += w * data_ptrs[v][loc];
+    }
+    else
+    {
+      for (int v = 0; v < num_var; v++)
+      {
+        const double *SoA = soa_ptrs[v];
+        double sgn = 1.0;
+        if (mask & 1u)
+          sgn *= SoA[0];
+        if (mask & 2u)
+          sgn *= SoA[1];
+        if (mask & 4u)
+          sgn *= SoA[2];
+        out[v] += w * sgn * data_ptrs[v][loc];
+      }
+    }
+  }
+
+  if (interp_fast_compare_enabled())
+  {
+    static int report_count = 0;
+    static long long compare_calls = 0;
+    if (compare_calls++ >= interp_fast_compare_limit())
+      return true;
+    const double tol = interp_fast_compare_tol();
+    varl = VarList;
+    k = 0;
+    while (varl)
+    {
+      var *vp = vars[k];
+      double ref = 0.0;
+      double x = pox[0], y = pox[1], z = pox[2];
+      f_global_interp(BP->shape, BP->X[0], BP->X[1], BP->X[2],
+                      BP->fgfs[vp->sgfn], ref,
+                      x, y, z, ordn, vp->SoA, Symmetry);
+      const double diff = fabs(ref - out[k]);
+      const double scale = 1.0 + fabs(ref);
+      if (diff > tol * scale && report_count < 32)
+      {
+        int rank = 0;
+        MPI_Comm_rank(MPI_COMM_WORLD, &rank);
+        fprintf(stderr,
+                "[AMSS-INTERP-CMP][rank %d] var=%s diff=%.17e ref=%.17e fast=%.17e p=(%.17e,%.17e,%.17e)\n",
+                rank, vp->name, diff, ref, out[k], pox[0], pox[1], pox[2]);
+        report_count++;
+      }
+      varl = varl->next;
+      k++;
+    }
+  }
+
+  return true;
+}
+
+bool interpolate_var_list_fast(Block *BP, MyList<var> *VarList, int num_var,
+                               const double *pox, int ordn, int Symmetry,
+                               double *out)
+{
+  if (!interp_fast_enabled())
+    return false;
+
+  FastInterpStencil st;
+  if (!prepare_fast_interp_stencil(BP, pox, ordn, Symmetry, st))
+    return false;
+
+  return interpolate_var_list_with_stencil(BP, VarList, num_var, pox,
+                                           ordn, Symmetry, st, out);
+}
+
+struct CachedInterpPoint
+{
+  Block *bp;
+  int owner_rank;
+  FastInterpStencil stencil;
+};
+
+struct SurfaceInterpCache
+{
+  Patch *patch;
+  int NN;
+  int symmetry;
+  double key[9];
+  vector<CachedInterpPoint> points;
+
+  SurfaceInterpCache() : patch(0), NN(0), symmetry(-1) {}
+};
+
+bool surface_cache_key_matches(const SurfaceInterpCache &cache, Patch *patch,
+                               int NN, double **XX, int Symmetry)
+{
+  if (cache.patch != patch || cache.NN != NN || cache.symmetry != Symmetry ||
+      int(cache.points.size()) != NN || NN <= 0)
+    return false;
+  const int mid = NN / 2;
+  const int last = NN - 1;
+  const int ids[3] = {0, mid, last};
+  int p = 0;
+  for (int q = 0; q < 3; q++)
+    for (int d = 0; d < dim; d++)
+      if (cache.key[p++] != XX[d][ids[q]])
+        return false;
+  return true;
+}
+
+SurfaceInterpCache *find_surface_cache(Patch *patch, int NN, double **XX,
+                                       int Symmetry)
+{
+  static vector<SurfaceInterpCache> caches;
+  for (size_t i = 0; i < caches.size(); i++)
+    if (surface_cache_key_matches(caches[i], patch, NN, XX, Symmetry))
+      return &caches[i];
+  if (caches.size() >= 24)
+    caches.erase(caches.begin());
+  caches.push_back(SurfaceInterpCache());
+  return &caches.back();
+}
+
+bool build_surface_cache(SurfaceInterpCache &cache, Patch *patch, int NN,
+                         double **XX, int Symmetry, const double *DH,
+                         const BlockBinIndex &block_index, int ordn)
+{
+  int myrank = 0;
+  MPI_Comm_rank(MPI_COMM_WORLD, &myrank);
+  cache.patch = patch;
+  cache.NN = NN;
+  cache.symmetry = Symmetry;
+  cache.points.clear();
+  cache.points.resize(NN);
+  const int mid = NN / 2;
+  const int last = NN - 1;
+  const int ids[3] = {0, mid, last};
+  int p = 0;
+  for (int q = 0; q < 3; q++)
+    for (int d = 0; d < dim; d++)
+      cache.key[p++] = XX[d][ids[q]];
+
+  for (int j = 0; j < NN; j++)
+  {
+    double pox[dim];
+    for (int d = 0; d < dim; d++)
+      pox[d] = XX[d][j];
+    const int block_i = find_block_index_for_point(block_index, pox, DH);
+    if (block_i < 0)
+    {
+      cache.points[j].bp = 0;
+      cache.points[j].owner_rank = -1;
+      continue;
+    }
+    Block *BP = block_index.views[block_i].bp;
+    cache.points[j].bp = BP;
+    cache.points[j].owner_rank = BP->rank;
+    cache.points[j].stencil.nsamples = 0;
+    if (BP->rank == myrank)
+    {
+      if (!prepare_fast_interp_stencil(BP, pox, ordn, Symmetry,
+                                       cache.points[j].stencil))
+        return false;
+    }
+  }
+  return true;
+}
+} // namespace
 
 Patch::Patch(int DIM, int *shapei, double *bboxi, int levi, bool buflog, int Symmetry) : lev(levi)
 {
@@ -561,22 +1002,26 @@ void Patch::Interp_Points(MyList<var> *VarList,
     if (block_i >= 0)
     {
       Block *BP = block_index.views[block_i].bp;
-      owner_rank[j] = BP->rank;
-      if (myrank == BP->rank)
-      {
-        //---> interpolation
-        varl = VarList;
-        int k = 0;
-        while (varl) // run along variables
-        {
-          f_global_interp(BP->shape, BP->X[0], BP->X[1], BP->X[2], BP->fgfs[varl->data->sgfn], Shellf[j * num_var + k],
-                          pox[0], pox[1], pox[2], ordn, varl->data->SoA, Symmetry);
-          varl = varl->next;
-          k++;
-        }
-      }
-    }
-  }
+      owner_rank[j] = BP->rank;
+      if (myrank == BP->rank)
+      {
+        //---> interpolation
+        if (!interpolate_var_list_fast(BP, VarList, num_var, pox, ordn,
+                                       Symmetry, Shellf + j * num_var))
+        {
+          varl = VarList;
+          int k = 0;
+          while (varl) // run along variables
+          {
+            f_global_interp(BP->shape, BP->X[0], BP->X[1], BP->X[2], BP->fgfs[varl->data->sgfn], Shellf[j * num_var + k],
+                            pox[0], pox[1], pox[2], ordn, varl->data->SoA, Symmetry);
+            varl = varl->next;
+            k++;
+          }
+        }
+      }
+    }
+  }
 
   // Replace MPI_Allreduce with per-owner MPI_Bcast:
   // Group consecutive points by owner rank and broadcast each group.
@@ -659,10 +1104,8 @@ void Patch::Interp_Points(MyList<var> *VarList,
     varl = varl->next;
   }
 
-  memset(Shellf, 0, sizeof(double) * NN * num_var);
-
-  // owner_rank[j] records which MPI rank owns point j
-  int *owner_rank;
+  // owner_rank[j] records which MPI rank owns point j
+  int *owner_rank;
   owner_rank = new int[NN];
   for (int j = 0; j < NN; j++)
     owner_rank[j] = -1;
@@ -670,12 +1113,117 @@ void Patch::Interp_Points(MyList<var> *VarList,
   double DH[dim];
   for (int i = 0; i < dim; i++)
     DH[i] = getdX(i);
-  BlockBinIndex block_index;
-  build_block_bin_index(this, DH, block_index);
-
-  // --- Interpolation phase (identical to original) ---
-  for (int j = 0; j < NN; j++)
-  {
+  BlockBinIndex block_index;
+  build_block_bin_index(this, DH, block_index);
+  SurfaceInterpCache *surface_cache = 0;
+  bool use_surface_cache = false;
+  if (interp_fast_enabled())
+  {
+    surface_cache = find_surface_cache(this, NN, XX, Symmetry);
+    use_surface_cache = surface_cache_key_matches(*surface_cache, this, NN, XX, Symmetry);
+    if (!use_surface_cache)
+      use_surface_cache = build_surface_cache(*surface_cache, this, NN, XX,
+                                              Symmetry, DH, block_index, ordn);
+  }
+
+  // --- Interpolation phase (identical to original) ---
+#if USE_CUDA_BSSN
+  const bool use_gpu_interp = interp_gpu_enabled() && use_surface_cache && num_var == 2 &&
+                              VarList && VarList->next && !VarList->next->next;
+#else
+  const bool use_gpu_interp = false;
+#endif
+  if (use_gpu_interp)
+  {
+#if USE_CUDA_BSSN
+    vector<vector<int> > local_points(block_index.views.size());
+    for (int j = 0; j < NN; j++)
+    {
+      for (int i = 0; i < dim; i++)
+      {
+        if (myrank == 0 && (XX[i][j] < bbox[i] + lli[i] * DH[i] || XX[i][j] > bbox[dim + i] - uui[i] * DH[i]))
+        {
+          cout << "Patch::Interp_Points: point (";
+          for (int k = 0; k < dim; k++)
+          {
+            cout << XX[k][j];
+            if (k < dim - 1)
+              cout << ",";
+            else
+              cout << ") is out of current Patch." << endl;
+          }
+          MPI_Abort(MPI_COMM_WORLD, 1);
+        }
+      }
+
+      CachedInterpPoint &cp = surface_cache->points[j];
+      Block *BP = cp.bp;
+      owner_rank[j] = cp.owner_rank;
+      if (BP && myrank == BP->rank)
+      {
+        for (size_t bi = 0; bi < block_index.views.size(); bi++)
+        {
+          if (block_index.views[bi].bp == BP)
+          {
+            local_points[bi].push_back(j);
+            break;
+          }
+        }
+      }
+    }
+
+    var *v0 = VarList->data;
+    var *v1 = VarList->next->data;
+    double soa6[6] = {
+        v0->SoA[0], v0->SoA[1], v0->SoA[2],
+        v1->SoA[0], v1->SoA[1], v1->SoA[2]};
+
+    for (size_t bi = 0; bi < local_points.size(); bi++)
+    {
+      const int count = int(local_points[bi].size());
+      if (count <= 0)
+        continue;
+
+      Block *BP = block_index.views[bi].bp;
+      vector<double> px(count), py(count), pz(count), out(2 * count);
+      for (int q = 0; q < count; q++)
+      {
+        const int j = local_points[bi][q];
+        px[q] = XX[0][j];
+        py[q] = XX[1][j];
+        pz[q] = XX[2][j];
+      }
+
+      const double dx = BP->X[0][1] - BP->X[0][0];
+      const double dy = BP->X[1][1] - BP->X[1][0];
+      const double dz = BP->X[2][1] - BP->X[2][0];
+      const int ok = bssn_cuda_interp_host_two_fields(
+          BP, BP->shape,
+          BP->fgfs[v0->sgfn], BP->fgfs[v1->sgfn],
+          BP->X[0][0], BP->X[1][0], BP->X[2][0],
+          dx, dy, dz,
+          &px[0], &py[0], &pz[0], count,
+          ordn, Symmetry, soa6, &out[0]);
+      if (ok != 0)
+      {
+        if (myrank == 0)
+          cout << "Patch::Interp_Points: CUDA two-field interpolation failed" << endl;
+        MPI_Abort(MPI_COMM_WORLD, 1);
+      }
+
+      for (int q = 0; q < count; q++)
+      {
+        const int j = local_points[bi][q];
+        Shellf[j * num_var] = out[2 * q];
+        Shellf[j * num_var + 1] = out[2 * q + 1];
+      }
+    }
+#endif
+  }
+  else
+  {
+  for (int j = 0; j < NN; j++)
+  {
     double pox[dim];
     for (int i = 0; i < dim; i++)
     {
@@ -692,28 +1240,59 @@ void Patch::Interp_Points(MyList<var> *VarList,
             cout << ") is out of current Patch." << endl;
         }
         MPI_Abort(MPI_COMM_WORLD, 1);
-      }
-    }
-
-    const int block_i = find_block_index_for_point(block_index, pox, DH);
-    if (block_i >= 0)
-    {
-      Block *BP = block_index.views[block_i].bp;
-      owner_rank[j] = BP->rank;
-      if (myrank == BP->rank)
-      {
-        varl = VarList;
-        int k = 0;
-        while (varl)
-        {
-          f_global_interp(BP->shape, BP->X[0], BP->X[1], BP->X[2], BP->fgfs[varl->data->sgfn], Shellf[j * num_var + k],
-                          pox[0], pox[1], pox[2], ordn, varl->data->SoA, Symmetry);
-          varl = varl->next;
-          k++;
-        }
-      }
-    }
-  }
+      }
+    }
+
+    if (use_surface_cache)
+    {
+      CachedInterpPoint &cp = surface_cache->points[j];
+      Block *BP = cp.bp;
+      owner_rank[j] = cp.owner_rank;
+      if (BP && myrank == BP->rank)
+      {
+        if (!interpolate_var_list_with_stencil(BP, VarList, num_var, pox,
+                                               ordn, Symmetry, cp.stencil,
+                                               Shellf + j * num_var))
+        {
+          MyList<var> *varl_fallback = VarList;
+          int k = 0;
+          while (varl_fallback)
+          {
+            f_global_interp(BP->shape, BP->X[0], BP->X[1], BP->X[2], BP->fgfs[varl_fallback->data->sgfn], Shellf[j * num_var + k],
+                            pox[0], pox[1], pox[2], ordn, varl_fallback->data->SoA, Symmetry);
+            varl_fallback = varl_fallback->next;
+            k++;
+          }
+        }
+      }
+    }
+    else
+    {
+      const int block_i = find_block_index_for_point(block_index, pox, DH);
+      if (block_i >= 0)
+      {
+        Block *BP = block_index.views[block_i].bp;
+        owner_rank[j] = BP->rank;
+      if (myrank == BP->rank)
+      {
+        if (!interpolate_var_list_fast(BP, VarList, num_var, pox, ordn,
+                                       Symmetry, Shellf + j * num_var))
+        {
+          MyList<var> *varl_fallback = VarList;
+          int k = 0;
+          while (varl_fallback)
+          {
+            f_global_interp(BP->shape, BP->X[0], BP->X[1], BP->X[2], BP->fgfs[varl_fallback->data->sgfn], Shellf[j * num_var + k],
+                            pox[0], pox[1], pox[2], ordn, varl_fallback->data->SoA, Symmetry);
+            varl_fallback = varl_fallback->next;
+            k++;
+          }
+        }
+      }
+      }
+    }
+  }
+  }
 
 #ifdef INTERP_LB_PROFILE
   double t_interp_end = MPI_Wtime();
@@ -965,22 +1544,26 @@ void Patch::Interp_Points(MyList<var> *VarList,
     if (block_i >= 0)
     {
       Block *BP = block_index.views[block_i].bp;
-      owner_rank[j] = BP->rank;
-      if (myrank == BP->rank)
-      {
-        //---> interpolation
-        varl = VarList;
-        int k = 0;
-        while (varl) // run along variables
-        {
-          f_global_interp(BP->shape, BP->X[0], BP->X[1], BP->X[2], BP->fgfs[varl->data->sgfn], Shellf[j * num_var + k],
-                          pox[0], pox[1], pox[2], ordn, varl->data->SoA, Symmetry);
-          varl = varl->next;
-          k++;
-        }
-      }
-    }
-  }
+      owner_rank[j] = BP->rank;
+      if (myrank == BP->rank)
+      {
+        //---> interpolation
+        if (!interpolate_var_list_fast(BP, VarList, num_var, pox, ordn,
+                                       Symmetry, Shellf + j * num_var))
+        {
+          varl = VarList;
+          int k = 0;
+          while (varl) // run along variables
+          {
+            f_global_interp(BP->shape, BP->X[0], BP->X[1], BP->X[2], BP->fgfs[varl->data->sgfn], Shellf[j * num_var + k],
+                            pox[0], pox[1], pox[2], ordn, varl->data->SoA, Symmetry);
+            varl = varl->next;
+            k++;
+          }
+        }
+      }
+    }
+  }
 
   // Collect unique global owner ranks and translate to local ranks in Comm_here
   // Then broadcast each owner's points via MPI_Bcast on Comm_here

AMSS_NCKU_source/Parallel.h

@@ -104,6 +104,14 @@ namespace Parallel
     double **recv_bufs;
     int *send_buf_caps;
     int *recv_buf_caps;
+    unsigned char *send_buf_pinned;
+    unsigned char *recv_buf_pinned;
+    unsigned char *send_buf_is_dev;
+    unsigned char *recv_buf_is_dev;
+    int *send_buf_caps_dev;
+    int *recv_buf_caps_dev;
+    double **send_bufs_dev;
+    double **recv_bufs_dev;
     MPI_Request *reqs;
     MPI_Status *stats;
     int max_reqs;
@@ -111,12 +119,14 @@ namespace Parallel
     int *tc_req_node;
     int *tc_req_is_recv;
     int *tc_completed;
+    bool cuda_aware_mode;
     SyncCache();
     void invalidate();
     void destroy();
   };
 
   void Sync_cached(MyList<Patch> *PatL, MyList<var> *VarList, int Symmetry, SyncCache &cache);
+  void Sync_ensure_cache(MyList<Patch> *PatL, int Symmetry, SyncCache &cache);
   void transfer_cached(MyList<gridseg> **src, MyList<gridseg> **dst,
                        MyList<var> *VarList1, MyList<var> *VarList2,
                        int Symmetry, SyncCache &cache);
@@ -179,13 +189,13 @@ namespace Parallel
   MyList<Parallel::gridseg> *clone_gsl(MyList<Parallel::gridseg> *p, bool first_only);
   MyList<Parallel::gridseg> *build_bulk_gsl(Patch *Pat); // similar to build_owned_gsl0 but does not care rank issue
   MyList<Parallel::gridseg> *build_bulk_gsl(Block *bp, Patch *Pat);
-  void build_PhysBD_gstl(Patch *Pat, MyList<Parallel::gridseg> *srci, MyList<Parallel::gridseg> *dsti,
-                         MyList<Parallel::gridseg> **out_src, MyList<Parallel::gridseg> **out_dst);
-  void PeriodicBD(Patch *Pat, MyList<var> *VarList, int Symmetry);
-  double L2Norm(Patch *Pat, var *vf);
-  void L2Norm7(Patch *Pat, var **vf, double *norms);
-  void checkgsl(MyList<Parallel::gridseg> *pp, bool first_only);
-  void checkvarl(MyList<var> *pp, bool first_only);
+  void build_PhysBD_gstl(Patch *Pat, MyList<Parallel::gridseg> *srci, MyList<Parallel::gridseg> *dsti,
+                         MyList<Parallel::gridseg> **out_src, MyList<Parallel::gridseg> **out_dst);
+  void PeriodicBD(Patch *Pat, MyList<var> *VarList, int Symmetry);
+  double L2Norm(Patch *Pat, var *vf);
+  void L2Norm7(Patch *Pat, var **vf, double *norms);
+  void checkgsl(MyList<Parallel::gridseg> *pp, bool first_only);
+  void checkvarl(MyList<var> *pp, bool first_only);
   MyList<Parallel::gridseg> *divide_gsl(MyList<Parallel::gridseg> *p, Patch *Pat);
   MyList<Parallel::gridseg> *divide_gs(MyList<Parallel::gridseg> *p, Patch *Pat);
   void prepare_inter_time_level(Patch *Pat,
@@ -217,12 +227,12 @@ namespace Parallel
   void aligncheck(double *bbox0, double *bboxl, int lev, double *DH0, int *shape);
   bool point_locat_gsl(double *pox, MyList<Parallel::gridseg> *gsl);
   void checkpatchlist(MyList<Patch> *PatL, bool buflog);
-
-  double L2Norm(Patch *Pat, var *vf, MPI_Comm Comm_here);
-  void L2Norm7(Patch *Pat, var **vf, double *norms, MPI_Comm Comm_here);
-  bool PatList_Interp_Points(MyList<Patch> *PatL, MyList<var> *VarList,
-                             int NN, double **XX,
-                             double *Shellf, int Symmetry, MPI_Comm Comm_here);
+
+  double L2Norm(Patch *Pat, var *vf, MPI_Comm Comm_here);
+  void L2Norm7(Patch *Pat, var **vf, double *norms, MPI_Comm Comm_here);
+  bool PatList_Interp_Points(MyList<Patch> *PatL, MyList<var> *VarList,
+                             int NN, double **XX,
+                             double *Shellf, int Symmetry, MPI_Comm Comm_here);
 #if (PSTR == 1 || PSTR == 2 || PSTR == 3)
   MyList<Block> *distribute(MyList<Patch> *PatchLIST, int cpusize, int ingfsi, int fngfsi,
                             bool periodic, int start_rank, int end_rank, int nodes = 0);

AMSS_NCKU_source/ShellPatch.h

@@ -102,16 +102,6 @@ public:
                  //-1: means no dumy dimension at all; 0: means rho; 1: means sigma
    };
 
-   // Thread-safe search result (no pointers to shared mutable state)
-   struct PointSearchResult
-   {
-      bool found;
-      Block *Bg;
-      double gx, gy, gz; // global Cartesian coordinates
-      double lx, ly, lz; // local coordinates within the found block
-      int ssst;          // source shell-patch type (-1 = Cartesian)
-   };
-
    int myrank;
    int shape[dim];   // for (rho, sigma, R), for rho and sigma means number of points for every pi/2
    double Rrange[2]; // for Rmin and Rmax
@@ -185,12 +175,6 @@ public:
                          MyList<Patch> *Pp, double CDH[dim], MyList<pointstru> *pss);
    bool prolongpointstru(MyList<pointstru> *&psul, bool ssyn, int tsst, MyList<ss_patch> *sPp, double DH[dim],
                          MyList<Patch> *Pp, double CDH[dim], double x, double y, double z, int Symmetry, int rank_in);
-   // Read-only point search — thread-safe (no shared mutable state modified)
-   PointSearchResult prolongpointstru_search(bool ssyn, int tsst, MyList<ss_patch> *sPp, double DH[dim],
-                                             MyList<Patch> *Pp, double CDH[dim], double x, double y, double z,
-                                             int Symmetry, int rank_in);
-   // Append a search result to a linked list — use inside omp critical section
-   void prolongpointstru_append(MyList<pointstru> *&psul, const PointSearchResult &sr, int tsst);
    void setupintintstuff(int cpusize, MyList<Patch> *CPatL, int Symmetry);
    void intertransfer(MyList<pointstru> **src, MyList<pointstru> **dst,
                       MyList<var> *VarList1 /* source */, MyList<var> *VarList2 /*target */,
@@ -211,11 +195,11 @@ public:
    bool Interp_One_Point(MyList<var> *VarList,
                          double *XX, /*input global Cartesian coordinate*/
                          double *Shellf, int Symmetry);
-   void write_Pablo_file_ss(int *ext, double xmin, double xmax, double ymin, double ymax, double zmin, double zmax,
-                            char *filename, int sst);
-   double L2Norm(var *vf);
-   void L2Norm7(var **vf, double *norms);
-   void Find_Maximum(MyList<var> *VarList, double *XX, double *Shellf);
-};
+   void write_Pablo_file_ss(int *ext, double xmin, double xmax, double ymin, double ymax, double zmin, double zmax,
+                            char *filename, int sst);
+   double L2Norm(var *vf);
+   void L2Norm7(var **vf, double *norms);
+   void Find_Maximum(MyList<var> *VarList, double *XX, double *Shellf);
+};
 
 #endif /* SHELLPATCH_H */

AMSS_NCKU_source/Z4c_class.C

@@ -1,9 +1,10 @@
 
-#ifdef newc
-#include <sstream>
-#include <cstdio>
-#include <map>
-using namespace std;
+#ifdef newc
+#include <sstream>
+#include <cstdio>
+#include <cstdlib>
+#include <map>
+using namespace std;
 #else
 #include <stdio.h>
 #include <map.h>
@@ -26,12 +27,20 @@ using namespace std;
 #include "shellfunctions.h"
 #include "cpbc.h"
 #include "kodiss.h"
-#include "parameters.h"
-
-#ifdef With_AHF
-#include "derivatives.h"
-#include "myglobal.h"
-#endif
+#include "parameters.h"
+
+#ifndef USE_CUDA_Z4C
+#define USE_CUDA_Z4C 0
+#endif
+
+#if USE_CUDA_Z4C && (ABEtype == 2)
+#include "z4c_rhs_cuda.h"
+#endif
+
+#ifdef With_AHF
+#include "derivatives.h"
+#include "myglobal.h"
+#endif
 
 //================================================================================================
 
@@ -105,20 +114,22 @@ void Z4c_class::Initialize()
   else
     GH->compose_cgh(nprocs);
 
-#ifdef WithShell
-  SH = new ShellPatch(0, ngfs, pname, Symmetry, myrank, ErrorMonitor);
-  if (!checkrun)
-    SH->matchcheck(GH->PatL[0]);
+#ifdef WithShell
+  SH = new ShellPatch(0, ngfs, pname, Symmetry, myrank, ErrorMonitor);
+  if (!checkrun)
+    SH->matchcheck(GH->PatL[0]);
   SH->compose_sh(nprocs);
   SH->setupcordtrans();
   SH->Dump_xyz(0, 0, 1);
   SH->setupintintstuff(nprocs, GH->PatL[0], Symmetry);
 
-  if (checkrun)
-    CheckPoint->readcheck_sh(SH, myrank);
-#endif
-
-  double h = GH->PatL[0]->data->blb->data->getdX(0);
+  if (checkrun)
+    CheckPoint->readcheck_sh(SH, myrank);
+#endif
+
+  Initialize_Level_Runtime();
+
+  double h = GH->PatL[0]->data->blb->data->getdX(0);
   for (int i = 1; i < dim; i++)
     h = Mymin(h, GH->PatL[0]->data->blb->data->getdX(i));
   dT = Courant * h;
@@ -167,12 +178,753 @@ Z4c_class::~Z4c_class()
 
 #define MRBD 0 // 0: fix BD for meshrefinement level; 1: sommerfeld_bam for them; 2: sommerfeld_yo for them
 
-#ifndef CPBC
-// for sommerfeld boundary
-
-void Z4c_class::Step(int lev, int YN)
-{
-  double dT_lev = dT * pow(0.5, Mymax(lev, trfls));
+#ifndef CPBC
+// for sommerfeld boundary
+
+#if USE_CUDA_Z4C && (ABEtype == 2)
+#ifdef WithShell
+#error "USE_CUDA_Z4C resident path currently supports Cartesian non-shell Z4C only"
+#endif
+#if (MRBD == 2)
+#error "USE_CUDA_Z4C resident path does not support MRBD == 2"
+#endif
+
+namespace {
+
+static const int k_z4c_cuda_bh_state_indices[3] = {18, 19, 20};
+
+bool fill_z4c_cuda_views(Block *cg, MyList<var> *vars,
+                         double **host_views,
+                         double *propspeeds = 0,
+                         double *soa_flat = 0)
+{
+  int idx = 0;
+  while (vars && idx < Z4C_CUDA_STATE_COUNT)
+  {
+    host_views[idx] = cg->fgfs[vars->data->sgfn];
+    if (propspeeds)
+      propspeeds[idx] = vars->data->propspeed;
+    if (soa_flat)
+    {
+      soa_flat[3 * idx + 0] = vars->data->SoA[0];
+      soa_flat[3 * idx + 1] = vars->data->SoA[1];
+      soa_flat[3 * idx + 2] = vars->data->SoA[2];
+    }
+    vars = vars->next;
+    ++idx;
+  }
+  return idx == Z4C_CUDA_STATE_COUNT && vars == 0;
+}
+
+bool z4c_cuda_keep_resident_after_step(int lev, int trfls_in, int analysis_lev)
+{
+  static int keep_all_levels = -1;
+  if (keep_all_levels < 0)
+  {
+    const char *env = getenv("AMSS_CUDA_KEEP_ALL_LEVELS");
+    keep_all_levels = (env && atoi(env) != 0) ? 1 : 0;
+  }
+  static int enabled = -1;
+  if (enabled < 0)
+  {
+    const char *env = getenv("AMSS_CUDA_Z4C_KEEP_RESIDENT_AFTER_STEP");
+    if (env)
+      enabled = (atoi(env) != 0) ? 1 : 0;
+    else
+    {
+      env = getenv("AMSS_CUDA_KEEP_RESIDENT_AFTER_STEP");
+      enabled = (env && atoi(env) != 0) ? 1 : 0;
+    }
+  }
+  if (!enabled)
+    return false;
+  if (lev == analysis_lev)
+    return false;
+  if (keep_all_levels)
+    return true;
+  return lev < trfls_in;
+}
+
+void z4c_cuda_download_level_state(MyList<Patch> *PatL, MyList<var> *vars, int myrank, bool release_ctx)
+{
+  MyList<Patch> *Pp = PatL;
+  while (Pp)
+  {
+    MyList<Block> *BP = Pp->data->blb;
+    while (BP)
+    {
+      Block *cg = BP->data;
+      if (myrank == cg->rank && z4c_cuda_has_resident_state(cg))
+      {
+        double *state_out[Z4C_CUDA_STATE_COUNT];
+        if (!fill_z4c_cuda_views(cg, vars, state_out))
+        {
+          cout << "CUDA Z4C state list mismatch on resident state download" << endl;
+          MPI_Abort(MPI_COMM_WORLD, 1);
+        }
+        if (z4c_cuda_download_resident_state(cg, cg->shape, state_out))
+        {
+          cout << "CUDA Z4C resident state download failed" << endl;
+          MPI_Abort(MPI_COMM_WORLD, 1);
+        }
+        if (release_ctx)
+          z4c_cuda_release_step_ctx(cg);
+      }
+      if (BP == Pp->data->ble)
+        break;
+      BP = BP->next;
+    }
+    Pp = Pp->next;
+  }
+}
+
+bool z4c_cuda_patch_contains_point(Patch *patch, const double *point)
+{
+  if (!patch)
+    return false;
+  for (int d = 0; d < dim; d++)
+  {
+    const double h = patch->getdX(d);
+    const double lo = patch->bbox[d] + patch->lli[d] * h;
+    const double hi = patch->bbox[dim + d] - patch->uui[d] * h;
+    if (point[d] < lo || point[d] > hi)
+      return false;
+  }
+  return true;
+}
+
+bool z4c_cuda_point_in_block(Patch *patch, Block *block,
+                             const double *point, const double *DH)
+{
+  if (!patch || !block)
+    return false;
+  for (int d = 0; d < dim; d++)
+  {
+    double llb;
+    double uub;
+#ifdef Vertex
+#ifdef Cell
+#error Both Cell and Vertex are defined
+#endif
+    llb = (feq(block->bbox[d], patch->bbox[d], DH[d] / 2))
+              ? block->bbox[d] + patch->lli[d] * DH[d]
+              : block->bbox[d] + (ghost_width - 0.5) * DH[d];
+    uub = (feq(block->bbox[dim + d], patch->bbox[dim + d], DH[d] / 2))
+              ? block->bbox[dim + d] - patch->uui[d] * DH[d]
+              : block->bbox[dim + d] - (ghost_width - 0.5) * DH[d];
+#else
+#ifdef Cell
+    llb = (feq(block->bbox[d], patch->bbox[d], DH[d] / 2))
+              ? block->bbox[d] + patch->lli[d] * DH[d]
+              : block->bbox[d] + ghost_width * DH[d];
+    uub = (feq(block->bbox[dim + d], patch->bbox[dim + d], DH[d] / 2))
+              ? block->bbox[dim + d] - patch->uui[d] * DH[d]
+              : block->bbox[dim + d] - ghost_width * DH[d];
+#else
+#error Not define Vertex nor Cell
+#endif
+#endif
+    if (point[d] - llb < -DH[d] / 2 || point[d] - uub > DH[d] / 2)
+      return false;
+  }
+  return true;
+}
+
+int z4c_cuda_interp_tile_start(const double *coords, int n, double x, double dx, int ordn)
+{
+  if (!coords || n <= ordn)
+    return 0;
+  int cxi = int((x - coords[0]) / dx + 0.4) + 1;
+  int start = cxi - ordn / 2;
+  if (start < 0)
+    start = 0;
+  const int max_start = n - ordn;
+  if (start > max_start)
+    start = max_start;
+  return start;
+}
+
+bool z4c_cuda_interp_bh_point_resident(MyList<Patch> *PatL,
+                                       int myrank,
+                                       const double *point,
+                                       var *forx, var *fory, var *forz,
+                                       int Symmetry,
+                                       double *shellf)
+{
+  const int ordn = 2 * ghost_width;
+  int owner_rank = -1;
+
+  shellf[0] = shellf[1] = shellf[2] = 0.0;
+
+  MyList<Patch> *PL = PatL;
+  while (PL)
+  {
+    Patch *patch = PL->data;
+    if (!z4c_cuda_patch_contains_point(patch, point))
+    {
+      PL = PL->next;
+      continue;
+    }
+
+    double DH[dim];
+    for (int d = 0; d < dim; d++)
+      DH[d] = patch->getdX(d);
+
+    MyList<Block> *BP = patch->blb;
+    while (BP)
+    {
+      Block *block = BP->data;
+      if (z4c_cuda_point_in_block(patch, block, point, DH))
+      {
+        owner_rank = block->rank;
+        if (myrank == owner_rank)
+        {
+          int interp_ordn = ordn;
+          int interp_sym = Symmetry;
+          double x = point[0];
+          double y = point[1];
+          double z = point[2];
+
+          if (z4c_cuda_has_resident_state(block) &&
+              block->shape[0] >= ordn && block->shape[1] >= ordn && block->shape[2] >= ordn)
+          {
+            var *vars[3] = {forx, fory, forz};
+            static int use_device_bh_interp = -1;
+            if (use_device_bh_interp < 0)
+            {
+              const char *env = getenv("AMSS_CUDA_Z4C_BH_INTERP_DEVICE");
+              use_device_bh_interp = (env && atoi(env) != 0) ? 1 : 0;
+            }
+            bool used_device_interp = false;
+            if (use_device_bh_interp)
+            {
+              double soa3[9];
+              for (int f = 0; f < 3; f++)
+              {
+                soa3[3 * f + 0] = vars[f]->SoA[0];
+                soa3[3 * f + 1] = vars[f]->SoA[1];
+                soa3[3 * f + 2] = vars[f]->SoA[2];
+              }
+              used_device_interp =
+                  (z4c_cuda_interp_state_point3(block, block->shape,
+                                                k_z4c_cuda_bh_state_indices[0],
+                                                k_z4c_cuda_bh_state_indices[1],
+                                                k_z4c_cuda_bh_state_indices[2],
+                                                block->X[0][0], block->X[1][0], block->X[2][0],
+                                                DH[0], DH[1], DH[2],
+                                                x, y, z,
+                                                interp_ordn, interp_sym,
+                                                soa3, shellf) == 0);
+            }
+            if (!used_device_interp)
+            {
+              double *shift_views[3] = {
+                  block->fgfs[forx->sgfn],
+                  block->fgfs[fory->sgfn],
+                  block->fgfs[forz->sgfn]};
+              if (z4c_cuda_download_state_subset(block, block->shape, 3,
+                                                 k_z4c_cuda_bh_state_indices,
+                                                 shift_views) != 0)
+              {
+                cout << "CUDA Z4C BH shift download failed" << endl;
+                MPI_Abort(MPI_COMM_WORLD, 1);
+              }
+              f_global_interp(block->shape, block->X[0], block->X[1], block->X[2],
+                              block->fgfs[forx->sgfn], shellf[0],
+                              x, y, z, interp_ordn, forx->SoA, interp_sym);
+              f_global_interp(block->shape, block->X[0], block->X[1], block->X[2],
+                              block->fgfs[fory->sgfn], shellf[1],
+                              x, y, z, interp_ordn, fory->SoA, interp_sym);
+              f_global_interp(block->shape, block->X[0], block->X[1], block->X[2],
+                              block->fgfs[forz->sgfn], shellf[2],
+                              x, y, z, interp_ordn, forz->SoA, interp_sym);
+            }
+          }
+          else
+          {
+            f_global_interp(block->shape, block->X[0], block->X[1], block->X[2],
+                            block->fgfs[forx->sgfn], shellf[0],
+                            x, y, z, interp_ordn, forx->SoA, interp_sym);
+            f_global_interp(block->shape, block->X[0], block->X[1], block->X[2],
+                            block->fgfs[fory->sgfn], shellf[1],
+                            x, y, z, interp_ordn, fory->SoA, interp_sym);
+            f_global_interp(block->shape, block->X[0], block->X[1], block->X[2],
+                            block->fgfs[forz->sgfn], shellf[2],
+                            x, y, z, interp_ordn, forz->SoA, interp_sym);
+          }
+        }
+        break;
+      }
+      if (BP == patch->ble)
+        break;
+      BP = BP->next;
+    }
+
+    if (owner_rank >= 0)
+      break;
+    PL = PL->next;
+  }
+
+  if (owner_rank < 0)
+    return false;
+
+  MPI_Bcast(shellf, 3, MPI_DOUBLE, owner_rank, MPI_COMM_WORLD);
+  return true;
+}
+
+bool z4c_cuda_compute_porg_rhs_resident(cgh *GH,
+                                        int ilev,
+                                        int myrank,
+                                        int BH_num,
+                                        double **BH_PS,
+                                        double **BH_RHS,
+                                        var *forx, var *fory, var *forz,
+                                        int Symmetry)
+{
+  for (int n = 0; n < BH_num; n++)
+  {
+    double shellf[3] = {0.0, 0.0, 0.0};
+    int lev = ilev;
+    while (lev >= 0 &&
+           !z4c_cuda_interp_bh_point_resident(GH->PatL[lev], myrank, BH_PS[n],
+                                              forx, fory, forz, Symmetry, shellf))
+    {
+      --lev;
+    }
+    if (lev < 0)
+      return false;
+    BH_RHS[n][0] = -shellf[0];
+    BH_RHS[n][1] = -shellf[1];
+    BH_RHS[n][2] = -shellf[2];
+  }
+  return true;
+}
+
+bool z4c_cuda_download_bh_shift_level(MyList<Patch> *PatL,
+                                      int myrank,
+                                      var *forx, var *fory, var *forz)
+{
+  MyList<Patch> *Pp = PatL;
+  while (Pp)
+  {
+    MyList<Block> *BP = Pp->data->blb;
+    while (BP)
+    {
+      Block *cg = BP->data;
+      if (myrank == cg->rank && z4c_cuda_has_resident_state(cg))
+      {
+        double *fields[3] = {
+            cg->fgfs[forx->sgfn],
+            cg->fgfs[fory->sgfn],
+            cg->fgfs[forz->sgfn]};
+        if (z4c_cuda_download_state_subset(cg, cg->shape, 3,
+                                           k_z4c_cuda_bh_state_indices,
+                                           fields))
+          return false;
+      }
+      if (BP == Pp->data->ble)
+        break;
+      BP = BP->next;
+    }
+    Pp = Pp->next;
+  }
+  return true;
+}
+
+bool z4c_cuda_refresh_constraint_level(MyList<Patch> *PatL,
+                                       int myrank,
+                                       var *Cons_Ham, var *Cons_Px,
+                                       var *Cons_Py, var *Cons_Pz,
+                                       var *Cons_Gx, var *Cons_Gy,
+                                       var *Cons_Gz, var *TZ0,
+                                       int Symmetry, int lev, double eps)
+{
+  bool all_resident = true;
+  const int tz_index = 24;
+  MyList<Patch> *Pp = PatL;
+  while (Pp)
+  {
+    MyList<Block> *BP = Pp->data->blb;
+    while (BP)
+    {
+      Block *cg = BP->data;
+      if (myrank == cg->rank)
+      {
+        if (!z4c_cuda_has_resident_state(cg))
+        {
+          all_resident = false;
+        }
+        else
+        {
+          double *constraints[7] = {
+              cg->fgfs[Cons_Ham->sgfn], cg->fgfs[Cons_Px->sgfn],
+              cg->fgfs[Cons_Py->sgfn], cg->fgfs[Cons_Pz->sgfn],
+              cg->fgfs[Cons_Gx->sgfn], cg->fgfs[Cons_Gy->sgfn],
+              cg->fgfs[Cons_Gz->sgfn]};
+          double *tz_out[1] = {cg->fgfs[TZ0->sgfn]};
+          int co = 0;
+          if (z4c_cuda_compute_constraints_resident(cg, cg->shape,
+                                                   cg->X[0], cg->X[1], cg->X[2],
+                                                   Symmetry, eps, co,
+                                                   constraints) ||
+              z4c_cuda_download_state_subset(cg, cg->shape, 1, &tz_index, tz_out))
+          {
+            cout << "CUDA Z4C resident constraint refresh failed" << endl;
+            MPI_Abort(MPI_COMM_WORLD, 1);
+          }
+        }
+      }
+      if (BP == Pp->data->ble)
+        break;
+      BP = BP->next;
+    }
+    Pp = Pp->next;
+  }
+  return all_resident;
+}
+
+long long &z4c_constraint_output_counter()
+{
+  static long long counter = 0;
+  return counter;
+}
+
+int z4c_constraint_output_every()
+{
+  static int every = -1;
+  if (every < 0)
+  {
+    const char *env = getenv("AMSS_CUDA_Z4C_CONSTRAINT_EVERY");
+    every = (env && atoi(env) > 0) ? atoi(env) : 1;
+  }
+  return every;
+}
+
+bool z4c_constraint_output_due_now()
+{
+  const int every = z4c_constraint_output_every();
+  return every <= 1 || (z4c_constraint_output_counter() % every) == 0;
+}
+
+void z4c_constraint_output_advance()
+{
+  z4c_constraint_output_counter()++;
+}
+
+} // namespace
+#endif
+
+void Z4c_class::Step(int lev, int YN)
+{
+#if USE_CUDA_Z4C && (ABEtype == 2)
+  double dT_lev = dT * pow(0.5, Mymax(lev, trfls));
+#ifdef With_AHF
+  AH_Step_Find(lev, dT_lev);
+#endif
+  bool BB = fgt(PhysTime, StartTime, dT_lev / 2);
+  double ndeps = numepss;
+  if (lev < GH->movls)
+    ndeps = numepsb;
+  double TRK4 = PhysTime;
+  int iter_count = 0;
+  int pre = 0, cor = 1;
+  int ERROR = 0;
+  const double dT_mon = dT * pow(0.5, Mymax(0, trfls));
+  const bool need_constraint_after_step =
+      (LastConsOut + dT_mon >= AnasTime) && z4c_constraint_output_due_now();
+
+  if (BH_num > 0 && lev == GH->levels - 1)
+  {
+    if (!z4c_cuda_download_bh_shift_level(GH->PatL[lev], myrank, Sfx0, Sfy0, Sfz0))
+    {
+      if (myrank == 0 && ErrorMonitor->outfile)
+        ErrorMonitor->outfile << "CUDA Z4C failed to download predictor black-hole shift at t = "
+                              << PhysTime << endl;
+      MPI_Abort(MPI_COMM_WORLD, 1);
+    }
+    compute_Porg_rhs(Porg0, Porg_rhs, Sfx0, Sfy0, Sfz0, lev);
+    for (int ithBH = 0; ithBH < BH_num; ithBH++)
+    {
+      f_rungekutta4_scalar(dT_lev, Porg0[ithBH][0], Porg[ithBH][0], Porg_rhs[ithBH][0], iter_count);
+      f_rungekutta4_scalar(dT_lev, Porg0[ithBH][1], Porg[ithBH][1], Porg_rhs[ithBH][1], iter_count);
+      f_rungekutta4_scalar(dT_lev, Porg0[ithBH][2], Porg[ithBH][2], Porg_rhs[ithBH][2], iter_count);
+      if (Symmetry > 0)
+        Porg[ithBH][2] = fabs(Porg[ithBH][2]);
+      if (Symmetry == 2)
+      {
+        Porg[ithBH][0] = fabs(Porg[ithBH][0]);
+        Porg[ithBH][1] = fabs(Porg[ithBH][1]);
+      }
+    }
+  }
+
+  MyList<Patch> *Pp = GH->PatL[lev];
+  while (Pp)
+  {
+    MyList<Block> *BP = Pp->data->blb;
+    while (BP)
+    {
+      Block *cg = BP->data;
+      if (myrank == cg->rank)
+      {
+        double *state_in[Z4C_CUDA_STATE_COUNT];
+        double *state_out[Z4C_CUDA_STATE_COUNT];
+        double propspeed[Z4C_CUDA_STATE_COUNT];
+        double soa_flat[3 * Z4C_CUDA_STATE_COUNT];
+        if (!fill_z4c_cuda_views(cg, StateList, state_in, propspeed, soa_flat) ||
+            !fill_z4c_cuda_views(cg, SynchList_pre, state_out))
+        {
+          cout << "CUDA Z4C state list mismatch on predictor step" << endl;
+          MPI_Abort(MPI_COMM_WORLD, 1);
+        }
+        int apply_bam_bc = 0;
+#if (MRBD == 0)
+#if (SommerType == 0)
+        apply_bam_bc = (lev == 0) ? 1 : 0;
+#endif
+#elif (MRBD == 1)
+        apply_bam_bc = 1;
+#endif
+        int keep_resident_state = 1;
+        int apply_enforce_ga = 0;
+#if (AGM == 0)
+        apply_enforce_ga = 1;
+#endif
+        if (z4c_cuda_rk4_substep(cg,
+                                 cg->shape, cg->X[0], cg->X[1], cg->X[2],
+                                 state_in, state_out,
+                                 propspeed, soa_flat, Pp->data->bbox,
+                                 dT_lev, TRK4, iter_count, apply_bam_bc,
+                                 Symmetry, lev, ndeps, pre,
+                                 keep_resident_state, apply_enforce_ga, chitiny))
+        {
+          cout << "CUDA Z4C predictor substep failed in domain: ("
+               << cg->bbox[0] << ":" << cg->bbox[3] << ","
+               << cg->bbox[1] << ":" << cg->bbox[4] << ","
+               << cg->bbox[2] << ":" << cg->bbox[5] << ")" << endl;
+          ERROR = 1;
+        }
+      }
+      if (BP == Pp->data->ble)
+        break;
+      BP = BP->next;
+    }
+    Pp = Pp->next;
+  }
+
+  {
+    int erh = ERROR;
+    MPI_Allreduce(&erh, &ERROR, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD);
+  }
+  if (ERROR)
+  {
+    if (myrank == 0 && ErrorMonitor->outfile)
+      ErrorMonitor->outfile << "CUDA Z4C failed in predictor at t = " << PhysTime
+                            << ", lev = " << lev << endl;
+    MPI_Abort(MPI_COMM_WORLD, 1);
+  }
+
+  {
+    Parallel::AsyncSyncState async_pre;
+    Parallel::Sync_start(GH->PatL[lev], SynchList_pre, Symmetry, sync_cache_pre[lev], async_pre);
+    Parallel::Sync_finish(sync_cache_pre[lev], async_pre, SynchList_pre, Symmetry);
+  }
+
+  if ((lev == a_lev) && (LastAnas + dT_lev >= AnasTime))
+    z4c_cuda_download_level_state(GH->PatL[lev], SynchList_pre, myrank, false);
+  if (lev == a_lev)
+    AnalysisStuff(lev, dT_lev);
+
+  for (iter_count = 1; iter_count < 4; iter_count++)
+  {
+    if (iter_count == 1 || iter_count == 3)
+      TRK4 += dT_lev / 2;
+    Pp = GH->PatL[lev];
+    while (Pp)
+    {
+      MyList<Block> *BP = Pp->data->blb;
+      while (BP)
+      {
+        Block *cg = BP->data;
+        if (myrank == cg->rank)
+        {
+          double *state_in[Z4C_CUDA_STATE_COUNT];
+          double *state_out[Z4C_CUDA_STATE_COUNT];
+          double propspeed[Z4C_CUDA_STATE_COUNT];
+          double soa_flat[3 * Z4C_CUDA_STATE_COUNT];
+          if (!fill_z4c_cuda_views(cg, SynchList_pre, state_in, propspeed, soa_flat) ||
+              !fill_z4c_cuda_views(cg, SynchList_cor, state_out))
+          {
+            cout << "CUDA Z4C state list mismatch on corrector step" << endl;
+            MPI_Abort(MPI_COMM_WORLD, 1);
+          }
+          int apply_bam_bc = 0;
+#if (MRBD == 0)
+#if (SommerType == 0)
+          apply_bam_bc = (lev == 0) ? 1 : 0;
+#endif
+#elif (MRBD == 1)
+          apply_bam_bc = 1;
+#endif
+          int keep_resident_state = 1;
+          int apply_enforce_ga = 0;
+#if (AGM == 0)
+          apply_enforce_ga = 1;
+#elif (AGM == 1)
+          apply_enforce_ga = (iter_count == 3) ? 1 : 0;
+#endif
+          if (z4c_cuda_rk4_substep(cg,
+                                   cg->shape, cg->X[0], cg->X[1], cg->X[2],
+                                   state_in, state_out,
+                                   propspeed, soa_flat, Pp->data->bbox,
+                                   dT_lev, TRK4, iter_count, apply_bam_bc,
+                                   Symmetry, lev, ndeps, cor,
+                                   keep_resident_state, apply_enforce_ga, chitiny))
+          {
+            cout << "CUDA Z4C corrector substep failed in domain: ("
+                 << cg->bbox[0] << ":" << cg->bbox[3] << ","
+                 << cg->bbox[1] << ":" << cg->bbox[4] << ","
+                 << cg->bbox[2] << ":" << cg->bbox[5] << ")" << endl;
+            ERROR = 1;
+          }
+          if (!ERROR && iter_count == 3 && need_constraint_after_step)
+          {
+            double *constraints[7] = {
+                cg->fgfs[Cons_Ham->sgfn], cg->fgfs[Cons_Px->sgfn],
+                cg->fgfs[Cons_Py->sgfn], cg->fgfs[Cons_Pz->sgfn],
+                cg->fgfs[Cons_Gx->sgfn], cg->fgfs[Cons_Gy->sgfn],
+                cg->fgfs[Cons_Gz->sgfn]};
+            double *tz_out[1] = {cg->fgfs[TZ0->sgfn]};
+            const int tz_index = 24;
+            if (z4c_cuda_download_constraint_outputs(cg->shape, constraints) ||
+                z4c_cuda_download_state_subset(cg, cg->shape, 1, &tz_index, tz_out))
+            {
+              cout << "CUDA Z4C constraint download failed in domain: ("
+                   << cg->bbox[0] << ":" << cg->bbox[3] << ","
+                   << cg->bbox[1] << ":" << cg->bbox[4] << ","
+                   << cg->bbox[2] << ":" << cg->bbox[5] << ")" << endl;
+              ERROR = 1;
+            }
+          }
+        }
+        if (BP == Pp->data->ble)
+          break;
+        BP = BP->next;
+      }
+      Pp = Pp->next;
+    }
+
+    {
+      int erh = ERROR;
+      MPI_Allreduce(&erh, &ERROR, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD);
+    }
+    if (ERROR)
+    {
+      if (myrank == 0 && ErrorMonitor->outfile)
+        ErrorMonitor->outfile << "CUDA Z4C failed in RK4 substep#" << iter_count
+                              << " at t = " << PhysTime
+                              << ", lev = " << lev << endl;
+      MPI_Abort(MPI_COMM_WORLD, 1);
+    }
+
+    {
+      Parallel::AsyncSyncState async_cor;
+      Parallel::Sync_start(GH->PatL[lev], SynchList_cor, Symmetry, sync_cache_cor[lev], async_cor);
+      Parallel::Sync_finish(sync_cache_cor[lev], async_cor, SynchList_cor, Symmetry);
+    }
+
+    if (BH_num > 0 && lev == GH->levels - 1)
+    {
+      if (!z4c_cuda_compute_porg_rhs_resident(GH, lev, myrank, BH_num,
+                                              Porg, Porg1,
+                                              Sfx, Sfy, Sfz, Symmetry))
+      {
+        if (myrank == 0 && ErrorMonitor->outfile)
+          ErrorMonitor->outfile << "CUDA Z4C failed to interpolate black-hole shift at t = "
+                                << PhysTime << endl;
+        MPI_Abort(MPI_COMM_WORLD, 1);
+      }
+      for (int ithBH = 0; ithBH < BH_num; ithBH++)
+      {
+        f_rungekutta4_scalar(dT_lev, Porg0[ithBH][0], Porg1[ithBH][0], Porg_rhs[ithBH][0], iter_count);
+        f_rungekutta4_scalar(dT_lev, Porg0[ithBH][1], Porg1[ithBH][1], Porg_rhs[ithBH][1], iter_count);
+        f_rungekutta4_scalar(dT_lev, Porg0[ithBH][2], Porg1[ithBH][2], Porg_rhs[ithBH][2], iter_count);
+        if (Symmetry > 0)
+          Porg1[ithBH][2] = fabs(Porg1[ithBH][2]);
+        if (Symmetry == 2)
+        {
+          Porg1[ithBH][0] = fabs(Porg1[ithBH][0]);
+          Porg1[ithBH][1] = fabs(Porg1[ithBH][1]);
+        }
+      }
+    }
+
+    if (iter_count < 3)
+    {
+      Pp = GH->PatL[lev];
+      while (Pp)
+      {
+        MyList<Block> *BP = Pp->data->blb;
+        while (BP)
+        {
+          Block *cg = BP->data;
+          cg->swapList(SynchList_pre, SynchList_cor, myrank);
+          if (BP == Pp->data->ble)
+            break;
+          BP = BP->next;
+        }
+        Pp = Pp->next;
+      }
+      if (BH_num > 0 && lev == GH->levels - 1)
+      {
+        for (int ithBH = 0; ithBH < BH_num; ithBH++)
+        {
+          Porg[ithBH][0] = Porg1[ithBH][0];
+          Porg[ithBH][1] = Porg1[ithBH][1];
+          Porg[ithBH][2] = Porg1[ithBH][2];
+        }
+      }
+    }
+  }
+
+  {
+    const bool keep_resident = z4c_cuda_keep_resident_after_step(lev, trfls, a_lev);
+    const bool need_host_after_step =
+        ((lev == a_lev) && (LastAnas + dT_lev >= AnasTime));
+    if (!keep_resident || need_host_after_step)
+      z4c_cuda_download_level_state(GH->PatL[lev], SynchList_cor, myrank, !keep_resident);
+  }
+
+#if (RPS == 0)
+  RestrictProlong(lev, YN, BB);
+#endif
+
+  Pp = GH->PatL[lev];
+  while (Pp)
+  {
+    MyList<Block> *BP = Pp->data->blb;
+    while (BP)
+    {
+      Block *cg = BP->data;
+      cg->swapList(StateList, SynchList_cor, myrank);
+      cg->swapList(OldStateList, SynchList_cor, myrank);
+      if (BP == Pp->data->ble)
+        break;
+      BP = BP->next;
+    }
+    Pp = Pp->next;
+  }
+  if (BH_num > 0 && lev == GH->levels - 1)
+  {
+    for (int ithBH = 0; ithBH < BH_num; ithBH++)
+    {
+      Porg0[ithBH][0] = Porg1[ithBH][0];
+      Porg0[ithBH][1] = Porg1[ithBH][1];
+      Porg0[ithBH][2] = Porg1[ithBH][2];
+    }
+  }
+#else
+  double dT_lev = dT * pow(0.5, Mymax(lev, trfls));
 #ifdef With_AHF
   AH_Step_Find(lev, dT_lev);
 #endif
@@ -1039,15 +1791,19 @@ void Z4c_class::Step(int lev, int YN)
     {
       Porg0[ithBH][0] = Porg1[ithBH][0];
       Porg0[ithBH][1] = Porg1[ithBH][1];
-      Porg0[ithBH][2] = Porg1[ithBH][2];
-    }
-  }
-}
-#else
-// for constraint preserving boundary (CPBC)
-#ifndef WithShell
-#error "CPBC only supports Shell"
-#endif
+      Porg0[ithBH][2] = Porg1[ithBH][2];
+    }
+  }
+#endif
+}
+#else
+// for constraint preserving boundary (CPBC)
+#if USE_CUDA_Z4C && (ABEtype == 2)
+#error "USE_CUDA_Z4C resident path does not support CPBC"
+#endif
+#ifndef WithShell
+#error "CPBC only supports Shell"
+#endif
 
 // 0: extroplate rhs, 1: extroplate variable
 // 2: extroplate variable but before RHS calculation
@@ -2408,17 +3164,23 @@ void Z4c_class::Check_extrop()
 
 //================================================================================================
 
-void Z4c_class::Constraint_Out()
-{
-  // here we have to use the same variable name as in the parent class
-  LastConsOut += dT * pow(0.5, Mymax(0, trfls));
-  
-  if (LastConsOut >= AnasTime)
-  // Constraint violation
-  {
-    // recompute least the constraint data lost for moved new grid
-    for (int lev = 0; lev < GH->levels; lev++)
-    {
+void Z4c_class::Constraint_Out()
+{
+  // here we have to use the same variable name as in the parent class
+  LastConsOut += dT * pow(0.5, Mymax(0, trfls));
+  
+  if (LastConsOut >= AnasTime)
+  // Constraint violation
+  {
+#if USE_CUDA_Z4C && (ABEtype == 2)
+    bool cuda_constraints_ready = true;
+#else
+    const bool cuda_constraints_ready = false;
+#endif
+    // recompute least the constraint data lost for moved new grid
+    if (!cuda_constraints_ready)
+      for (int lev = 0; lev < GH->levels; lev++)
+    {
       // make sure the data consistent for higher levels
       if (lev > 0)
       {

AMSS_NCKU_source/bssnEM_class.C

@@ -15,10 +15,13 @@ using namespace std;
 #include "misc.h"
 #include "Ansorg.h"
 #include "fmisc.h"
-#include "Parallel.h"
-#include "bssnEM_class.h"
-#include "bssn_rhs.h"
-#include "empart.h"
+#include "Parallel.h"
+#include "bssnEM_class.h"
+#include "bssn_rhs.h"
+#if USE_CUDA_BSSN
+#include "bssn_rhs_cuda.h"
+#endif
+#include "empart.h"
 #include "initial_puncture.h"
 #include "initial_maxwell.h"
 #include "enforce_algebra.h"
@@ -32,11 +35,111 @@ using namespace std;
 #ifdef With_AHF
 #include "derivatives.h"
 #include "myglobal.h"
-#endif
-
-//================================================================================================
-
-// Define bssnEM_class
+#endif
+
+//================================================================================================
+
+#if USE_CUDA_BSSN
+namespace {
+
+bool fill_bssn_cuda_views_prefix(Block *cg, MyList<var> *vars,
+                                 double **host_views,
+                                 double *propspeeds = nullptr,
+                                 double *soa_flat = nullptr)
+{
+  int idx = 0;
+  while (vars && idx < BSSN_CUDA_STATE_COUNT)
+  {
+    host_views[idx] = cg->fgfs[vars->data->sgfn];
+    if (propspeeds)
+      propspeeds[idx] = vars->data->propspeed;
+    if (soa_flat)
+    {
+      soa_flat[3 * idx + 0] = vars->data->SoA[0];
+      soa_flat[3 * idx + 1] = vars->data->SoA[1];
+      soa_flat[3 * idx + 2] = vars->data->SoA[2];
+    }
+    vars = vars->next;
+    ++idx;
+  }
+  return idx == BSSN_CUDA_STATE_COUNT;
+}
+
+void skip_bssn_cuda_prefix(MyList<var> *&a, MyList<var> *&b, MyList<var> *&c)
+{
+  for (int i = 0; i < BSSN_CUDA_STATE_COUNT && a && b && c; ++i)
+  {
+    a = a->next;
+    b = b->next;
+    c = c->next;
+  }
+}
+
+void skip_bssn_cuda_prefix(MyList<var> *&a, MyList<var> *&b,
+                           MyList<var> *&c, MyList<var> *&d)
+{
+  for (int i = 0; i < BSSN_CUDA_STATE_COUNT && a && b && c && d; ++i)
+  {
+    a = a->next;
+    b = b->next;
+    c = c->next;
+    d = d->next;
+  }
+}
+
+int run_bssn_em_cuda_substep(Block *cg,
+                             MyList<var> *state_in_list,
+                             MyList<var> *state_out_list,
+                             Patch *patch,
+                             double &dT_lev,
+                             double &TRK4,
+                             int &iter_count,
+                             int &Symmetry,
+                             int lev,
+                             double &ndeps,
+                             int &co,
+                             double &chitiny,
+                             var *rho, var *Sx, var *Sy, var *Sz,
+                             var *Sxx, var *Sxy, var *Sxz,
+                             var *Syy, var *Syz, var *Szz)
+{
+  double *state_in[BSSN_CUDA_STATE_COUNT];
+  double *state_out[BSSN_CUDA_STATE_COUNT];
+  double *matter[BSSN_CUDA_MATTER_COUNT] = {
+      cg->fgfs[rho->sgfn], cg->fgfs[Sx->sgfn], cg->fgfs[Sy->sgfn], cg->fgfs[Sz->sgfn],
+      cg->fgfs[Sxx->sgfn], cg->fgfs[Sxy->sgfn], cg->fgfs[Sxz->sgfn],
+      cg->fgfs[Syy->sgfn], cg->fgfs[Syz->sgfn], cg->fgfs[Szz->sgfn]};
+  double propspeed[BSSN_CUDA_STATE_COUNT];
+  double soa_flat[3 * BSSN_CUDA_STATE_COUNT];
+  if (!fill_bssn_cuda_views_prefix(cg, state_in_list, state_in, propspeed, soa_flat) ||
+      !fill_bssn_cuda_views_prefix(cg, state_out_list, state_out))
+    return 1;
+
+  int apply_bam_bc = 0;
+#if (SommerType == 0)
+#ifndef WithShell
+  apply_bam_bc = (lev == 0) ? 1 : 0;
+#endif
+#endif
+  int use_zero_matter = 0;
+  int keep_resident_state = 0;
+  int apply_enforce_ga = 0;
+  return bssn_cuda_rk4_substep(cg,
+                               cg->shape, cg->X[0], cg->X[1], cg->X[2],
+                               state_in, state_out, matter,
+                               propspeed, soa_flat, patch->bbox,
+                               dT_lev, TRK4, iter_count, apply_bam_bc,
+                               Symmetry, lev, ndeps, co,
+                               use_zero_matter,
+                               keep_resident_state, apply_enforce_ga, chitiny);
+}
+
+}
+#endif
+
+//================================================================================================
+
+// Define bssnEM_class
 
 // It inherits some members and methods from the parent class bssn_class and modifies others.
 // The modified members and methods are defined below (and in the header bssnEM_class.h).
@@ -232,19 +335,21 @@ void bssnEM_class::Initialize()
   else
     GH->compose_cgh(nprocs);
 
-#ifdef WithShell
-  SH = new ShellPatch(0, ngfs, pname, Symmetry, myrank, ErrorMonitor);
-  SH->matchcheck(GH->PatL[0]);
+#ifdef WithShell
+  SH = new ShellPatch(0, ngfs, pname, Symmetry, myrank, ErrorMonitor);
+  SH->matchcheck(GH->PatL[0]);
   SH->compose_sh(nprocs);
   SH->setupcordtrans();
   SH->Dump_xyz(0, 0, 1);
   SH->setupintintstuff(nprocs, GH->PatL[0], Symmetry);
 
-  if (checkrun)
-    CheckPoint->readcheck_sh(SH, myrank);
-#endif
-
-  double h = GH->PatL[0]->data->blb->data->getdX(0);
+  if (checkrun)
+    CheckPoint->readcheck_sh(SH, myrank);
+#endif
+
+  Initialize_Level_Runtime();
+
+  double h = GH->PatL[0]->data->blb->data->getdX(0);
   for (int i = 1; i < dim; i++)
     h = Mymin(h, GH->PatL[0]->data->blb->data->getdX(i));
   dT = Courant * h;
@@ -258,8 +363,6 @@ void bssnEM_class::Initialize()
     PhysTime = StartTime;
     Setup_Black_Hole_position();
   }
-
-  setup_transfer_caches();
 }
 
 //================================================================================================
@@ -853,10 +956,11 @@ void bssnEM_class::Step(int lev, int YN)
                      cg->fgfs[gyy0->sgfn], cg->fgfs[gyz0->sgfn], cg->fgfs[gzz0->sgfn],
                      cg->fgfs[Axx0->sgfn], cg->fgfs[Axy0->sgfn], cg->fgfs[Axz0->sgfn], 
                      cg->fgfs[Ayy0->sgfn], cg->fgfs[Ayz0->sgfn], cg->fgfs[Azz0->sgfn]);
-#endif
-
-        if (
-            f_compute_rhs_empart(cg->shape, cg->X[0], cg->X[1], cg->X[2],
+#endif
+
+        bool used_gpu_substep = false;
+        if (
+            f_compute_rhs_empart(cg->shape, cg->X[0], cg->X[1], cg->X[2],
                                  cg->fgfs[phi0->sgfn],
                                  cg->fgfs[gxx0->sgfn], cg->fgfs[gxy0->sgfn], cg->fgfs[gxz0->sgfn], 
                                  cg->fgfs[gyy0->sgfn], cg->fgfs[gyz0->sgfn], cg->fgfs[gzz0->sgfn],
@@ -873,11 +977,20 @@ void bssnEM_class::Step(int lev, int YN)
                                  cg->fgfs[Kpsi_rhs->sgfn], cg->fgfs[Kphi_rhs->sgfn],
                                  cg->fgfs[rho->sgfn], 
                                  cg->fgfs[Sx->sgfn], cg->fgfs[Sy->sgfn], cg->fgfs[Sz->sgfn],
-                                 cg->fgfs[Sxx->sgfn], cg->fgfs[Sxy->sgfn], cg->fgfs[Sxz->sgfn], 
-                                 cg->fgfs[Syy->sgfn], cg->fgfs[Syz->sgfn], cg->fgfs[Szz->sgfn],
-                                 Symmetry, lev, ndeps) ||
-            f_compute_rhs_bssn(cg->shape, TRK4, cg->X[0], cg->X[1], cg->X[2],
-                               cg->fgfs[phi0->sgfn], cg->fgfs[trK0->sgfn],
+                                 cg->fgfs[Sxx->sgfn], cg->fgfs[Sxy->sgfn], cg->fgfs[Sxz->sgfn],
+                                 cg->fgfs[Syy->sgfn], cg->fgfs[Syz->sgfn], cg->fgfs[Szz->sgfn],
+                                 Symmetry, lev, ndeps) ||
+#if USE_CUDA_BSSN
+            ((used_gpu_substep =
+                  (run_bssn_em_cuda_substep(cg, StateList, SynchList_pre, Pp->data,
+                                            dT_lev, TRK4, iter_count, Symmetry, lev,
+                                            ndeps, pre, chitiny,
+                                            rho, Sx, Sy, Sz, Sxx, Sxy, Sxz, Syy, Syz, Szz) == 0))
+                 ? 0
+                 : 1) ||
+#endif
+            (!used_gpu_substep && f_compute_rhs_bssn(cg->shape, TRK4, cg->X[0], cg->X[1], cg->X[2],
+                               cg->fgfs[phi0->sgfn], cg->fgfs[trK0->sgfn],
                                cg->fgfs[gxx0->sgfn], cg->fgfs[gxy0->sgfn], cg->fgfs[gxz0->sgfn], 
                                cg->fgfs[gyy0->sgfn], cg->fgfs[gyz0->sgfn], cg->fgfs[gzz0->sgfn],
                                cg->fgfs[Axx0->sgfn], cg->fgfs[Axy0->sgfn], cg->fgfs[Axz0->sgfn], 
@@ -906,10 +1019,10 @@ void bssnEM_class::Step(int lev, int YN)
                                cg->fgfs[Gamzyy->sgfn], cg->fgfs[Gamzyz->sgfn], cg->fgfs[Gamzzz->sgfn],
                                cg->fgfs[Rxx->sgfn], cg->fgfs[Rxy->sgfn], cg->fgfs[Rxz->sgfn], 
                                cg->fgfs[Ryy->sgfn], cg->fgfs[Ryz->sgfn], cg->fgfs[Rzz->sgfn],
-                               cg->fgfs[Cons_Ham->sgfn],
-                               cg->fgfs[Cons_Px->sgfn], cg->fgfs[Cons_Py->sgfn], cg->fgfs[Cons_Pz->sgfn],
-                               cg->fgfs[Cons_Gx->sgfn], cg->fgfs[Cons_Gy->sgfn], cg->fgfs[Cons_Gz->sgfn],
-                               Symmetry, lev, ndeps, pre))
+                               cg->fgfs[Cons_Ham->sgfn],
+                               cg->fgfs[Cons_Px->sgfn], cg->fgfs[Cons_Py->sgfn], cg->fgfs[Cons_Pz->sgfn],
+                               cg->fgfs[Cons_Gx->sgfn], cg->fgfs[Cons_Gy->sgfn], cg->fgfs[Cons_Gz->sgfn],
+                               Symmetry, lev, ndeps, pre)))
         {
           cout << "find NaN in domain: (" 
                << cg->bbox[0] << ":" << cg->bbox[3] << "," 
@@ -919,11 +1032,15 @@ void bssnEM_class::Step(int lev, int YN)
         }
 
         // rk4 substep and boundary
-        {
-          MyList<var> *varl0 = StateList, *varl = SynchList_pre, *varlrhs = RHSList; 
-          // we do not check the correspondence here
-          
-          while (varl0)
+        {
+          MyList<var> *varl0 = StateList, *varl = SynchList_pre, *varlrhs = RHSList;
+          // we do not check the correspondence here
+#if USE_CUDA_BSSN
+          if (used_gpu_substep)
+            skip_bssn_cuda_prefix(varl0, varl, varlrhs);
+#endif
+
+          while (varl0)
           {
 #ifndef WithShell
             if (lev == 0) // sommerfeld indeed
@@ -1223,7 +1340,7 @@ void bssnEM_class::Step(int lev, int YN)
   }
 #endif
 
-  Parallel::Sync(GH->PatL[lev], SynchList_pre, Symmetry);
+  Parallel::Sync_cached(GH->PatL[lev], SynchList_pre, Symmetry, sync_cache_pre[lev]);
 
 #ifdef WithShell
   if (lev == 0)
@@ -1309,10 +1426,11 @@ void bssnEM_class::Step(int lev, int YN)
                          cg->fgfs[gyy->sgfn], cg->fgfs[gyz->sgfn], cg->fgfs[gzz->sgfn],
                          cg->fgfs[Axx->sgfn], cg->fgfs[Axy->sgfn], cg->fgfs[Axz->sgfn], 
                          cg->fgfs[Ayy->sgfn], cg->fgfs[Ayz->sgfn], cg->fgfs[Azz->sgfn]);
-#endif
-
-          if (
-              f_compute_rhs_empart(cg->shape, cg->X[0], cg->X[1], cg->X[2],
+#endif
+
+          bool used_gpu_substep = false;
+          if (
+              f_compute_rhs_empart(cg->shape, cg->X[0], cg->X[1], cg->X[2],
                                    cg->fgfs[phi->sgfn],
                                    cg->fgfs[gxx->sgfn], cg->fgfs[gxy->sgfn], cg->fgfs[gxz->sgfn], 
                                    cg->fgfs[gyy->sgfn], cg->fgfs[gyz->sgfn], cg->fgfs[gzz->sgfn],
@@ -1329,11 +1447,20 @@ void bssnEM_class::Step(int lev, int YN)
                                    cg->fgfs[Kpsi1->sgfn], cg->fgfs[Kphi1->sgfn],
                                    cg->fgfs[rho->sgfn], 
                                    cg->fgfs[Sx->sgfn], cg->fgfs[Sy->sgfn], cg->fgfs[Sz->sgfn],
-                                   cg->fgfs[Sxx->sgfn], cg->fgfs[Sxy->sgfn], cg->fgfs[Sxz->sgfn], 
-                                   cg->fgfs[Syy->sgfn], cg->fgfs[Syz->sgfn], cg->fgfs[Szz->sgfn],
-                                   Symmetry, lev, ndeps) ||
-              f_compute_rhs_bssn(cg->shape, TRK4, cg->X[0], cg->X[1], cg->X[2],
-                                 cg->fgfs[phi->sgfn], cg->fgfs[trK->sgfn],
+                                   cg->fgfs[Sxx->sgfn], cg->fgfs[Sxy->sgfn], cg->fgfs[Sxz->sgfn],
+                                   cg->fgfs[Syy->sgfn], cg->fgfs[Syz->sgfn], cg->fgfs[Szz->sgfn],
+                                   Symmetry, lev, ndeps) ||
+#if USE_CUDA_BSSN
+              ((used_gpu_substep =
+                    (run_bssn_em_cuda_substep(cg, SynchList_pre, SynchList_cor, Pp->data,
+                                              dT_lev, TRK4, iter_count, Symmetry, lev,
+                                              ndeps, cor, chitiny,
+                                              rho, Sx, Sy, Sz, Sxx, Sxy, Sxz, Syy, Syz, Szz) == 0))
+                   ? 0
+                   : 1) ||
+#endif
+              (!used_gpu_substep && f_compute_rhs_bssn(cg->shape, TRK4, cg->X[0], cg->X[1], cg->X[2],
+                                 cg->fgfs[phi->sgfn], cg->fgfs[trK->sgfn],
                                  cg->fgfs[gxx->sgfn], cg->fgfs[gxy->sgfn], cg->fgfs[gxz->sgfn], 
                                  cg->fgfs[gyy->sgfn], cg->fgfs[gyz->sgfn], cg->fgfs[gzz->sgfn],
                                  cg->fgfs[Axx->sgfn], cg->fgfs[Axy->sgfn], cg->fgfs[Axz->sgfn], 
@@ -1361,10 +1488,10 @@ void bssnEM_class::Step(int lev, int YN)
                                  cg->fgfs[Gamzyy->sgfn], cg->fgfs[Gamzyz->sgfn], cg->fgfs[Gamzzz->sgfn],
                                  cg->fgfs[Rxx->sgfn], cg->fgfs[Rxy->sgfn], cg->fgfs[Rxz->sgfn], 
                                  cg->fgfs[Ryy->sgfn], cg->fgfs[Ryz->sgfn], cg->fgfs[Rzz->sgfn],
-                                 cg->fgfs[Cons_Ham->sgfn],
-                                 cg->fgfs[Cons_Px->sgfn], cg->fgfs[Cons_Py->sgfn], cg->fgfs[Cons_Pz->sgfn],
-                                 cg->fgfs[Cons_Gx->sgfn], cg->fgfs[Cons_Gy->sgfn], cg->fgfs[Cons_Gz->sgfn],
-                                 Symmetry, lev, ndeps, cor))
+                                 cg->fgfs[Cons_Ham->sgfn],
+                                 cg->fgfs[Cons_Px->sgfn], cg->fgfs[Cons_Py->sgfn], cg->fgfs[Cons_Pz->sgfn],
+                                 cg->fgfs[Cons_Gx->sgfn], cg->fgfs[Cons_Gy->sgfn], cg->fgfs[Cons_Gz->sgfn],
+                                 Symmetry, lev, ndeps, cor)))
           {
             cout << "find NaN in domain: (" 
                  << cg->bbox[0] << ":" << cg->bbox[3] << "," 
@@ -1373,11 +1500,15 @@ void bssnEM_class::Step(int lev, int YN)
             ERROR = 1;
           }
           // rk4 substep and boundary
-          {
-            MyList<var> *varl0 = StateList, *varl = SynchList_pre, *varl1 = SynchList_cor, *varlrhs = RHSList; 
-            // we do not check the correspondence here
-            
-            while (varl0)
+          {
+            MyList<var> *varl0 = StateList, *varl = SynchList_pre, *varl1 = SynchList_cor, *varlrhs = RHSList;
+            // we do not check the correspondence here
+#if USE_CUDA_BSSN
+            if (used_gpu_substep)
+              skip_bssn_cuda_prefix(varl0, varl, varl1, varlrhs);
+#endif
+
+            while (varl0)
             {
 #ifndef WithShell
               if (lev == 0) // sommerfeld indeed
@@ -1685,7 +1816,7 @@ void bssnEM_class::Step(int lev, int YN)
     }
 #endif
 
-    Parallel::Sync(GH->PatL[lev], SynchList_cor, Symmetry);
+    Parallel::Sync_cached(GH->PatL[lev], SynchList_cor, Symmetry, sync_cache_cor[lev]);
 
 #ifdef WithShell
     if (lev == 0)

AMSS_NCKU_source/bssnEScalar_class.h

@@ -51,20 +51,24 @@ public:
      void Compute_Psi4(int lev);
      void Step(int lev, int YN);
      void AnalysisStuff_EScalar(int lev, double dT_lev);
-     void Interp_Constraint(bool infg);
+     void Interp_Constraint();
      void Constraint_Out(); 
 
-protected:
-     var *Sphio, *Spio;
-     var *Sphi0, *Spi0;
+protected:
+     var *Sphio, *Spio;
+     var *Sphi0, *Spi0;
      var *Sphi,  *Spi;
      var *Sphi1, *Spi1;
      var *Sphi_rhs, *Spi_rhs;
-
-     var *Cons_fR;
-
-     monitor *MaxScalar_Monitor;
-};
+
+     var *Cons_fR;
+
+     MyList<var> *BSSNStateList, *BSSNSynchList_pre, *BSSNSynchList_cor;
+     MyList<var> *ScalarSynchList_pre, *ScalarSynchList_cor;
+     Parallel::SyncCache *sync_cache_scalar_pre, *sync_cache_scalar_cor;
+
+     monitor *MaxScalar_Monitor;
+};
 
 #endif /* BSSNESCALAR_CLASS_H */
 

AMSS_NCKU_source/bssnEScalar_rhs.f90

@@ -3,11 +3,143 @@
 !! note that the potential for scalar field in F(R) gravity
 !!       is defined in the file Set_Rho_ADM.f90
 
-#include "macrodef.fh"
-
-! rhs for scalar and GR variables
-! here we consider vacuum spacetime only
-  function compute_rhs_bssn_escalar(ex, T,X, Y, Z,                             &
+#include "macrodef.fh"
+
+! scalar RHS and stress-energy only; BSSN RHS can be supplied by CUDA.
+  function compute_rhs_bssn_escalar_matter(ex, T, X, Y, Z,                    &
+               chi    ,   trK    ,                                            &
+               dxx    ,   gxy    ,   gxz    ,   dyy    ,   gyz    ,   dzz,    &
+               Axx    ,   Axy    ,   Axz    ,   Ayy    ,   Ayz    ,   Azz,    &
+               Gamx   ,  Gamy    ,  Gamz    ,                                 &
+               Lap    ,  betax   ,  betay   ,  betaz   ,                      &
+               dtSfx  ,  dtSfy   ,  dtSfz   ,                                 &
+               Sphi   ,   Spi    ,                                            &
+               Sphi_rhs ,   Spi_rhs ,                                         &
+               rho,Sx,Sy,Sz,Sxx,Sxy,Sxz,Syy,Syz,Szz,                          &
+               Symmetry,Lev,eps) result(gont)
+  implicit none
+
+  integer,intent(in ):: ex(1:3), Symmetry,Lev
+  real*8, intent(in ):: T
+  real*8, intent(in ):: X(1:ex(1)),Y(1:ex(2)),Z(1:ex(3))
+  real*8, dimension(ex(1),ex(2),ex(3)),intent(inout) :: chi,dxx,dyy,dzz
+  real*8, dimension(ex(1),ex(2),ex(3)),intent(in ) :: trK
+  real*8, dimension(ex(1),ex(2),ex(3)),intent(in ) :: gxy,gxz,gyz
+  real*8, dimension(ex(1),ex(2),ex(3)),intent(in ) :: Axx,Axy,Axz,Ayy,Ayz,Azz
+  real*8, dimension(ex(1),ex(2),ex(3)),intent(in ) :: Gamx,Gamy,Gamz
+  real*8, dimension(ex(1),ex(2),ex(3)),intent(inout) :: Lap, betax, betay, betaz
+  real*8, dimension(ex(1),ex(2),ex(3)),intent(in ) :: dtSfx,  dtSfy,  dtSfz
+  real*8, dimension(ex(1),ex(2),ex(3)),intent(in ) :: Sphi,Spi
+  real*8, dimension(ex(1),ex(2),ex(3)),intent(out) :: Sphi_rhs,Spi_rhs
+  real*8, dimension(ex(1),ex(2),ex(3)),intent(inout) :: rho,Sx,Sy,Sz
+  real*8, dimension(ex(1),ex(2),ex(3)),intent(inout) :: Sxx,Sxy,Sxz,Syy,Syz,Szz
+  real*8,intent(in) :: eps
+  integer::gont
+
+  real*8, dimension(ex(1),ex(2),ex(3)) :: gxx,gyy,gzz
+  real*8, dimension(ex(1),ex(2),ex(3)) :: chix,chiy,chiz
+  real*8, dimension(ex(1),ex(2),ex(3)) :: Lapx,Lapy,Lapz
+  real*8, dimension(ex(1),ex(2),ex(3)) :: Kx,Ky,Kz,S
+  real*8, dimension(ex(1),ex(2),ex(3)) :: f,fxx,fxy,fxz,fyy,fyz,fzz
+  real*8, dimension(ex(1),ex(2),ex(3)) :: alpn1,chin1
+  real*8, dimension(ex(1),ex(2),ex(3)) :: gupxx,gupxy,gupxz
+  real*8, dimension(ex(1),ex(2),ex(3)) :: gupyy,gupyz,gupzz
+
+  real*8            :: dX
+  real*8, parameter :: ZEO=0.d0, ONE = 1.D0, TWO = 2.D0, HALF = 0.5D0
+  real*8, parameter :: SYM = 1.D0
+
+  dX = sum(chi)+sum(trK)+sum(dxx)+sum(gxy)+sum(gxz)+sum(dyy)+sum(gyz)+sum(dzz) &
+      +sum(Gamx)+sum(Gamy)+sum(Gamz)                                           &
+      +sum(Lap)+sum(Sphi)+sum(Spi)
+  if(dX.ne.dX) then
+     if(sum(chi).ne.sum(chi))write(*,*)"bssn_escalar_matter: find NaN in chi"
+     if(sum(trK).ne.sum(trK))write(*,*)"bssn_escalar_matter: find NaN in trk"
+     if(sum(dxx).ne.sum(dxx))write(*,*)"bssn_escalar_matter: find NaN in dxx"
+     if(sum(gxy).ne.sum(gxy))write(*,*)"bssn_escalar_matter: find NaN in gxy"
+     if(sum(gxz).ne.sum(gxz))write(*,*)"bssn_escalar_matter: find NaN in gxz"
+     if(sum(dyy).ne.sum(dyy))write(*,*)"bssn_escalar_matter: find NaN in dyy"
+     if(sum(gyz).ne.sum(gyz))write(*,*)"bssn_escalar_matter: find NaN in gyz"
+     if(sum(dzz).ne.sum(dzz))write(*,*)"bssn_escalar_matter: find NaN in dzz"
+     if(sum(Gamx).ne.sum(Gamx))write(*,*)"bssn_escalar_matter: find NaN in Gamx"
+     if(sum(Gamy).ne.sum(Gamy))write(*,*)"bssn_escalar_matter: find NaN in Gamy"
+     if(sum(Gamz).ne.sum(Gamz))write(*,*)"bssn_escalar_matter: find NaN in Gamz"
+     if(sum(Lap).ne.sum(Lap))write(*,*)"bssn_escalar_matter: find NaN in Lap"
+     if(sum(Sphi).ne.sum(Sphi))write(*,*)"bssn_escalar_matter: find NaN in Sphi"
+     if(sum(Spi).ne.sum(Spi))write(*,*)"bssn_escalar_matter: find NaN in Spi"
+     gont = 1
+     return
+  endif
+
+  alpn1 = Lap + ONE
+  chin1 = chi + ONE
+  gxx = dxx + ONE
+  gyy = dyy + ONE
+  gzz = dzz + ONE
+
+  call fderivs(ex,chi,chix,chiy,chiz,X,Y,Z,SYM,SYM,SYM,Symmetry,Lev)
+  call fderivs(ex,Lap,Lapx,Lapy,Lapz,X,Y,Z,SYM,SYM,SYM,Symmetry,Lev)
+
+  gupzz =  gxx * gyy * gzz + gxy * gyz * gxz + gxz * gxy * gyz - &
+           gxz * gyy * gxz - gxy * gxy * gzz - gxx * gyz * gyz
+  gupxx =   ( gyy * gzz - gyz * gyz ) / gupzz
+  gupxy = - ( gxy * gzz - gyz * gxz ) / gupzz
+  gupxz =   ( gxy * gyz - gyy * gxz ) / gupzz
+  gupyy =   ( gxx * gzz - gxz * gxz ) / gupzz
+  gupyz = - ( gxx * gyz - gxy * gxz ) / gupzz
+  gupzz =   ( gxx * gyy - gxy * gxy ) / gupzz
+
+#if 1
+  Sphi_rhs = alpn1 * Spi
+  call fderivs(ex,Sphi,Kx,Ky,Kz,X,Y,Z,SYM,SYM,SYM,Symmetry,Lev)
+  call fdderivs(ex,Sphi,fxx,fxy,fxz,fyy,fyz,fzz,X,Y,Z,SYM,SYM,SYM,Symmetry,Lev)
+  Spi_rhs =   gupxx * fxx + gupyy * fyy + gupzz * fzz +           &
+             ( gupxy * fxy + gupxz * fxz + gupyz * fyz ) * TWO -  &
+             ((Gamx+(gupxx*chix+gupxy*chiy+gupxz*chiz)/TWO/chin1)*Kx  &
+            + (Gamy+(gupxy*chix+gupyy*chiy+gupyz*chiz)/TWO/chin1)*Ky  &
+            + (Gamz+(gupxz*chix+gupyz*chiy+gupzz*chiz)/TWO/chin1)*Kz)
+  Spi_rhs = Spi_rhs*alpn1 +                                      &
+            (gupxx*Lapx*Kx + gupxy*Lapx*Ky + gupxz*Lapx*Kz       &
+            +gupxy*Lapy*Kx + gupyy*Lapy*Ky + gupyz*Lapy*Kz       &
+            +gupxz*Lapz*Kx + gupyz*Lapz*Ky + gupzz*Lapz*Kz)
+
+  call frpotential(ex,Sphi,f,S)
+  Spi_rhs = Spi_rhs*chin1 + alpn1*(trK*Spi - S)
+  rho = chin1*((gupxx * Kx * Kx + gupyy * Ky * Ky + gupzz * Kz * Kz)/TWO + &
+                gupxy * Kx * Ky + gupxz * Kx * Kz + gupyz * Ky * Kz )      &
+        + Spi*Spi/TWO+f
+  Sx = -Spi*Kx
+  Sy = -Spi*Ky
+  Sz = -Spi*Kz
+  f = (rho - Spi*Spi)/chin1
+  Sxx = Kx*Kx-f*gxx
+  Sxy = Kx*Ky-f*gxy
+  Sxz = Kx*Kz-f*gxz
+  Syy = Ky*Ky-f*gyy
+  Syz = Ky*Kz-f*gyz
+  Szz = Kz*Kz-f*gzz
+#else
+  Sphi_rhs = ZEO
+  Spi_rhs = ZEO
+  rho = ZEO
+  Sx = ZEO
+  Sy = ZEO
+  Sz = ZEO
+  Sxx = ZEO
+  Sxy = ZEO
+  Sxz = ZEO
+  Syy = ZEO
+  Syz = ZEO
+  Szz = ZEO
+#endif
+
+  gont = 0
+  return
+end function compute_rhs_bssn_escalar_matter
+
+! rhs for scalar and GR variables
+! here we consider vacuum spacetime only
+  function compute_rhs_bssn_escalar(ex, T,X, Y, Z,                             &
                chi    ,   trK    ,                                             &
                dxx    ,   gxy    ,   gxz    ,   dyy    ,   gyz    ,   dzz,     &
                Axx    ,   Axy    ,   Axz    ,   Ayy    ,   Ayz    ,   Azz,     &

AMSS_NCKU_source/bssn_class.h

@@ -31,19 +31,11 @@ using namespace std;
 #include "surface_integral.h"
 #include "checkpoint.h"
 
-extern void setpbh(int iBHN, double **iPBH, double *iMass, int rBHN);
-
-#ifndef BSSN_USE_TRANSFER_CACHE
-#define BSSN_USE_TRANSFER_CACHE 1
-#endif
-
-#ifndef BSSN_USE_ESCALAR_C_KERNEL
-#define BSSN_USE_ESCALAR_C_KERNEL 1
-#endif
-
-class bssn_class
-{
-public:
+extern void setpbh(int iBHN, double **iPBH, double *iMass, int rBHN);
+
+class bssn_class
+{
+public:
        int ngfs;
        int nprocs, myrank;
        cgh *GH;
@@ -56,6 +48,7 @@ public:
        double StartTime, TotalTime;
        double AnasTime, DumpTime, d2DumpTime, CheckTime;
        double LastAnas, LastConsOut;
+       bool cuda_level0_constraint_cache_valid;
        int *ConstraintRefreshLevels;
        double Courant;
        double numepss, numepsb, numepsh;
@@ -175,35 +168,27 @@ public:
        void Setup_KerrSchild();
        void Enforce_algcon(int lev, int fg);
 
-       void testRestrict();
-       void testOutBd();
-       
-       bool check_Stdin_Abort(); 
-       bool use_transfer_cache() const;
-       void setup_transfer_caches();
-       void invalidate_transfer_caches();
-       void destroy_transfer_caches();
-       void sync_predictor_start(int lev, MyList<var> *VarList, Parallel::AsyncSyncState &async_state);
-       void sync_predictor_finish(int lev, Parallel::AsyncSyncState &async_state, MyList<var> *VarList);
-       void sync_corrector_start(int lev, MyList<var> *VarList, Parallel::AsyncSyncState &async_state);
-       void sync_corrector_finish(int lev, Parallel::AsyncSyncState &async_state, MyList<var> *VarList);
-       void sync_evolution(int lev, MyList<var> *VarList, Parallel::SyncCache *cache_array = 0);
-       void restrict_evolution(int lev, MyList<var> *src_var_list, MyList<var> *dst_var_list);
-       void outbdlow2hi_evolution(int lev, MyList<var> *src_var_list, MyList<var> *dst_var_list);
-
-       virtual void Setup_Initial_Data_Cao();
-       virtual void Setup_Initial_Data_Lousto();
-       virtual void Initialize();
+       void testRestrict();
+       void testOutBd();
+       
+       bool check_Stdin_Abort(); 
+
+       virtual void Setup_Initial_Data_Cao();
+       virtual void Setup_Initial_Data_Lousto();
+       virtual void Initialize();
        virtual void Read_Ansorg();
        virtual void Read_Pablo() {};
-       virtual void Compute_Psi4(int lev);
-       virtual void Step(int lev, int YN);
-       virtual void Interp_Constraint(bool infg);
-       virtual void Constraint_Out();
-       virtual void Compute_Constraint();
-
-#ifdef With_AHF
-protected:
+       virtual void Compute_Psi4(int lev);
+       virtual void Step(int lev, int YN);
+       virtual void Interp_Constraint(bool infg);
+       virtual void Constraint_Out();
+       virtual void Compute_Constraint();
+
+protected:
+       void Initialize_Level_Runtime();
+
+#ifdef With_AHF
+protected:
        MyList<var> *AHList, *AHDList, *GaugeList;
        int AHfindevery;
        double AHdumptime;

AMSS_NCKU_source/bssn_em_rhs_c.C

@@ -1,323 +0,0 @@
-#include "macrodef.h"
-#include "bssn_rhs.h"
-#include "share_func.h"
-#include "tool.h"
-#include <cstddef>
-
-/*
- * C 版 BSSN-EM RHS kernel — replaces empart.f90 + bssn_rhs.f90 for BSSN+Maxwell.
- *
- * Computes:
- *   1. All metric and EM field derivatives
- *   2. Physical metric, Christoffel-like terms
- *   3. EM field RHS (E, B, Kpsi, Kphi)
- *   4. Stress-energy tensor (rho, Si, Sij)
- *   5. Calls f_compute_rhs_bssn (C BSSN RHS) with stress-energy
- *   6. Advection + KO dissipation for EM fields
- *   7. NaN check
- */
-int f_compute_rhs_bssn_em_c(int *ex, double &T,
-                            double *X, double *Y, double *Z,
-                            double *chi, double *trK,
-                            double *dxx, double *gxy, double *gxz, double *dyy, double *gyz, double *dzz,
-                            double *Axx, double *Axy, double *Axz, double *Ayy, double *Ayz, double *Azz,
-                            double *Gamx, double *Gamy, double *Gamz,
-                            double *Lap, double *betax, double *betay, double *betaz,
-                            double *dtSfx, double *dtSfy, double *dtSfz,
-                            double *Ex,  double *Ey,  double *Ez,
-                            double *Bx,  double *By,  double *Bz,
-                            double *Kpsi, double *Kphi,
-                            double *Jx, double *Jy, double *Jz, double *qchar,
-                            double *chi_rhs, double *trK_rhs,
-                            double *gxx_rhs, double *gxy_rhs, double *gxz_rhs,
-                            double *gyy_rhs, double *gyz_rhs, double *gzz_rhs,
-                            double *Axx_rhs, double *Axy_rhs, double *Axz_rhs,
-                            double *Ayy_rhs, double *Ayz_rhs, double *Azz_rhs,
-                            double *Gamx_rhs, double *Gamy_rhs, double *Gamz_rhs,
-                            double *Lap_rhs, double *betax_rhs, double *betay_rhs, double *betaz_rhs,
-                            double *dtSfx_rhs, double *dtSfy_rhs, double *dtSfz_rhs,
-                            double *Ex_rhs,  double *Ey_rhs,  double *Ez_rhs,
-                            double *Bx_rhs,  double *By_rhs,  double *Bz_rhs,
-                            double *Kpsi_rhs, double *Kphi_rhs,
-                            double *rho, double *Sx, double *Sy, double *Sz,
-                            double *Sxx, double *Sxy, double *Sxz,
-                            double *Syy, double *Syz, double *Szz,
-                            double *Gamxxx, double *Gamxxy, double *Gamxxz,
-                            double *Gamxyy, double *Gamxyz, double *Gamxzz,
-                            double *Gamyxx, double *Gamyxy, double *Gamyxz,
-                            double *Gamyyy, double *Gamyyz, double *Gamyzz,
-                            double *Gamzxx, double *Gamzxy, double *Gamzxz,
-                            double *Gamzyy, double *Gamzyz, double *Gamzzz,
-                            double *Rxx, double *Rxy, double *Rxz,
-                            double *Ryy, double *Ryz, double *Rzz,
-                            double *ham_Res, double *movx_Res, double *movy_Res, double *movz_Res,
-                            double *Gmx_Res, double *Gmy_Res, double *Gmz_Res,
-                            int &Symmetry, int &Lev, double &eps, int &co)
-{
-    (void)T;
-    int gont = 0;
-    const int nx = ex[0], ny = ex[1], nz = ex[2];
-    const int all = nx * ny * nz;
-    const size_t n = (size_t)all;
-
-    const double ZEO = 0.0, ONE = 1.0, TWO = 2.0, FOUR = 4.0, EIT = 8.0;
-    const double HALF = 0.5, THR = 3.0, F3o2 = 1.5, PI = 3.14159265358979323846;
-    const double SYM = 1.0, ANTI = -1.0;
-    const double kappa = 1.0;
-    const double SSS[3]={SYM,SYM,SYM},   AAS[3]={ANTI,ANTI,SYM};
-    const double ASA[3]={ANTI,SYM,ANTI}, SAA[3]={SYM,ANTI,ANTI};
-    const double ASS[3]={ANTI,SYM,SYM},   SAS[3]={SYM,ANTI,SYM};
-    const double SSA[3]={SYM,SYM,ANTI};
-
-    /* ---- allocate temporary arrays ---- */
-    double *chix = (double*)malloc(n*sizeof(double));
-    double *chiy = (double*)malloc(n*sizeof(double));
-    double *chiz = (double*)malloc(n*sizeof(double));
-    double *Exx=(double*)malloc(n*sizeof(double)),*Exy=(double*)malloc(n*sizeof(double)),*Exz=(double*)malloc(n*sizeof(double));
-    double *Eyx=(double*)malloc(n*sizeof(double)),*Eyy=(double*)malloc(n*sizeof(double)),*Eyz=(double*)malloc(n*sizeof(double));
-    double *Ezx=(double*)malloc(n*sizeof(double)),*Ezy=(double*)malloc(n*sizeof(double)),*Ezz=(double*)malloc(n*sizeof(double));
-    double *Bxx=(double*)malloc(n*sizeof(double)),*Bxy=(double*)malloc(n*sizeof(double)),*Bxz=(double*)malloc(n*sizeof(double));
-    double *Byx=(double*)malloc(n*sizeof(double)),*Byy=(double*)malloc(n*sizeof(double)),*Byz=(double*)malloc(n*sizeof(double));
-    double *Bzx=(double*)malloc(n*sizeof(double)),*Bzy=(double*)malloc(n*sizeof(double)),*Bzz=(double*)malloc(n*sizeof(double));
-    double *Kpsix=(double*)malloc(n*sizeof(double)),*Kpsiy=(double*)malloc(n*sizeof(double)),*Kpsiz=(double*)malloc(n*sizeof(double));
-    double *Kphix=(double*)malloc(n*sizeof(double)),*Kphiy=(double*)malloc(n*sizeof(double)),*Kphiz=(double*)malloc(n*sizeof(double));
-    double *Lapx=(double*)malloc(n*sizeof(double)),*Lapy=(double*)malloc(n*sizeof(double)),*Lapz=(double*)malloc(n*sizeof(double));
-    double *betaxx=(double*)malloc(n*sizeof(double)),*betaxy=(double*)malloc(n*sizeof(double)),*betaxz=(double*)malloc(n*sizeof(double));
-    double *betayx=(double*)malloc(n*sizeof(double)),*betayy=(double*)malloc(n*sizeof(double)),*betayz=(double*)malloc(n*sizeof(double));
-    double *betazx=(double*)malloc(n*sizeof(double)),*betazy=(double*)malloc(n*sizeof(double)),*betazz=(double*)malloc(n*sizeof(double));
-    double *gxxx=(double*)malloc(n*sizeof(double)),*gxxy=(double*)malloc(n*sizeof(double)),*gxxz=(double*)malloc(n*sizeof(double));
-    double *gxyx=(double*)malloc(n*sizeof(double)),*gxyy=(double*)malloc(n*sizeof(double)),*gxyz=(double*)malloc(n*sizeof(double));
-    double *gxzx=(double*)malloc(n*sizeof(double)),*gxzy=(double*)malloc(n*sizeof(double)),*gxzz=(double*)malloc(n*sizeof(double));
-    double *gyyx=(double*)malloc(n*sizeof(double)),*gyyy=(double*)malloc(n*sizeof(double)),*gyyz=(double*)malloc(n*sizeof(double));
-    double *gyzx=(double*)malloc(n*sizeof(double)),*gyzy=(double*)malloc(n*sizeof(double)),*gyzz=(double*)malloc(n*sizeof(double));
-    double *gzzx=(double*)malloc(n*sizeof(double)),*gzzy=(double*)malloc(n*sizeof(double)),*gzzz=(double*)malloc(n*sizeof(double));
-    double *gupxx=(double*)malloc(n*sizeof(double)),*gupxy=(double*)malloc(n*sizeof(double)),*gupxz=(double*)malloc(n*sizeof(double));
-    double *gupyy=(double*)malloc(n*sizeof(double)),*gupyz=(double*)malloc(n*sizeof(double)),*gupzz=(double*)malloc(n*sizeof(double));
-
-    if (!chix||!chiy||!chiz||!Exx||!Exy||!Exz||!Eyx||!Eyy||!Eyz||!Ezx||!Ezy||!Ezz||
-        !Bxx||!Bxy||!Bxz||!Byx||!Byy||!Byz||!Bzx||!Bzy||!Bzz||
-        !Kpsix||!Kpsiy||!Kpsiz||!Kphix||!Kphiy||!Kphiz||
-        !Lapx||!Lapy||!Lapz||
-        !betaxx||!betaxy||!betaxz||!betayx||!betayy||!betayz||!betazx||!betazy||!betazz||
-        !gxxx||!gxxy||!gxxz||!gxyx||!gxyy||!gxyz||!gxzx||!gxzy||!gxzz||
-        !gyyx||!gyyy||!gyyz||!gyzx||!gyzy||!gyzz||!gzzx||!gzzy||!gzzz||
-        !gupxx||!gupxy||!gupxz||!gupyy||!gupyz||!gupzz) {
-        gont = 1;
-    }
-
-    /* ==== 1. Compute all derivatives ==== */
-    if (!gont) {
-
-    /* metric derivatives */
-    fderivs(ex, Lap, Lapx, Lapy, Lapz, X, Y, Z, SYM, SYM, SYM, Symmetry, Lev);
-    fderivs(ex, betax, betaxx, betaxy, betaxz, X, Y, Z, ANTI, SYM, SYM, Symmetry, Lev);
-    fderivs(ex, betay, betayx, betayy, betayz, X, Y, Z, SYM, ANTI, SYM, Symmetry, Lev);
-    fderivs(ex, betaz, betazx, betazy, betazz, X, Y, Z, SYM, SYM, ANTI, Symmetry, Lev);
-    fderivs(ex, chi, chix, chiy, chiz, X, Y, Z, SYM, SYM, SYM, Symmetry, Lev);
-    fderivs(ex, dxx, gxxx, gxxy, gxxz, X, Y, Z, SYM, SYM, SYM, Symmetry, Lev);
-    fderivs(ex, gxy, gxyx, gxyy, gxyz, X, Y, Z, ANTI, ANTI, SYM, Symmetry, Lev);
-    fderivs(ex, gxz, gxzx, gxzy, gxzz, X, Y, Z, ANTI, SYM, ANTI, Symmetry, Lev);
-    fderivs(ex, dyy, gyyx, gyyy, gyyz, X, Y, Z, SYM, SYM, SYM, Symmetry, Lev);
-    fderivs(ex, gyz, gyzx, gyzy, gyzz, X, Y, Z, SYM, ANTI, ANTI, Symmetry, Lev);
-    fderivs(ex, dzz, gzzx, gzzy, gzzz, X, Y, Z, SYM, SYM, SYM, Symmetry, Lev);
-
-    /* EM field derivatives */
-    fderivs(ex, Kpsi, Kpsix, Kpsiy, Kpsiz, X, Y, Z, SYM, SYM, SYM, Symmetry, Lev);
-    fderivs(ex, Kphi, Kphix, Kphiy, Kphiz, X, Y, Z, SYM, SYM, SYM, Symmetry, Lev);
-    fderivs(ex, Ex, Exx, Exy, Exz, X, Y, Z, ANTI, SYM, SYM, Symmetry, Lev);
-    fderivs(ex, Ey, Eyx, Eyy, Eyz, X, Y, Z, SYM, ANTI, SYM, Symmetry, Lev);
-    fderivs(ex, Ez, Ezx, Ezy, Ezz, X, Y, Z, SYM, SYM, ANTI, Symmetry, Lev);
-    fderivs(ex, Bx, Bxx, Bxy, Bxz, X, Y, Z, SYM, ANTI, ANTI, Symmetry, Lev);
-    fderivs(ex, By, Byx, Byy, Byz, X, Y, Z, ANTI, SYM, ANTI, Symmetry, Lev);
-    fderivs(ex, Bz, Bzx, Bzy, Bzz, X, Y, Z, ANTI, ANTI, SYM, Symmetry, Lev);
-
-    /* ==== 2. Compute EM RHS and stress-energy ==== */
-    const double F1o4PI = ONE / (FOUR * PI);
-    for (size_t i = 0; i < n; ++i) {
-        const double alpn1 = Lap[i] + ONE;
-        const double chin1 = chi[i] + ONE;
-        const double chi3o2 = sqrt(chin1) * chin1;  // chi^{3/2}
-        const double ichi = ONE / chin1;
-
-        /* physical metric */
-        const double pgxx = (dxx[i] + ONE) * ichi;
-        const double pgyy = (dyy[i] + ONE) * ichi;
-        const double pgzz = (dzz[i] + ONE) * ichi;
-        const double pgxy = gxy[i] * ichi;
-        const double pgxz = gxz[i] * ichi;
-        const double pgyz = gyz[i] * ichi;
-
-        /* inverse physical metric */
-        const double det = pgxx * pgyy * pgzz + pgxy * pgyz * pgxz + pgxz * pgxy * pgyz
-                         - pgxz * pgyy * pgxz - pgxy * pgxy * pgzz - pgxx * pgyz * pgyz;
-        const double idet = ONE / det;
-        const double upxx = (pgyy * pgzz - pgyz * pgyz) * idet;
-        const double upxy = -(pgxy * pgzz - pgyz * pgxz) * idet;
-        const double upxz = (pgxy * pgyz - pgyy * pgxz) * idet;
-        const double upyy = (pgxx * pgzz - pgxz * pgxz) * idet;
-        const double upyz = -(pgxx * pgyz - pgxy * pgxz) * idet;
-        const double upzz = (pgxx * pgyy - pgxy * pgxy) * idet;
-        gupxx[i]=upxx; gupxy[i]=upxy; gupxz[i]=upxz;
-        gupyy[i]=upyy; gupyz[i]=upyz; gupzz[i]=upzz;
-
-        /* E-field RHS */
-        /* curl(B) part: epsilon^{ijk} ∂_j (alpha * B_k) in coordinate basis */
-        /* Using lower-index B fields: B_i_lower = pg_{ij} * B^j */
-        const double BxL = pgxx*Bx[i] + pgxy*By[i] + pgxz*Bz[i];
-        const double ByL = pgxy*Bx[i] + pgyy*By[i] + pgyz*Bz[i];
-        const double BzL = pgxz*Bx[i] + pgyz*By[i] + pgzz*Bz[i];
-
-        /* Physical metric derivatives (chain rule from conformal) */
-        const double pgxx_x = (gxxx[i] - pgxx*chix[i]) * ichi;
-        /* const double pgxx_y = (gxxy[i] - pgxx*chiy[i]) * ichi; */
-        const double pgxy_x = (gxyx[i] - pgxy*chix[i]) * ichi;
-        const double pgxy_y = (gxyy[i] - pgxy*chiy[i]) * ichi;
-        const double pgxz_x = (gxzx[i] - pgxz*chix[i]) * ichi;
-        const double pgxz_z = (gxzz[i] - pgxz*chiz[i]) * ichi;
-        const double pgyy_y = (gyyy[i] - pgyy*chiy[i]) * ichi;
-        const double pgyz_y = (gyzy[i] - pgyz*chiy[i]) * ichi;
-        const double pgyz_z = (gyzz[i] - pgyz*chiz[i]) * ichi;
-        const double pgzz_z = (gzzz[i] - pgzz*chiz[i]) * ichi;
-
-        /* Curl_x(B) = ∂_y (alpha*BzL) - ∂_z (alpha*ByL) */
-        const double aBx = alpn1*BxL, aBy = alpn1*ByL, aBz = alpn1*BzL;
-        const double curlBx = (aBz*Lapy[i] + alpn1*(pgxz*Bxy[i]+pgyz*Byy[i]+pgzz*Bzy[i]) + alpn1*(Bx[i]*gxzy[i]+By[i]*gyzy[i]+Bz[i]*gzzy[i]))
-                            - (aBy*Lapz[i] + alpn1*(pgxy*Bxz[i]+pgyy*Byz[i]+pgyz*Bzz[i]) + alpn1*(Bx[i]*gxyz[i]+By[i]*gyyz[i]+Bz[i]*gyzz[i]));
-        double curlBy = (aBx*Lapz[i] + alpn1*(pgxx*Bxz[i]+pgxy*Byz[i]+pgxz*Bzz[i]) + alpn1*(Bx[i]*gxxz[i]+By[i]*gxyz[i]+Bz[i]*gxzz[i]))
-                      - (aBz*Lapx[i] + alpn1*(pgxz*Bxx[i]+pgyz*Byx[i]+pgzz*Bzx[i]) + alpn1*(Bx[i]*gxzx[i]+By[i]*gyzx[i]+Bz[i]*gzzx[i]));
-        double curlBz = (aBy*Lapx[i] + alpn1*(pgxy*Bxx[i]+pgyy*Byx[i]+pgyz*Bzx[i]) + alpn1*(Bx[i]*gxyx[i]+By[i]*gyyx[i]+Bz[i]*gyzx[i]))
-                      - (aBx*Lapy[i] + alpn1*(pgxx*Bxy[i]+pgxy*Byy[i]+pgxz*Bzy[i]) + alpn1*(Bx[i]*gxxy[i]+By[i]*gxyy[i]+Bz[i]*gxzy[i]));
-
-        /* Advection part: -beta^j * ∂_j E^i */
-        const double advEx = Ex[i]*betaxx[i] + Ey[i]*betaxy[i] + Ez[i]*betaxz[i];
-        const double advEy = Ex[i]*betayx[i] + Ey[i]*betayy[i] + Ez[i]*betayz[i];
-        const double advEz = Ex[i]*betazx[i] + Ey[i]*betazy[i] + Ez[i]*betazz[i];
-
-        /* grad(Kpsi) contracted with inverse metric */
-        const double gupKx = upxx*Kpsix[i] + upxy*Kpsiy[i] + upxz*Kpsiz[i];
-        const double gupKy = upxy*Kpsix[i] + upyy*Kpsiy[i] + upyz*Kpsiz[i];
-        const double gupKz = upxz*Kpsix[i] + upyz*Kpsiy[i] + upzz*Kpsiz[i];
-
-        Ex_rhs[i] = alpn1*trK[i]*Ex[i] - advEx - FOUR*PI*alpn1*Jx[i] - alpn1*gupKx + chi3o2*curlBx;
-        Ey_rhs[i] = alpn1*trK[i]*Ey[i] - advEy - FOUR*PI*alpn1*Jy[i] - alpn1*gupKy + chi3o2*curlBy;
-        Ez_rhs[i] = alpn1*trK[i]*Ez[i] - advEz - FOUR*PI*alpn1*Jz[i] - alpn1*gupKz + chi3o2*curlBz;
-
-        /* B-field RHS: similar but with -chi^{3/2} * curl(E) and grad(Kphi) */
-        const double ExL = pgxx*Ex[i] + pgxy*Ey[i] + pgxz*Ez[i];
-        const double EyL = pgxy*Ex[i] + pgyy*Ey[i] + pgyz*Ez[i];
-        const double EzL = pgxz*Ex[i] + pgyz*Ey[i] + pgzz*Ez[i];
-
-        const double aEx = alpn1*ExL, aEy = alpn1*EyL, aEz = alpn1*EzL;
-        const double curlEx = (aEz*Lapy[i] + alpn1*(pgxz*Exy[i]+pgyz*Eyy[i]+pgzz*Ezy[i]) + alpn1*(Ex[i]*gxzy[i]+Ey[i]*gyzy[i]+Ez[i]*gzzy[i]))
-                            - (aEy*Lapz[i] + alpn1*(pgxy*Exz[i]+pgyy*Eyz[i]+pgyz*Ezz[i]) + alpn1*(Ex[i]*gxyz[i]+Ey[i]*gyyz[i]+Ez[i]*gyzz[i]));
-        double curlEy = (aEx*Lapz[i] + alpn1*(pgxx*Exz[i]+pgxy*Eyz[i]+pgxz*Ezz[i]) + alpn1*(Ex[i]*gxxz[i]+Ey[i]*gxyz[i]+Ez[i]*gxzz[i]))
-                      - (aEz*Lapx[i] + alpn1*(pgxz*Exx[i]+pgyz*Eyx[i]+pgzz*Ezx[i]) + alpn1*(Ex[i]*gxzx[i]+Ey[i]*gyzx[i]+Ez[i]*gzzx[i]));
-        double curlEz = (aEy*Lapx[i] + alpn1*(pgxy*Exx[i]+pgyy*Eyx[i]+pgyz*Ezx[i]) + alpn1*(Ex[i]*gxyx[i]+Ey[i]*gyyx[i]+Ez[i]*gyzx[i]))
-                      - (aEx*Lapy[i] + alpn1*(pgxx*Exy[i]+pgxy*Eyy[i]+pgxz*Ezy[i]) + alpn1*(Ex[i]*gxxy[i]+Ey[i]*gxyy[i]+Ez[i]*gxzy[i]));
-
-        const double advBx = Bx[i]*betaxx[i] + By[i]*betaxy[i] + Bz[i]*betaxz[i];
-        const double advBy = Bx[i]*betayx[i] + By[i]*betayy[i] + Bz[i]*betayz[i];
-        const double advBz = Bx[i]*betazx[i] + By[i]*betazy[i] + Bz[i]*betazz[i];
-
-        const double gupKphix = upxx*Kphix[i] + upxy*Kphiy[i] + upxz*Kphiz[i];
-        const double gupKphiy = upxy*Kphix[i] + upyy*Kphiy[i] + upyz*Kphiz[i];
-        const double gupKphiz = upxz*Kphix[i] + upyz*Kphiy[i] + upzz*Kphiz[i];
-
-        Bx_rhs[i] = alpn1*trK[i]*Bx[i] - advBx - alpn1*gupKphix - chi3o2*curlEx;
-        By_rhs[i] = alpn1*trK[i]*By[i] - advBy - alpn1*gupKphiy - chi3o2*curlEy;
-        Bz_rhs[i] = alpn1*trK[i]*Bz[i] - advBz - alpn1*gupKphiz - chi3o2*curlEz;
-
-        /* Scalar potential RHS */
-        const double divE = Exx[i] + Eyy[i] + Ezz[i];
-        const double divB = Bxx[i] + Byy[i] + Bzz[i];
-        const double chiCont = F3o2 * ichi * (chix[i]*Ex[i] + chiy[i]*Ey[i] + chiz[i]*Ez[i]);
-        Kpsi_rhs[i] = FOUR*PI*alpn1*qchar[i] - alpn1*kappa*Kpsi[i] - alpn1*(divE - chiCont);
-        Kphi_rhs[i] = -alpn1*kappa*Kphi[i] - alpn1*(divB - F3o2*ichi*(chix[i]*Bx[i] + chiy[i]*By[i] + chiz[i]*Bz[i]));
-
-        /* Stress-energy tensor */
-        const double E2 = pgxx*Ex[i]*Ex[i] + pgyy*Ey[i]*Ey[i] + pgzz*Ez[i]*Ez[i]
-                        + TWO*(pgxy*Ex[i]*Ey[i] + pgxz*Ex[i]*Ez[i] + pgyz*Ey[i]*Ez[i]);
-        const double B2 = pgxx*Bx[i]*Bx[i] + pgyy*By[i]*By[i] + pgzz*Bz[i]*Bz[i]
-                        + TWO*(pgxy*Bx[i]*By[i] + pgxz*Bx[i]*Bz[i] + pgyz*By[i]*Bz[i]);
-        rho[i] = (E2 + B2) / (EIT * PI);
-        const double ichi3o2 = ONE / chi3o2;
-        Sx[i] = (Ey[i]*Bz[i] - Ez[i]*By[i]) * F1o4PI * ichi3o2;
-        Sy[i] = (Ez[i]*Bx[i] - Ex[i]*Bz[i]) * F1o4PI * ichi3o2;
-        Sz[i] = (Ex[i]*By[i] - Ey[i]*Bx[i]) * F1o4PI * ichi3o2;
-        const double lExi = pgxx*Ex[i] + pgxy*Ey[i] + pgxz*Ez[i];
-        const double lEyi = pgxy*Ex[i] + pgyy*Ey[i] + pgyz*Ez[i];
-        const double lEzi = pgxz*Ex[i] + pgyz*Ey[i] + pgzz*Ez[i];
-        const double lBxi = pgxx*Bx[i] + pgxy*By[i] + pgxz*Bz[i];
-        const double lByi = pgxy*Bx[i] + pgyy*By[i] + pgyz*Bz[i];
-        const double lBzi = pgxz*Bx[i] + pgyz*By[i] + pgzz*Bz[i];
-        Sxx[i] = rho[i]*pgxx - (lExi*lExi + lBxi*lBxi) * F1o4PI;
-        Sxy[i] = rho[i]*pgxy - (lExi*lEyi + lBxi*lByi) * F1o4PI;
-        Sxz[i] = rho[i]*pgxz - (lExi*lEzi + lBxi*lBzi) * F1o4PI;
-        Syy[i] = rho[i]*pgyy - (lEyi*lEyi + lByi*lByi) * F1o4PI;
-        Syz[i] = rho[i]*pgyz - (lEyi*lEzi + lByi*lBzi) * F1o4PI;
-        Szz[i] = rho[i]*pgzz - (lEzi*lEzi + lBzi*lBzi) * F1o4PI;
-    }
-
-    /* ==== 3. Call BSSN RHS with EM stress-energy ==== */
-    gont = f_compute_rhs_bssn(ex, T, X, Y, Z,
-        chi, trK, dxx, gxy, gxz, dyy, gyz, dzz,
-        Axx, Axy, Axz, Ayy, Ayz, Azz,
-        Gamx, Gamy, Gamz, Lap, betax, betay, betaz, dtSfx, dtSfy, dtSfz,
-        chi_rhs, trK_rhs,
-        gxx_rhs, gxy_rhs, gxz_rhs, gyy_rhs, gyz_rhs, gzz_rhs,
-        Axx_rhs, Axy_rhs, Axz_rhs, Ayy_rhs, Ayz_rhs, Azz_rhs,
-        Gamx_rhs, Gamy_rhs, Gamz_rhs, Lap_rhs, betax_rhs, betay_rhs, betaz_rhs,
-        dtSfx_rhs, dtSfy_rhs, dtSfz_rhs,
-        rho, Sx, Sy, Sz, Sxx, Sxy, Sxz, Syy, Syz, Szz,
-        Gamxxx, Gamxxy, Gamxxz, Gamxyy, Gamxyz, Gamxzz,
-        Gamyxx, Gamyxy, Gamyxz, Gamyyy, Gamyyz, Gamyzz,
-        Gamzxx, Gamzxy, Gamzxz, Gamzyy, Gamzyz, Gamzzz,
-        Rxx, Rxy, Rxz, Ryy, Ryz, Rzz,
-        ham_Res, movx_Res, movy_Res, movz_Res,
-        Gmx_Res, Gmy_Res, Gmz_Res,
-        Symmetry, Lev, eps, co);
-    if (!gont) {
-
-    /* ==== 4. Advection terms for EM fields ==== */
-    lopsided(ex, X, Y, Z, Kpsi, Kpsi_rhs, betax, betay, betaz, Symmetry, SSS);
-    lopsided(ex, X, Y, Z, Kphi, Kphi_rhs, betax, betay, betaz, Symmetry, SSS);
-    lopsided(ex, X, Y, Z, Ex,   Ex_rhs,   betax, betay, betaz, Symmetry, ASS);
-    lopsided(ex, X, Y, Z, Ey,   Ey_rhs,   betax, betay, betaz, Symmetry, SAS);
-    lopsided(ex, X, Y, Z, Ez,   Ez_rhs,   betax, betay, betaz, Symmetry, SSA);
-    lopsided(ex, X, Y, Z, Bx,   Bx_rhs,   betax, betay, betaz, Symmetry, SAA);
-    lopsided(ex, X, Y, Z, By,   By_rhs,   betax, betay, betaz, Symmetry, ASA);
-    lopsided(ex, X, Y, Z, Bz,   Bz_rhs,   betax, betay, betaz, Symmetry, AAS);
-
-    /* ==== 5. KO dissipation for EM fields ==== */
-    if (eps > ZEO) {
-        kodis(ex, X, Y, Z, Kpsi, Kpsi_rhs, SSS, Symmetry, eps);
-        kodis(ex, X, Y, Z, Kphi, Kphi_rhs, SSS, Symmetry, eps);
-        kodis(ex, X, Y, Z, Ex,   Ex_rhs,   ASS, Symmetry, eps);
-        kodis(ex, X, Y, Z, Ey,   Ey_rhs,   SAS, Symmetry, eps);
-        kodis(ex, X, Y, Z, Ez,   Ez_rhs,   SSA, Symmetry, eps);
-        kodis(ex, X, Y, Z, Bx,   Bx_rhs,   SAA, Symmetry, eps);
-        kodis(ex, X, Y, Z, By,   By_rhs,   ASA, Symmetry, eps);
-        kodis(ex, X, Y, Z, Bz,   Bz_rhs,   AAS, Symmetry, eps);
-    }
-
-    /* ==== 6. NaN check ==== */
-        for (int i = 0; i < all; ++i) {
-            if (!isfinite(Ex_rhs[i]+Ey_rhs[i]+Ez_rhs[i]+Bx_rhs[i]+By_rhs[i]+Bz_rhs[i]+Kpsi_rhs[i]+Kphi_rhs[i])) {
-                gont = 1; break;
-            }
-        }
-        } /* inner if (!gont) */
-    } /* outer if (!gont) */
-
-    free(chix);free(chiy);free(chiz);
-    free(Exx);free(Exy);free(Exz);free(Eyx);free(Eyy);free(Eyz);free(Ezx);free(Ezy);free(Ezz);
-    free(Bxx);free(Bxy);free(Bxz);free(Byx);free(Byy);free(Byz);free(Bzx);free(Bzy);free(Bzz);
-    free(Kpsix);free(Kpsiy);free(Kpsiz);
-    free(Kphix);free(Kphiy);free(Kphiz);
-    free(Lapx);free(Lapy);free(Lapz);
-    free(betaxx);free(betaxy);free(betaxz);free(betayx);free(betayy);free(betayz);free(betazx);free(betazy);free(betazz);
-    free(gxxx);free(gxxy);free(gxxz);free(gxyx);free(gxyy);free(gxyz);free(gxzx);free(gxzy);free(gxzz);
-    free(gyyx);free(gyyy);free(gyyz);free(gyzx);free(gyzy);free(gyzz);free(gzzx);free(gzzy);free(gzzz);
-    free(gupxx);free(gupxy);free(gupxz);free(gupyy);free(gupyz);free(gupzz);
-    return gont;
-}

AMSS_NCKU_source/bssn_escalar_rhs_c.C

@@ -1,169 +0,0 @@
-#include "macrodef.h"
-#include "bssn_rhs.h"
-#include "share_func.h"
-#include "tool.h"
-#include <vector>
-
-namespace
-{
-    // Reuse the temporary workspace across block calls to avoid repeated heap churn
-    // in the EScalar wrapper. MPI ranks execute this path sequentially, so a single
-    // process-local buffer is sufficient here.
-    std::vector<double> g_escalar_tmp_store;
-}
-
-#ifdef fortran1
-#define f_frpotential frpotential
-#endif
-#ifdef fortran2
-#define f_frpotential FRPOTENTIAL
-#endif
-#ifdef fortran3
-#define f_frpotential frpotential_
-#endif
-
-extern "C"
-{
-    void f_frpotential(int *, double *, double *, double *);
-}
-
-int f_compute_rhs_bssn_escalar_c(int *ex, double &T,
-                                 double *X, double *Y, double *Z,
-                                 double *chi, double *trK,
-                                 double *dxx, double *gxy, double *gxz, double *dyy, double *gyz, double *dzz,
-                                 double *Axx, double *Axy, double *Axz, double *Ayy, double *Ayz, double *Azz,
-                                 double *Gamx, double *Gamy, double *Gamz,
-                                 double *Lap, double *betax, double *betay, double *betaz,
-                                 double *dtSfx, double *dtSfy, double *dtSfz,
-                                 double *Sphi, double *Spi,
-                                 double *chi_rhs, double *trK_rhs,
-                                 double *gxx_rhs, double *gxy_rhs, double *gxz_rhs, double *gyy_rhs, double *gyz_rhs, double *gzz_rhs,
-                                 double *Axx_rhs, double *Axy_rhs, double *Axz_rhs, double *Ayy_rhs, double *Ayz_rhs, double *Azz_rhs,
-                                 double *Gamx_rhs, double *Gamy_rhs, double *Gamz_rhs,
-                                 double *Lap_rhs, double *betax_rhs, double *betay_rhs, double *betaz_rhs,
-                                 double *dtSfx_rhs, double *dtSfy_rhs, double *dtSfz_rhs,
-                                 double *Sphi_rhs, double *Spi_rhs,
-                                 double *rho, double *Sx, double *Sy, double *Sz,
-                                 double *Sxx, double *Sxy, double *Sxz, double *Syy, double *Syz, double *Szz,
-                                 double *Gamxxx, double *Gamxxy, double *Gamxxz, double *Gamxyy, double *Gamxyz, double *Gamxzz,
-                                 double *Gamyxx, double *Gamyxy, double *Gamyxz, double *Gamyyy, double *Gamyyz, double *Gamyzz,
-                                 double *Gamzxx, double *Gamzxy, double *Gamzxz, double *Gamzyy, double *Gamzyz, double *Gamzzz,
-                                 double *Rxx, double *Rxy, double *Rxz, double *Ryy, double *Ryz, double *Rzz,
-                                 double *ham_Res, double *movx_Res, double *movy_Res, double *movz_Res,
-                                 double *Gmx_Res, double *Gmy_Res, double *Gmz_Res,
-                                 int &Symmetry, int &Lev, double &eps, int &co)
-{
-    const int nx = ex[0], ny = ex[1], nz = ex[2];
-    const int all = nx * ny * nz;
-
-    const size_t workspace_size = size_t(all) * 17;
-    if (g_escalar_tmp_store.size() < workspace_size)
-        g_escalar_tmp_store.resize(workspace_size);
-
-    double *tmp_ptr = g_escalar_tmp_store.data();
-    auto alloc_tmp = [&](int n = 1) -> double *
-    {
-        double *ptr = tmp_ptr;
-        tmp_ptr += size_t(all) * n;
-        return ptr;
-    };
-
-    double *chix = alloc_tmp(), *chiy = alloc_tmp(), *chiz = alloc_tmp();
-    double *Kx = alloc_tmp(), *Ky = alloc_tmp(), *Kz = alloc_tmp();
-    double *fxx = alloc_tmp(), *fxy = alloc_tmp(), *fxz = alloc_tmp();
-    double *fyy = alloc_tmp(), *fyz = alloc_tmp(), *fzz = alloc_tmp();
-    double *Lapx = alloc_tmp(), *Lapy = alloc_tmp(), *Lapz = alloc_tmp();
-    double *V = alloc_tmp(), *dVdSphi = alloc_tmp();
-
-    const double ZEO = 0.0, ONE = 1.0, TWO = 2.0, HALF = 0.5;
-    const double SSS[3] = {1.0, 1.0, 1.0};
-
-    fderivs(ex, chi, chix, chiy, chiz, X, Y, Z, 1.0, 1.0, 1.0, Symmetry, Lev);
-    fderivs(ex, Lap, Lapx, Lapy, Lapz, X, Y, Z, 1.0, 1.0, 1.0, Symmetry, Lev);
-    fderivs(ex, Sphi, Kx, Ky, Kz, X, Y, Z, 1.0, 1.0, 1.0, Symmetry, Lev);
-    fdderivs(ex, Sphi, fxx, fxy, fxz, fyy, fyz, fzz, X, Y, Z, 1.0, 1.0, 1.0, Symmetry, Lev);
-
-    f_frpotential(ex, Sphi, V, dVdSphi);
-
-    for (int i = 0; i < all; ++i)
-    {
-        const double alpn1 = Lap[i] + ONE;
-        const double chin1 = chi[i] + ONE;
-        const double gxx = dxx[i] + ONE;
-        const double gyy = dyy[i] + ONE;
-        const double gzz = dzz[i] + ONE;
-        const double det = gxx * gyy * gzz + gxy[i] * gyz[i] * gxz[i] + gxz[i] * gxy[i] * gyz[i]
-                         - gxz[i] * gyy * gxz[i] - gxy[i] * gxy[i] * gzz - gxx * gyz[i] * gyz[i];
-        const double gupxx = (gyy * gzz - gyz[i] * gyz[i]) / det;
-        const double gupxy = -(gxy[i] * gzz - gyz[i] * gxz[i]) / det;
-        const double gupxz = (gxy[i] * gyz[i] - gyy * gxz[i]) / det;
-        const double gupyy = (gxx * gzz - gxz[i] * gxz[i]) / det;
-        const double gupyz = -(gxx * gyz[i] - gxy[i] * gxz[i]) / det;
-        const double gupzz = (gxx * gyy - gxy[i] * gxy[i]) / det;
-
-        Sphi_rhs[i] = alpn1 * Spi[i];
-
-        Spi_rhs[i] = gupxx * fxx[i] + gupyy * fyy[i] + gupzz * fzz[i]
-                   + TWO * (gupxy * fxy[i] + gupxz * fxz[i] + gupyz * fyz[i])
-                   - ((Gamx[i] + (gupxx * chix[i] + gupxy * chiy[i] + gupxz * chiz[i]) / TWO / chin1) * Kx[i]
-                   +  (Gamy[i] + (gupxy * chix[i] + gupyy * chiy[i] + gupyz * chiz[i]) / TWO / chin1) * Ky[i]
-                   +  (Gamz[i] + (gupxz * chix[i] + gupyz * chiy[i] + gupzz * chiz[i]) / TWO / chin1) * Kz[i]);
-
-        Spi_rhs[i] = Spi_rhs[i] * alpn1
-                   + gupxx * Lapx[i] * Kx[i] + gupxy * Lapx[i] * Ky[i] + gupxz * Lapx[i] * Kz[i]
-                   + gupxy * Lapy[i] * Kx[i] + gupyy * Lapy[i] * Ky[i] + gupyz * Lapy[i] * Kz[i]
-                   + gupxz * Lapz[i] * Kx[i] + gupyz * Lapz[i] * Ky[i] + gupzz * Lapz[i] * Kz[i];
-
-        Spi_rhs[i] = Spi_rhs[i] * chin1 + alpn1 * (trK[i] * Spi[i] - dVdSphi[i]);
-
-        rho[i] = chin1 * ((gupxx * Kx[i] * Kx[i] + gupyy * Ky[i] * Ky[i] + gupzz * Kz[i] * Kz[i]) * HALF
-               + gupxy * Kx[i] * Ky[i] + gupxz * Kx[i] * Kz[i] + gupyz * Ky[i] * Kz[i])
-               + Spi[i] * Spi[i] * HALF + V[i];
-        Sx[i] = -Spi[i] * Kx[i];
-        Sy[i] = -Spi[i] * Ky[i];
-        Sz[i] = -Spi[i] * Kz[i];
-
-        const double pressure = (rho[i] - Spi[i] * Spi[i]) / chin1;
-        Sxx[i] = Kx[i] * Kx[i] - pressure * gxx;
-        Sxy[i] = Kx[i] * Ky[i] - pressure * gxy[i];
-        Sxz[i] = Kx[i] * Kz[i] - pressure * gxz[i];
-        Syy[i] = Ky[i] * Ky[i] - pressure * gyy;
-        Syz[i] = Ky[i] * Kz[i] - pressure * gyz[i];
-        Szz[i] = Kz[i] * Kz[i] - pressure * gzz;
-    }
-
-    if (f_compute_rhs_bssn(ex, T, X, Y, Z,
-                           chi, trK,
-                           dxx, gxy, gxz, dyy, gyz, dzz,
-                           Axx, Axy, Axz, Ayy, Ayz, Azz,
-                           Gamx, Gamy, Gamz,
-                           Lap, betax, betay, betaz,
-                           dtSfx, dtSfy, dtSfz,
-                           chi_rhs, trK_rhs,
-                           gxx_rhs, gxy_rhs, gxz_rhs, gyy_rhs, gyz_rhs, gzz_rhs,
-                           Axx_rhs, Axy_rhs, Axz_rhs, Ayy_rhs, Ayz_rhs, Azz_rhs,
-                           Gamx_rhs, Gamy_rhs, Gamz_rhs,
-                           Lap_rhs, betax_rhs, betay_rhs, betaz_rhs,
-                           dtSfx_rhs, dtSfy_rhs, dtSfz_rhs,
-                           rho, Sx, Sy, Sz,
-                           Sxx, Sxy, Sxz, Syy, Syz, Szz,
-                           Gamxxx, Gamxxy, Gamxxz, Gamxyy, Gamxyz, Gamxzz,
-                           Gamyxx, Gamyxy, Gamyxz, Gamyyy, Gamyyz, Gamyzz,
-                           Gamzxx, Gamzxy, Gamzxz, Gamzyy, Gamzyz, Gamzzz,
-                           Rxx, Rxy, Rxz, Ryy, Ryz, Rzz,
-                           ham_Res, movx_Res, movy_Res, movz_Res,
-                           Gmx_Res, Gmy_Res, Gmz_Res,
-                           Symmetry, Lev, eps, co))
-        return 1;
-
-    lopsided_kodis(ex, X, Y, Z, Sphi, Sphi_rhs, betax, betay, betaz, Symmetry, SSS, eps);
-    lopsided_kodis(ex, X, Y, Z, Spi, Spi_rhs, betax, betay, betaz, Symmetry, SSS, eps);
-
-    for (int i = 0; i < all; ++i)
-    {
-        if (Sphi_rhs[i] != Sphi_rhs[i] || Spi_rhs[i] != Spi_rhs[i] || rho[i] != rho[i])
-            return 1;
-    }
-
-    return 0;
-}

AMSS_NCKU_source/bssn_gpu.h

@@ -1,73 +0,0 @@
-
-#ifndef BSSN_GPU_H_
-#define BSSN_GPU_H_
-#include "bssn_macro.h"
-#include "macrodef.fh"
-
-#define DEVICE_ID 0
-// #define DEVICE_ID_BY_MPI_RANK
-#define GRID_DIM 256
-#define BLOCK_DIM 128
-
-#define _FH2_(i, j, k) fh[(i) + (j) * _1D_SIZE[2] + (k) * _2D_SIZE[2]]
-#define _FH3_(i, j, k) fh[(i) + (j) * _1D_SIZE[3] + (k) * _2D_SIZE[3]]
-#define pow2(x) ((x) * (x))
-#define TimeBetween(a, b) ((b.tv_sec - a.tv_sec) + (b.tv_usec - a.tv_usec) / 1000000.0f)
-#define M_ metac.
-#define Mh_ meta->
-#define Ms_ metassc.
-#define Msh_ metass->
-
-// #define TIMING
-
-#define RHS_SS_PARA int calledby, int mpi_rank, int *ex, double &T, double *crho, double *sigma, double *R, double *X, double *Y, double *Z, double *drhodx, double *drhody, double *drhodz, double *dsigmadx, double *dsigmady, double *dsigmadz, double *dRdx, double *dRdy, double *dRdz, double *drhodxx, double *drhodxy, double *drhodxz, double *drhodyy, double *drhodyz, double *drhodzz, double *dsigmadxx, double *dsigmadxy, double *dsigmadxz, double *dsigmadyy, double *dsigmadyz, double *dsigmadzz, double *dRdxx, double *dRdxy, double *dRdxz, double *dRdyy, double *dRdyz, double *dRdzz, double *chi, double *trK, double *dxx, double *gxy, double *gxz, double *dyy, double *gyz, double *dzz, double *Axx, double *Axy, double *Axz, double *Ayy, double *Ayz, double *Azz, double *Gamx, double *Gamy, double *Gamz, double *Lap, double *betax, double *betay, double *betaz, double *dtSfx, double *dtSfy, double *dtSfz, double *chi_rhs, double *trK_rhs, double *gxx_rhs, double *gxy_rhs, double *gxz_rhs, double *gyy_rhs, double *gyz_rhs, double *gzz_rhs, double *Axx_rhs, double *Axy_rhs, double *Axz_rhs, double *Ayy_rhs, double *Ayz_rhs, double *Azz_rhs, double *Gamx_rhs, double *Gamy_rhs, double *Gamz_rhs, double *Lap_rhs, double *betax_rhs, double *betay_rhs, double *betaz_rhs, double *dtSfx_rhs, double *dtSfy_rhs, double *dtSfz_rhs, double *rho, double *Sx, double *Sy, double *Sz, double *Sxx, double *Sxy, double *Sxz, double *Syy, double *Syz, double *Szz, double *Gamxxx, double *Gamxxy, double *Gamxxz, double *Gamxyy, double *Gamxyz, double *Gamxzz, double *Gamyxx, double *Gamyxy, double *Gamyxz, double *Gamyyy, double *Gamyyz, double *Gamyzz, double *Gamzxx, double *Gamzxy, double *Gamzxz, double *Gamzyy, double *Gamzyz, double *Gamzzz, double *Rxx, double *Rxy, double *Rxz, double *Ryy, double *Ryz, double *Rzz, double *ham_Res, double *movx_Res, double *movy_Res, double *movz_Res, double *Gmx_Res, double *Gmy_Res, double *Gmz_Res, int &Symmetry, int &Lev, double &eps, int &sst, int &co
-
-/**  main function */
-int gpu_rhs(int calledby, int mpi_rank, int *ex, double &T,
-            double *X, double *Y, double *Z,
-
-            double *chi, double *trK,
-
-            double *dxx, double *gxy, double *gxz, double *dyy, double *gyz, double *dzz,
-
-            double *Axx, double *Axy, double *Axz, double *Ayy, double *Ayz, double *Azz,
-
-            double *Gamx, double *Gamy, double *Gamz,
-
-            double *Lap, double *betax, double *betay, double *betaz,
-
-            double *dtSfx, double *dtSfy, double *dtSfz,
-
-            double *chi_rhs, double *trK_rhs,
-
-            double *gxx_rhs, double *gxy_rhs, double *gxz_rhs, double *gyy_rhs, double *gyz_rhs, double *gzz_rhs,
-
-            double *Axx_rhs, double *Axy_rhs, double *Axz_rhs, double *Ayy_rhs, double *Ayz_rhs, double *Azz_rhs,
-
-            double *Gamx_rhs, double *Gamy_rhs, double *Gamz_rhs,
-
-            double *Lap_rhs, double *betax_rhs, double *betay_rhs, double *betaz_rhs,
-
-            double *dtSfx_rhs, double *dtSfy_rhs, double *dtSfz_rhs,
-
-            double *rho, double *Sx, double *Sy, double *Sz, double *Sxx,
-            double *Sxy, double *Sxz, double *Syy, double *Syz, double *Szz,
-
-            double *Gamxxx, double *Gamxxy, double *Gamxxz, double *Gamxyy, double *Gamxyz, double *Gamxzz,
-
-            double *Gamyxx, double *Gamyxy, double *Gamyxz, double *Gamyyy, double *Gamyyz, double *Gamyzz,
-
-            double *Gamzxx, double *Gamzxy, double *Gamzxz, double *Gamzyy, double *Gamzyz, double *Gamzzz,
-
-            double *Rxx, double *Rxy, double *Rxz, double *Ryy, double *Ryz, double *Rzz,
-
-            double *ham_Res, double *movx_Res, double *movy_Res, double *movz_Res,
-            double *Gmx_Res, double *Gmy_Res, double *Gmz_Res,
-            int &Symmetry, int &Lev, double &eps, int &co);
-
-int gpu_rhs_ss(RHS_SS_PARA);
-
-/** Init GPU side data in GPUMeta. */
-// void init_fluid_meta_gpu(GPUMeta *gpu_meta);
-
-#endif

AMSS_NCKU_source/bssn_gpu_class.h

@@ -1,210 +0,0 @@
-
-#ifndef BSSN_GPU_CLASS_H
-#define BSSN_GPU_CLASS_H
-
-#ifdef newc
-#include <iostream>
-#include <iomanip>
-#include <fstream>
-#include <cstdlib>
-#include <string>
-#include <cmath>
-using namespace std;
-#else
-#include <iostream.h>
-#include <iomanip.h>
-#include <fstream.h>
-#include <stdlib.h>
-#include <string.h>
-#include <math.h>
-#endif
-
-#include <mpi.h>
-
-#include "macrodef.h"
-#include "cgh.h"
-#include "ShellPatch.h"
-#include "misc.h"
-#include "var.h"
-#include "MyList.h"
-#include "monitor.h"
-#include "surface_integral.h"
-#include "checkpoint.h"
-
-// added by yangquan
-#include "bssn_macro.h"
-
-extern void setpbh(int iBHN, double **iPBH, double *iMass, int rBHN);
-
-class bssn_class
-{
-public:
-       // added by yangquan
-       //----------------------
-       int gpu_num_mynode;
-       int cpu_core_num_mynode;
-       int mpi_process_num_mynode;
-       int my_sequence_mynode;
-       int mynode_id;
-       int use_gpu;
-
-       virtual void Step_GPU(int lev, int YN);
-       virtual void Get_runtime_envirment();
-       // virtual void Step_OPENMP(int lev,int YN);
-       //----------------------
-
-       int ngfs;
-       int nprocs, myrank;
-       cgh *GH;
-       ShellPatch *SH;
-       double PhysTime;
-
-       int checkrun;
-       char checkfilename[50];
-       int Steps;
-       double StartTime, TotalTime;
-       double AnasTime, DumpTime, d2DumpTime, CheckTime;
-       double LastAnas, LastConsOut;
-       double Courant;
-       double numepss, numepsb, numepsh;
-       int Symmetry;
-       int maxl, decn;
-       double maxrex, drex;
-       int trfls, a_lev;
-
-       double dT;
-       double chitiny;
-
-       double **Porg0, **Porgbr, **Porg, **Porg1, **Porg_rhs;
-       int BH_num, BH_num_input;
-       double *Mass, *Pmom, *Spin;
-       double ADMMass;
-
-       var *phio, *trKo;
-       var *gxxo, *gxyo, *gxzo, *gyyo, *gyzo, *gzzo;
-       var *Axxo, *Axyo, *Axzo, *Ayyo, *Ayzo, *Azzo;
-       var *Gmxo, *Gmyo, *Gmzo;
-       var *Lapo, *Sfxo, *Sfyo, *Sfzo;
-       var *dtSfxo, *dtSfyo, *dtSfzo;
-
-       var *phi0, *trK0;
-       var *gxx0, *gxy0, *gxz0, *gyy0, *gyz0, *gzz0;
-       var *Axx0, *Axy0, *Axz0, *Ayy0, *Ayz0, *Azz0;
-       var *Gmx0, *Gmy0, *Gmz0;
-       var *Lap0, *Sfx0, *Sfy0, *Sfz0;
-       var *dtSfx0, *dtSfy0, *dtSfz0;
-
-       var *phi, *trK;
-       var *gxx, *gxy, *gxz, *gyy, *gyz, *gzz;
-       var *Axx, *Axy, *Axz, *Ayy, *Ayz, *Azz;
-       var *Gmx, *Gmy, *Gmz;
-       var *Lap, *Sfx, *Sfy, *Sfz;
-       var *dtSfx, *dtSfy, *dtSfz;
-
-       var *phi1, *trK1;
-       var *gxx1, *gxy1, *gxz1, *gyy1, *gyz1, *gzz1;
-       var *Axx1, *Axy1, *Axz1, *Ayy1, *Ayz1, *Azz1;
-       var *Gmx1, *Gmy1, *Gmz1;
-       var *Lap1, *Sfx1, *Sfy1, *Sfz1;
-       var *dtSfx1, *dtSfy1, *dtSfz1;
-
-       var *phi_rhs, *trK_rhs;
-       var *gxx_rhs, *gxy_rhs, *gxz_rhs, *gyy_rhs, *gyz_rhs, *gzz_rhs;
-       var *Axx_rhs, *Axy_rhs, *Axz_rhs, *Ayy_rhs, *Ayz_rhs, *Azz_rhs;
-       var *Gmx_rhs, *Gmy_rhs, *Gmz_rhs;
-       var *Lap_rhs, *Sfx_rhs, *Sfy_rhs, *Sfz_rhs;
-       var *dtSfx_rhs, *dtSfy_rhs, *dtSfz_rhs;
-
-       var *rho, *Sx, *Sy, *Sz, *Sxx, *Sxy, *Sxz, *Syy, *Syz, *Szz;
-
-       var *Gamxxx, *Gamxxy, *Gamxxz, *Gamxyy, *Gamxyz, *Gamxzz;
-       var *Gamyxx, *Gamyxy, *Gamyxz, *Gamyyy, *Gamyyz, *Gamyzz;
-       var *Gamzxx, *Gamzxy, *Gamzxz, *Gamzyy, *Gamzyz, *Gamzzz;
-
-       var *Rxx, *Rxy, *Rxz, *Ryy, *Ryz, *Rzz;
-
-       var *Rpsi4, *Ipsi4;
-       var *t1Rpsi4, *t1Ipsi4, *t2Rpsi4, *t2Ipsi4;
-
-       var *Cons_Ham, *Cons_Px, *Cons_Py, *Cons_Pz, *Cons_Gx, *Cons_Gy, *Cons_Gz;
-
-#ifdef Point_Psi4
-       var *phix, *phiy, *phiz;
-       var *trKx, *trKy, *trKz;
-       var *Axxx, *Axxy, *Axxz;
-       var *Axyx, *Axyy, *Axyz;
-       var *Axzx, *Axzy, *Axzz;
-       var *Ayyx, *Ayyy, *Ayyz;
-       var *Ayzx, *Ayzy, *Ayzz;
-       var *Azzx, *Azzy, *Azzz;
-#endif
-       // FIXME: uc = StateList, up = OldStateList, upp = SynchList_cor; so never touch these three data
-       MyList<var> *StateList, *SynchList_pre, *SynchList_cor, *RHSList;
-       MyList<var> *OldStateList, *DumpList;
-       MyList<var> *ConstraintList;
-
-       monitor *ErrorMonitor, *Psi4Monitor, *BHMonitor, *MAPMonitor;
-       monitor *ConVMonitor;
-       surface_integral *Waveshell;
-       checkpoint *CheckPoint;
-
-public:
-       bssn_class(double Couranti, double StartTimei, double TotalTimei, double DumpTimei, double d2DumpTimei, double CheckTimei, double AnasTimei,
-                  int Symmetryi, int checkruni, char *checkfilenamei, double numepssi, double numepsbi, double numepshi,
-                  int a_levi, int maxli, int decni, double maxrexi, double drexi);
-       ~bssn_class();
-
-       void Evolve(int Steps);
-       void RecursiveStep(int lev);
-#if (PSTR == 1)
-       void ParallelStep();
-       void SHStep();
-#endif
-       void RestrictProlong(int lev, int YN, bool BB, MyList<var> *SL, MyList<var> *OL, MyList<var> *corL);
-       void RestrictProlong_aux(int lev, int YN, bool BB, MyList<var> *SL, MyList<var> *OL, MyList<var> *corL);
-       void RestrictProlong(int lev, int YN, bool BB);
-       void ProlongRestrict(int lev, int YN, bool BB);
-       void Setup_Black_Hole_position();
-       void compute_Porg_rhs(double **BH_PS, double **BH_RHS, var *forx, var *fory, var *forz, int lev);
-       bool read_Pablo_file(int *ext, double *datain, char *filename);
-       void write_Pablo_file(int *ext, double xmin, double xmax, double ymin, double ymax, double zmin, double zmax,
-                             char *filename);
-       void AnalysisStuff(int lev, double dT_lev);
-       void Setup_KerrSchild();
-       void Enforce_algcon(int lev, int fg);
-
-       void testRestrict();
-       void testOutBd();
-
-       virtual void Setup_Initial_Data_Lousto();
-       virtual void Setup_Initial_Data_Cao();
-       virtual void Initialize();
-       virtual void Read_Ansorg();
-       virtual void Read_Pablo() {};
-       virtual void Compute_Psi4(int lev);
-       virtual void Step(int lev, int YN);
-       virtual void Interp_Constraint(bool infg);
-       virtual void Constraint_Out();
-       virtual void Compute_Constraint();
-
-#ifdef With_AHF
-protected:
-       MyList<var> *AHList, *AHDList, *GaugeList;
-       int AHfindevery;
-       double AHdumptime;
-       int *lastahdumpid, HN_num; // number of possible horizons
-       int *findeveryl;
-       double *xc, *yc, *zc, *xr, *yr, *zr;
-       bool *trigger;
-       double *dTT;
-       int *dumpid;
-
-public:
-       void AH_Prepare_derivatives();
-       bool AH_Interp_Points(MyList<var> *VarList,
-                             int NN, double **XX,
-                             double *Shellf, int Symmetryi);
-       void AH_Step_Find(int lev, double dT_lev);
-#endif
-};
-#endif /* BSSN_GPU_CLASS_H */

AMSS_NCKU_source/bssn_rhs.h

@@ -5,8 +5,9 @@
 #ifdef fortran1
 #define f_compute_rhs_bssn compute_rhs_bssn
 #define f_compute_rhs_bssn_ss compute_rhs_bssn_ss
-#define f_compute_rhs_bssn_escalar compute_rhs_bssn_escalar
-#define f_compute_rhs_bssn_escalar_ss compute_rhs_bssn_escalar_ss
+#define f_compute_rhs_bssn_escalar compute_rhs_bssn_escalar
+#define f_compute_rhs_bssn_escalar_matter compute_rhs_bssn_escalar_matter
+#define f_compute_rhs_bssn_escalar_ss compute_rhs_bssn_escalar_ss
 #define f_compute_rhs_Z4c compute_rhs_z4c
 #define f_compute_rhs_Z4cnot compute_rhs_z4cnot
 #define f_compute_rhs_Z4c_ss compute_rhs_z4c_ss
@@ -15,39 +16,41 @@
 #ifdef fortran2
 #define f_compute_rhs_bssn COMPUTE_RHS_BSSN
 #define f_compute_rhs_bssn_ss COMPUTE_RHS_BSSN_SS
-#define f_compute_rhs_bssn_escalar COMPUTE_RHS_BSSN_ESCALAR
-#define f_compute_rhs_bssn_escalar_ss COMPUTE_RHS_BSSN_ESCALAR_SS
+#define f_compute_rhs_bssn_escalar COMPUTE_RHS_BSSN_ESCALAR
+#define f_compute_rhs_bssn_escalar_matter COMPUTE_RHS_BSSN_ESCALAR_MATTER
+#define f_compute_rhs_bssn_escalar_ss COMPUTE_RHS_BSSN_ESCALAR_SS
 #define f_compute_rhs_Z4c COMPUTE_RHS_Z4C
 #define f_compute_rhs_Z4cnot COMPUTE_RHS_Z4CNOT
 #define f_compute_rhs_Z4c_ss COMPUTE_RHS_Z4C_SS
 #define f_compute_constraint_fr COMPUTE_CONSTRAINT_FR
 #endif
-#ifdef fortran3
-#define f_compute_rhs_bssn compute_rhs_bssn_
+#ifdef fortran3
+#define f_compute_rhs_bssn compute_rhs_bssn_
 #define f_compute_rhs_bssn_ss compute_rhs_bssn_ss_
-#define f_compute_rhs_bssn_escalar compute_rhs_bssn_escalar_
-#define f_compute_rhs_bssn_escalar_ss compute_rhs_bssn_escalar_ss_
+#define f_compute_rhs_bssn_escalar compute_rhs_bssn_escalar_
+#define f_compute_rhs_bssn_escalar_matter compute_rhs_bssn_escalar_matter_
+#define f_compute_rhs_bssn_escalar_ss compute_rhs_bssn_escalar_ss_
 #define f_compute_rhs_Z4c compute_rhs_z4c_
 #define f_compute_rhs_Z4cnot compute_rhs_z4cnot_
 #define f_compute_rhs_Z4c_ss compute_rhs_z4c_ss_
-#define f_compute_constraint_fr compute_constraint_fr_
-#endif
-
-#ifdef __cplusplus
-extern "C"
-{
-#endif
-        void f_bssn_rhs_kernel_timing_reset();
-        int f_bssn_rhs_kernel_timing_bucket_count();
-        const double *f_bssn_rhs_kernel_timing_local_seconds();
-        const char *f_bssn_rhs_kernel_timing_label(int);
-#ifdef __cplusplus
-}
-#endif
-
-extern "C"
-{
-        int f_compute_rhs_bssn(int *, double &, double *, double *, double *,                                                      // ex,T,X,Y,Z
+#define f_compute_constraint_fr compute_constraint_fr_
+#endif
+
+#ifdef __cplusplus
+extern "C"
+{
+#endif
+        void f_bssn_rhs_kernel_timing_reset();
+        int f_bssn_rhs_kernel_timing_bucket_count();
+        const double *f_bssn_rhs_kernel_timing_local_seconds();
+        const char *f_bssn_rhs_kernel_timing_label(int);
+#ifdef __cplusplus
+}
+#endif
+
+extern "C"
+{
+        int f_compute_rhs_bssn(int *, double &, double *, double *, double *,                                                      // ex,T,X,Y,Z
                                double *, double *,                                                                                 // chi, trK
                                double *, double *, double *, double *, double *, double *,                                         // gij
                                double *, double *, double *, double *, double *, double *,                                         // Aij
@@ -67,27 +70,6 @@ extern "C"
                                int &, int &, double &, int &);
 }
 
-int f_compute_rhs_bssn_escalar_c(int *, double &, double *, double *, double *,                                                      // ex,T,X,Y,Z
-                                 double *, double *,                                                                                 // chi, trK
-                                 double *, double *, double *, double *, double *, double *,                                         // gij
-                                 double *, double *, double *, double *, double *, double *,                                         // Aij
-                                 double *, double *, double *,                                                                       // Gam
-                                 double *, double *, double *, double *, double *, double *, double *,                               // Gauge
-                                 double *, double *,                                                                                 // Sphi, Spi
-                                 double *, double *,                                                                                 // chi, trK
-                                 double *, double *, double *, double *, double *, double *,                                         // gij
-                                 double *, double *, double *, double *, double *, double *,                                         // Aij
-                                 double *, double *, double *,                                                                       // Gam
-                                 double *, double *, double *, double *, double *, double *, double *,                               // Gauge
-                                 double *, double *,                                                                                 // Sphi, Spi
-                                 double *, double *, double *, double *, double *, double *, double *, double *, double *, double *, // stress-energy
-                                 double *, double *, double *, double *, double *, double *,                                         // Christoffel
-                                 double *, double *, double *, double *, double *, double *,                                         // Christoffel
-                                 double *, double *, double *, double *, double *, double *,                                         // Christoffel
-                                 double *, double *, double *, double *, double *, double *,                                         // Ricci
-                                 double *, double *, double *, double *, double *, double *, double *,                               // constraint violation
-                                 int &, int &, double &, int &);
-
 extern "C"
 {
         int f_compute_rhs_bssn_ss(int *, double &, double *, double *, double *,                                                      // ex,T,rho,sigma,R
@@ -117,10 +99,24 @@ extern "C"
                                   int &, int &, double &, int &, int &);
 }
 
-extern "C"
-{
-        int f_compute_rhs_bssn_escalar(int *, double &, double *, double *, double *,                                                      // ex,T,X,Y,Z
-                                       double *, double *,                                                                                 // chi, trK
+extern "C"
+{
+        int f_compute_rhs_bssn_escalar_matter(int *, double &, double *, double *, double *,                                             // ex,T,X,Y,Z
+                                              double *, double *,                                                                        // chi, trK
+                                              double *, double *, double *, double *, double *, double *,                                // gij
+                                              double *, double *, double *, double *, double *, double *,                                // Aij
+                                              double *, double *, double *,                                                              // Gam
+                                              double *, double *, double *, double *, double *, double *, double *,                      // Gauge
+                                              double *, double *,                                                                        // Sphi, Spi
+                                              double *, double *,                                                                        // Sphi, Spi rhs
+                                              double *, double *, double *, double *, double *, double *, double *, double *, double *, double *, // stress-energy
+                                              int &, int &, double &);
+}
+
+extern "C"
+{
+        int f_compute_rhs_bssn_escalar(int *, double &, double *, double *, double *,                                                      // ex,T,X,Y,Z
+                                       double *, double *,                                                                                 // chi, trK
                                        double *, double *, double *, double *, double *, double *,                                         // gij
                                        double *, double *, double *, double *, double *, double *,                                         // Aij
                                        double *, double *, double *,                                                                       // Gam
@@ -262,31 +258,4 @@ extern "C"
                                      double *);
 } // FR_cons
 
-// BSSN-EM C kernel (replaces empart.f90 + bssn_rhs.f90 for BSSN+Maxwell)
-int f_compute_rhs_bssn_em_c(int *, double &, double *, double *, double *,
-                             double *, double *,
-                             double *, double *, double *, double *, double *, double *,
-                             double *, double *, double *, double *, double *, double *,
-                             double *, double *, double *,
-                             double *, double *, double *, double *, double *, double *, double *,
-                             double *, double *, double *,
-                             double *, double *, double *, double *, double *, double *, double *, double *,
-                             double *, double *, double *,
-                             double *, double *,
-                             double *, double *,
-                             double *, double *, double *, double *, double *, double *,
-                             double *, double *, double *, double *, double *, double *,
-                             double *, double *, double *,
-                             double *, double *, double *, double *, double *, double *, double *,
-                             double *, double *, double *, double *, double *, double *, double *, double *,
-                             double *, double *, double *,
-                             double *, double *, double *, double *,
-                             double *, double *, double *, double *, double *, double *,
-                             double *, double *, double *, double *, double *, double *,
-                             double *, double *, double *, double *, double *, double *,
-                             double *, double *, double *, double *, double *, double *,
-                             double *, double *, double *, double *, double *, double *,
-                             double *, double *, double *, double *, double *, double *,
-                             int &, int &, double &, int &);
-
 #endif /* BSSN_H */

AMSS_NCKU_source/bssn_rhs_c.C

@@ -1075,10 +1075,6 @@ int f_compute_rhs_bssn(int *ex, double &T,
         }
         #endif
 
-        #if (GAUGE == 2 || GAUGE == 3 || GAUGE == 4 || GAUGE == 5)
-        fderivs(ex,chi,dtSfx_rhs,dtSfy_rhs,dtSfz_rhs,X,Y,Z,SYM,SYM,SYM,Symmetry,Lev);
-        #endif
-
         for (int i = 0; i < all; i += 1) {
             #if (GAUGE == 0)
             betax_rhs[i] = FF * dtSfx[i];
@@ -1102,12 +1098,12 @@ int f_compute_rhs_bssn(int *ex, double &T,
             betaz_rhs[i] = FF * dtSfz[i];
 
             reta[i] =
-                gupxx[i] * dtSfx_rhs[i] * dtSfx_rhs[i]
-                + gupyy[i] * dtSfy_rhs[i] * dtSfy_rhs[i]
-                + gupzz[i] * dtSfz_rhs[i] * dtSfz_rhs[i]
-                + TWO * ( gupxy[i] * dtSfx_rhs[i] * dtSfy_rhs[i]
-                        + gupxz[i] * dtSfx_rhs[i] * dtSfz_rhs[i]
-                        + gupyz[i] * dtSfy_rhs[i] * dtSfz_rhs[i] );
+                gupxx[i] * chix[i] * chix[i]
+                + gupyy[i] * chiy[i] * chiy[i]
+                + gupzz[i] * chiz[i] * chiz[i]
+                + TWO * ( gupxy[i] * chix[i] * chiy[i]
+                        + gupxz[i] * chix[i] * chiz[i]
+                        + gupyz[i] * chiy[i] * chiz[i] );
 
             #if (GAUGE == 2)
             reta[i] = 1.31 / 2.0 * sqrt( reta[i] / chin1[i] ) / pow( (ONE - sqrt(chin1[i])), 2.0 );
@@ -1120,12 +1116,12 @@ int f_compute_rhs_bssn(int *ex, double &T,
             dtSfz_rhs[i] = Gamz_rhs[i] - reta[i] * dtSfz[i];
             #elif (GAUGE == 4 || GAUGE == 5)
             reta[i] =
-                gupxx[i] * dtSfx_rhs[i] * dtSfx_rhs[i]
-                + gupyy[i] * dtSfy_rhs[i] * dtSfy_rhs[i]
-                + gupzz[i] * dtSfz_rhs[i] * dtSfz_rhs[i]
-                + TWO * ( gupxy[i] * dtSfx_rhs[i] * dtSfy_rhs[i]
-                        + gupxz[i] * dtSfx_rhs[i] * dtSfz_rhs[i]
-                        + gupyz[i] * dtSfy_rhs[i] * dtSfz_rhs[i] );
+                gupxx[i] * chix[i] * chix[i]
+                + gupyy[i] * chiy[i] * chiy[i]
+                + gupzz[i] * chiz[i] * chiz[i]
+                + TWO * ( gupxy[i] * chix[i] * chiy[i]
+                        + gupxz[i] * chix[i] * chiz[i]
+                        + gupyz[i] * chiy[i] * chiz[i] );
 
             #if (GAUGE == 4)
             reta[i] = 1.31 / 2.0 * sqrt( reta[i] / chin1[i] ) / pow( (ONE - sqrt(chin1[i])), 2.0 );
@@ -1164,17 +1160,11 @@ int f_compute_rhs_bssn(int *ex, double &T,
         lopsided_kodis(ex,X,Y,Z,gyz,gyz_rhs,betax,betay,betaz,Symmetry,SAA,eps);
         lopsided_kodis(ex,X,Y,Z,betaz,betaz_rhs,betax,betay,betaz,Symmetry,SSA,eps);
         lopsided_kodis(ex,X,Y,Z,dzz,gzz_rhs,betax,betay,betaz,Symmetry,SSS,eps);
-        #if (GAUGE == 0 || GAUGE == 2 || GAUGE == 3 || GAUGE == 6 || GAUGE == 7)
         lopsided_kodis(ex,X,Y,Z,dtSfx,dtSfx_rhs,betax,betay,betaz,Symmetry,ASS,eps);
-        #endif
         lopsided_kodis(ex,X,Y,Z,Axx,Axx_rhs,betax,betay,betaz,Symmetry,SSS,eps);
-        #if (GAUGE == 0 || GAUGE == 2 || GAUGE == 3 || GAUGE == 6 || GAUGE == 7)
         lopsided_kodis(ex,X,Y,Z,dtSfy,dtSfy_rhs,betax,betay,betaz,Symmetry,SAS,eps);
-        #endif
         lopsided_kodis(ex,X,Y,Z,Axy,Axy_rhs,betax,betay,betaz,Symmetry,AAS,eps);
-        #if (GAUGE == 0 || GAUGE == 2 || GAUGE == 3 || GAUGE == 6 || GAUGE == 7)
         lopsided_kodis(ex,X,Y,Z,dtSfz,dtSfz_rhs,betax,betay,betaz,Symmetry,SSA,eps);
-        #endif
         lopsided_kodis(ex,X,Y,Z,Axz,Axz_rhs,betax,betay,betaz,Symmetry,ASA,eps);
         lopsided_kodis(ex,X,Y,Z,Ayy,Ayy_rhs,betax,betay,betaz,Symmetry,SSS,eps);
         lopsided_kodis(ex,X,Y,Z,Ayz,Ayz_rhs,betax,betay,betaz,Symmetry,SAA,eps);

AMSS_NCKU_source/bssn_rhs_cuda.h

@@ -0,0 +1,470 @@
+#ifndef BSSN_RHS_CUDA_H
+#define BSSN_RHS_CUDA_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+enum {
+    BSSN_CUDA_STATE_COUNT = 24,
+    BSSN_CUDA_MATTER_COUNT = 10
+};
+
+int f_compute_rhs_bssn(int *ex, double &T,
+                       double *X, double *Y, double *Z,
+                       double *chi, double *trK,
+                       double *dxx, double *gxy, double *gxz, double *dyy, double *gyz, double *dzz,
+                       double *Axx, double *Axy, double *Axz, double *Ayy, double *Ayz, double *Azz,
+                       double *Gamx, double *Gamy, double *Gamz,
+                       double *Lap, double *betax, double *betay, double *betaz,
+                       double *dtSfx, double *dtSfy, double *dtSfz,
+                       double *chi_rhs, double *trK_rhs,
+                       double *gxx_rhs, double *gxy_rhs, double *gxz_rhs, double *gyy_rhs, double *gyz_rhs, double *gzz_rhs,
+                       double *Axx_rhs, double *Axy_rhs, double *Axz_rhs, double *Ayy_rhs, double *Ayz_rhs, double *Azz_rhs,
+                       double *Gamx_rhs, double *Gamy_rhs, double *Gamz_rhs,
+                       double *Lap_rhs, double *betax_rhs, double *betay_rhs, double *betaz_rhs,
+                       double *dtSfx_rhs, double *dtSfy_rhs, double *dtSfz_rhs,
+                       double *rho, double *Sx, double *Sy, double *Sz,
+                       double *Sxx, double *Sxy, double *Sxz, double *Syy, double *Syz, double *Szz,
+                       double *Gamxxx, double *Gamxxy, double *Gamxxz, double *Gamxyy, double *Gamxyz, double *Gamxzz,
+                       double *Gamyxx, double *Gamyxy, double *Gamyxz, double *Gamyyy, double *Gamyyz, double *Gamyzz,
+                       double *Gamzxx, double *Gamzxy, double *Gamzxz, double *Gamzyy, double *Gamzyz, double *Gamzzz,
+                       double *Rxx, double *Rxy, double *Rxz, double *Ryy, double *Ryz, double *Rzz,
+                       double *ham_Res, double *movx_Res, double *movy_Res, double *movz_Res,
+                       double *Gmx_Res, double *Gmy_Res, double *Gmz_Res,
+                       int &Symmetry, int &Lev, double &eps, int &co);
+
+int bssn_cuda_rk4_substep(void *block_tag,
+                          int *ex, double *X, double *Y, double *Z,
+                          double **state_host_in,
+                          double **state_host_out,
+                          double **matter_host,
+                          const double *propspeed,
+                          const double *soa_flat,
+                          const double *bbox,
+                          double &dT,
+                          double &T,
+                          int &RK4,
+                          int &apply_bam_bc,
+                          int &Symmetry,
+                          int &Lev,
+                          double &eps,
+                          int &co,
+                          int &use_zero_matter,
+                          int &keep_resident_state,
+                          int &apply_enforce_ga,
+                          double &chitiny);
+
+int bssn_cuda_compute_escalar_matter(void *block_tag,
+                                     int *ex, double *X, double *Y, double *Z,
+                                     double **state_host_in,
+                                     double *Sphi_host,
+                                     double *Spi_host,
+                                     double *Sphi_rhs_host,
+                                     double *Spi_rhs_host,
+                                     double a2,
+                                     int &Symmetry,
+                                     int &Lev,
+                                     double &eps,
+                                     int &co,
+                                     int &apply_enforce_ga);
+
+int bssn_cuda_escalar_finalize_scalar_fields(void *block_tag,
+                                             int *ex, double *X, double *Y, double *Z,
+                                             double *Sphi_out_host,
+                                             double *Spi_out_host,
+                                             const double *propspeed,
+                                             const double *soa_flat,
+                                             const double *bbox,
+                                             double &dT,
+                                             int &RK4,
+                                             int &apply_bam_bc,
+                                             int &Symmetry,
+                                             int &Lev,
+                                             double &eps,
+                                             int &precor);
+
+int bssn_cuda_escalar_has_resident_fields(void *block_tag,
+                                          double *Sphi_host,
+                                          double *Spi_host);
+
+int bssn_cuda_escalar_has_any_resident_fields(void *block_tag);
+
+int bssn_cuda_escalar_download_fields_if_present(void *block_tag,
+                                                 int *ex,
+                                                 double *Sphi_host,
+                                                 double *Spi_host);
+
+int bssn_cuda_pack_escalar_batch_to_host_buffer(void *block_tag,
+                                                double **scalar_host_key,
+                                                double *host_buffer,
+                                                int *ex,
+                                                int i0, int j0, int k0,
+                                                int sx, int sy, int sz);
+
+int bssn_cuda_unpack_escalar_batch_from_host_buffer(void *block_tag,
+                                                    double **scalar_host_key,
+                                                    double *host_buffer,
+                                                    int *ex,
+                                                    int i0, int j0, int k0,
+                                                    int sx, int sy, int sz);
+
+int bssn_cuda_pack_escalar_batch_to_device_buffer(void *block_tag,
+                                                  double **scalar_host_key,
+                                                  double *device_buffer,
+                                                  int *ex,
+                                                  int i0, int j0, int k0,
+                                                  int sx, int sy, int sz);
+
+int bssn_cuda_unpack_escalar_batch_from_device_buffer(void *block_tag,
+                                                      double **scalar_host_key,
+                                                      double *device_buffer,
+                                                      int *ex,
+                                                      int i0, int j0, int k0,
+                                                      int sx, int sy, int sz);
+
+int bssn_cuda_restrict_escalar_batch_to_host_buffer(void *block_tag,
+                                                    double **scalar_host_key,
+                                                    double *host_buffer,
+                                                    int *ex,
+                                                    int sx, int sy, int sz,
+                                                    int fi0, int fj0, int fk0,
+                                                    const double *scalar_soa);
+
+int bssn_cuda_prolong_escalar_batch_to_host_buffer(void *block_tag,
+                                                   double **scalar_host_key,
+                                                   double *host_buffer,
+                                                   int *ex,
+                                                   int sx, int sy, int sz,
+                                                   int ii0, int jj0, int kk0,
+                                                   int lbc_i, int lbc_j, int lbc_k,
+                                                   const double *scalar_soa);
+
+int bssn_cuda_restrict_escalar_batch_to_device_buffer(void *block_tag,
+                                                      double **scalar_host_key,
+                                                      double *device_buffer,
+                                                      int *ex,
+                                                      int sx, int sy, int sz,
+                                                      int fi0, int fj0, int fk0,
+                                                      const double *scalar_soa);
+
+int bssn_cuda_prolong_escalar_batch_to_device_buffer(void *block_tag,
+                                                     double **scalar_host_key,
+                                                     double *device_buffer,
+                                                     int *ex,
+                                                     int sx, int sy, int sz,
+                                                     int ii0, int jj0, int kk0,
+                                                     int lbc_i, int lbc_j, int lbc_k,
+                                                     const double *scalar_soa);
+
+int bssn_cuda_prepare_escalar_inter_time_level(void *block_tag,
+                                               int *ex,
+                                               double **src1_host_key,
+                                               double **src2_host_key,
+                                               double **src3_host_key,
+                                               double **dst_host_key,
+                                               int source_count,
+                                               int tindex);
+
+int bssn_cuda_copy_state_region_to_host(void *block_tag,
+                                        int state_index,
+                                        double *host_state,
+                                        int *ex,
+                                        int i0, int j0, int k0,
+                                        int sx, int sy, int sz);
+
+int bssn_cuda_copy_state_region_from_host(void *block_tag,
+                                          int state_index,
+                                          double *host_state,
+                                          int *ex,
+                                          int i0, int j0, int k0,
+                                          int sx, int sy, int sz);
+
+int bssn_cuda_download_resident_state(void *block_tag,
+                                      int *ex,
+                                      double **state_host_out);
+
+int bssn_cuda_download_resident_state_if_present(void *block_tag,
+                                                int *ex,
+                                                double **state_host_out);
+
+int bssn_cuda_resident_state_matches(void *block_tag,
+                                     double **state_host_key);
+
+int bssn_cuda_download_constraint_outputs(int *ex,
+                                          double **constraint_host_out);
+
+int bssn_cuda_pack_state_region_to_host_buffer(void *block_tag,
+                                               int state_index,
+                                               double *host_buffer,
+                                               int *ex,
+                                               int i0, int j0, int k0,
+                                               int sx, int sy, int sz);
+
+int bssn_cuda_interp_state_point3(void *block_tag,
+                                  int *ex,
+                                  int state0,
+                                  int state1,
+                                  int state2,
+                                  double x0,
+                                  double y0,
+                                  double z0,
+                                  double dx,
+                                  double dy,
+                                  double dz,
+                                  double px,
+                                  double py,
+                                  double pz,
+                                  int ordn,
+                                  int symmetry,
+                                  const double *soa3,
+                                  double *out3);
+
+int bssn_cuda_interp_host_two_fields(void *block_tag,
+                                     int *ex,
+                                     double *field0,
+                                     double *field1,
+                                     double x0,
+                                     double y0,
+                                     double z0,
+                                     double dx,
+                                     double dy,
+                                     double dz,
+                                     const double *px,
+                                     const double *py,
+                                     const double *pz,
+                                     int npoints,
+                                     int ordn,
+                                     int symmetry,
+                                     const double *soa6,
+                                     double *out_interleaved);
+
+int bssn_cuda_unpack_state_region_from_host_buffer(void *block_tag,
+                                                   int state_index,
+                                                   double *host_buffer,
+                                                   int *ex,
+                                                   int i0, int j0, int k0,
+                                                   int sx, int sy, int sz);
+
+int bssn_cuda_pack_state_batch_to_host_buffer(void *block_tag,
+                                              int state_count,
+                                              double *host_buffer,
+                                              int *ex,
+                                              int i0, int j0, int k0,
+                                              int sx, int sy, int sz);
+
+int bssn_cuda_pack_state_batch_to_host_buffer_for_host_views(void *block_tag,
+                                                             double **state_host_key,
+                                                             int state_count,
+                                                             double *host_buffer,
+                                                             int *ex,
+                                                             int i0, int j0, int k0,
+                                                             int sx, int sy, int sz);
+
+int bssn_cuda_unpack_state_batch_from_host_buffer(void *block_tag,
+                                                  int state_count,
+                                                  double *host_buffer,
+                                                  int *ex,
+                                                  int i0, int j0, int k0,
+                                                  int sx, int sy, int sz);
+
+int bssn_cuda_unpack_state_batch_from_host_buffer_for_host_views(void *block_tag,
+                                                                 double **state_host_key,
+                                                                 int state_count,
+                                                                 double *host_buffer,
+                                                                 int *ex,
+                                                                 int i0, int j0, int k0,
+                                                                 int sx, int sy, int sz);
+
+int bssn_cuda_restrict_state_batch_to_host_buffer_for_host_views(void *block_tag,
+                                                                 double **state_host_key,
+                                                                 int state_count,
+                                                                 double *host_buffer,
+                                                                 int *ex,
+                                                                 int sx, int sy, int sz,
+                                                                 int fi0, int fj0, int fk0,
+                                                                 const double *state_soa);
+
+int bssn_cuda_restrict_state_batch_to_host_buffer(void *block_tag,
+                                                  int state_count,
+                                                  double *host_buffer,
+                                                  int *ex,
+                                                  int sx, int sy, int sz,
+                                                  int fi0, int fj0, int fk0,
+                                                  const double *state_soa);
+
+int bssn_cuda_prolong_state_batch_to_host_buffer_for_host_views(void *block_tag,
+                                                                double **state_host_key,
+                                                                int state_count,
+                                                                double *host_buffer,
+                                                                int *ex,
+                                                                int sx, int sy, int sz,
+                                                                int ii0, int jj0, int kk0,
+                                                                int lbc_i, int lbc_j, int lbc_k,
+                                                                const double *state_soa);
+
+int bssn_cuda_prolong_state_batch_to_host_buffer(void *block_tag,
+                                                 int state_count,
+                                                 double *host_buffer,
+                                                 int *ex,
+                                                 int sx, int sy, int sz,
+                                                 int ii0, int jj0, int kk0,
+                                                 int lbc_i, int lbc_j, int lbc_k,
+                                                 const double *state_soa);
+
+int bssn_cuda_pack_state_batch_to_device_buffer(void *block_tag,
+                                                 int state_count,
+                                                double *device_buffer,
+                                                int *ex,
+                                                int i0, int j0, int k0,
+                                                int sx, int sy, int sz);
+
+int bssn_cuda_pack_state_batch_to_device_buffer_for_host_views(void *block_tag,
+                                                               double **state_host_key,
+                                                               int state_count,
+                                                               double *device_buffer,
+                                                               int *ex,
+                                                               int i0, int j0, int k0,
+                                                               int sx, int sy, int sz);
+
+int bssn_cuda_unpack_state_batch_from_device_buffer(void *block_tag,
+                                                    int state_count,
+                                                    double *device_buffer,
+                                                    int *ex,
+                                                    int i0, int j0, int k0,
+                                                    int sx, int sy, int sz);
+
+int bssn_cuda_unpack_state_batch_from_device_buffer_for_host_views(void *block_tag,
+                                                                   double **state_host_key,
+                                                                   int state_count,
+                                                                   double *device_buffer,
+                                                                   int *ex,
+                                                                   int i0, int j0, int k0,
+                                                                   int sx, int sy, int sz);
+
+int bssn_cuda_pack_state_segments_to_device_buffer(void *block_tag,
+                                                   int state_count,
+                                                   double *device_buffer,
+                                                   int *ex,
+                                                   int segment_count,
+                                                   const int *segment_meta);
+
+int bssn_cuda_pack_state_segments_to_device_buffer_for_host_views(void *block_tag,
+                                                                  double **state_host_key,
+                                                                  int state_count,
+                                                                  double *device_buffer,
+                                                                  int *ex,
+                                                                  int segment_count,
+                                                                  const int *segment_meta);
+
+int bssn_cuda_unpack_state_segments_from_device_buffer(void *block_tag,
+                                                       int state_count,
+                                                       double *device_buffer,
+                                                       int *ex,
+                                                       int segment_count,
+                                                       const int *segment_meta);
+
+int bssn_cuda_unpack_state_segments_from_device_buffer_for_host_views(void *block_tag,
+                                                                      double **state_host_key,
+                                                                      int state_count,
+                                                                      double *device_buffer,
+                                                                      int *ex,
+                                                                      int segment_count,
+                                                                      const int *segment_meta);
+
+int bssn_cuda_restrict_state_segments_to_device_buffer(void *block_tag,
+                                                       int state_count,
+                                                       double *device_buffer,
+                                                       int *ex,
+                                                       int segment_count,
+                                                       const int *segment_meta);
+
+int bssn_cuda_restrict_state_segments_to_device_buffer_for_host_views(void *block_tag,
+                                                                      double **state_host_key,
+                                                                      int state_count,
+                                                                      double *device_buffer,
+                                                                      int *ex,
+                                                                      int segment_count,
+                                                                      const int *segment_meta,
+                                                                      const double *state_soa);
+
+int bssn_cuda_prolong_state_segments_to_device_buffer(void *block_tag,
+                                                      int state_count,
+                                                      double *device_buffer,
+                                                      int *ex,
+                                                      int segment_count,
+                                                      const int *segment_meta);
+
+int bssn_cuda_prolong_state_segments_to_device_buffer_for_host_views(void *block_tag,
+                                                                     double **state_host_key,
+                                                                     int state_count,
+                                                                     double *device_buffer,
+                                                                     int *ex,
+                                                                     int segment_count,
+                                                                     const int *segment_meta,
+                                                                     const double *state_soa);
+
+int bssn_cuda_restrict_state_batch_to_device_buffer(void *block_tag,
+                                                    int state_count,
+                                                    double *device_buffer,
+                                                    int *ex,
+                                                    int sx, int sy, int sz,
+                                                    int fi0, int fj0, int fk0);
+
+int bssn_cuda_restrict_state_batch_to_device_buffer_for_host_views(void *block_tag,
+                                                                   double **state_host_key,
+                                                                   int state_count,
+                                                                   double *device_buffer,
+                                                                   int *ex,
+                                                                   int sx, int sy, int sz,
+                                                                   int fi0, int fj0, int fk0,
+                                                                   const double *state_soa);
+
+int bssn_cuda_prolong_state_batch_to_device_buffer(void *block_tag,
+                                                   int state_count,
+                                                   double *device_buffer,
+                                                   int *ex,
+                                                   int sx, int sy, int sz,
+                                                   int ii0, int jj0, int kk0,
+                                                   int lbc_i, int lbc_j, int lbc_k);
+
+int bssn_cuda_prolong_state_batch_to_device_buffer_for_host_views(void *block_tag,
+                                                                  double **state_host_key,
+                                                                  int state_count,
+                                                                  double *device_buffer,
+                                                                  int *ex,
+                                                                  int sx, int sy, int sz,
+                                                                  int ii0, int jj0, int kk0,
+                                                                  int lbc_i, int lbc_j, int lbc_k,
+                                                                  const double *state_soa);
+
+int bssn_cuda_download_state_subset(void *block_tag,
+                                    int *ex,
+                                    int subset_count,
+                                    const int *state_indices,
+                                    double **state_host_out);
+
+int bssn_cuda_upload_state_subset(void *block_tag,
+                                  int *ex,
+                                  int subset_count,
+                                  const int *state_indices,
+                                  double **state_host_in);
+
+int bssn_cuda_prepare_inter_time_level(void *block_tag,
+                                       int *ex,
+                                       double **src1_host_key,
+                                       double **src2_host_key,
+                                       double **src3_host_key,
+                                       double **dst_host_key,
+                                       int source_count,
+                                       int tindex);
+
+int bssn_cuda_has_resident_state(void *block_tag);
+
+void bssn_cuda_release_step_ctx(void *block_tag);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

AMSS_NCKU_source/fdderivs_sh_c.C

@@ -1,321 +0,0 @@
-#include "macrodef.h"
-#include "share_func.h"
-
-/*
- * fdderivs_sh — second derivatives on shell patch in (rho, sigma, R) coords.
- * Same stencil coefficients as Cartesian fdderivs. Uses symmetry_stbd.
- */
-extern "C" void fdderivs_sh_(const int ex[3],
-                 const double *f,
-                 double *fxx, double *fxy, double *fxz,
-                 double *fyy, double *fyz, double *fzz,
-                 const double *X, const double *Y, const double *Z,
-                 double SYM1, double SYM2, double SYM3,
-                 int Symmetry, int onoff, int sst)
-{
-    (void)SYM3; (void)onoff; (void)sst;
-
-    const int NO_SYMM=0, EQ_SYMM=1, OCTANT=2;
-    const double ZEO=0.0, ONE=1.0, TWO=2.0, F1o4=2.5e-1;
-    const double F8=8.0, F16=16.0, F30=30.0, F1o12=ONE/12.0, F1o144=ONE/144.0;
-    const double F9=9.0, F45=45.0, F60=60.0, F27=27.0, F270=270.0, F490=490.0;
-    const double F1o180=ONE/180.0, F1o3600=ONE/3600.0;
-    const double F32=32.0, F128=128.0, F168=168.0, F672=672.0, F840=840.0;
-    const double F1008=1008.0, F8064=8064.0, F14350=14350.0;
-    const double F1o5040=ONE/5040.0, F1o705600=ONE/705600.0;
-
-    const int ex1=ex[0], ex2=ex[1], ex3=ex[2];
-    const double dX=X[1]-X[0], dY=Y[1]-Y[0], dZ=Z[1]-Z[0];
-    const int imaxF=ex1, jmaxF=ex2, kmaxF=ex3;
-    const double SoA[2]={SYM1,SYM2};
-
-#if (ghost_width == 2)
-    {
-        const int ord=1;
-        int iminF=1,jminF=1,kminF=1;
-        if(Symmetry==OCTANT&&fabs(X[0])<dX)iminF=0;
-        if(Symmetry==OCTANT&&fabs(Y[0])<dY)jminF=0;
-        if((sst==2||sst==4)&&fabs(Y[0])<dY)jminF=0;
-
-        const size_t nx=(size_t)ex1+2*ord,ny=(size_t)ex2+2*ord,nz=(size_t)ex3,fh_size=nx*ny*nz;
-        static double *fh_buf=NULL;static size_t cap=0;
-        if(fh_size>cap){free(fh_buf);fh_buf=(double*)aligned_alloc(64,fh_size*sizeof(double));cap=fh_size;}
-        double *fh=fh_buf;if(!fh)return;
-        symmetry_stbd(ord,ex,f,fh,SoA);
-
-        const double Sdxdx=ONE/(dX*dX),Sdydy=ONE/(dY*dY),Sdzdz=ONE/(dZ*dZ);
-        const double Sdxdy=F1o4/(dX*dY),Sdxdz=F1o4/(dX*dZ),Sdydz=F1o4/(dY*dZ);
-        const size_t all=(size_t)ex1*ex2*ex3;
-        for(size_t p=0;p<all;++p){fxx[p]=fyy[p]=fzz[p]=ZEO;fxy[p]=fxz[p]=fyz[p]=ZEO;}
-
-        const int i2_lo=(iminF>0)?iminF:0,j2_lo=(jminF>0)?jminF:0,k2_lo=1,i2_hi=ex1-2,j2_hi=ex2-2,k2_hi=ex3-2;
-        #define FH(iF,jF,kF) fh[idx_fh_stbd(iF,jF,kF,ord,ex)]
-        if(i2_lo<=i2_hi&&j2_lo<=j2_hi&&k2_lo<=k2_hi){
-            for(int k0=k2_lo;k0<=k2_hi;++k0){const int kF=k0+1;
-            for(int j0=j2_lo;j0<=j2_hi;++j0){const int jF=j0+1;
-            for(int i0=i2_lo;i0<=i2_hi;++i0){const int iF=i0+1;const size_t p=idx_ex(i0,j0,k0,ex);
-                fxx[p]=Sdxdx*(FH(iF-1,jF,kF)-TWO*FH(iF,jF,kF)+FH(iF+1,jF,kF));
-                fyy[p]=Sdydy*(FH(iF,jF-1,kF)-TWO*FH(iF,jF,kF)+FH(iF,jF+1,kF));
-                fzz[p]=Sdzdz*(FH(iF,jF,kF-1)-TWO*FH(iF,jF,kF)+FH(iF,jF,kF+1));
-                fxy[p]=Sdxdy*(FH(iF-1,jF-1,kF)-FH(iF+1,jF-1,kF)-FH(iF-1,jF+1,kF)+FH(iF+1,jF+1,kF));
-                fxz[p]=Sdxdz*(FH(iF-1,jF,kF-1)-FH(iF+1,jF,kF-1)-FH(iF-1,jF,kF+1)+FH(iF+1,jF,kF+1));
-                fyz[p]=Sdydz*(FH(iF,jF-1,kF-1)-FH(iF,jF+1,kF-1)-FH(iF,jF-1,kF+1)+FH(iF,jF+1,kF+1));
-            }}}
-        }
-        #undef FH
-        return;
-    }
-#elif (ghost_width == 3)
-    {
-        const int ord=2;
-        int iminF=1,jminF=1,kminF=1;
-        if(Symmetry==OCTANT&&fabs(X[0])<dX)iminF=-1;
-        if(Symmetry==OCTANT&&fabs(Y[0])<dY)jminF=-1;
-        if((sst==2||sst==4)&&fabs(Y[0])<dY)jminF=-1;
-
-        const size_t nx=(size_t)ex1+2*ord,ny=(size_t)ex2+2*ord,nz=(size_t)ex3,fh_size=nx*ny*nz;
-        static double *fh_buf=NULL;static size_t cap=0;
-        if(fh_size>cap){free(fh_buf);fh_buf=(double*)aligned_alloc(64,fh_size*sizeof(double));cap=fh_size;}
-        double *fh=fh_buf;if(!fh)return;
-        symmetry_stbd(ord,ex,f,fh,SoA);
-
-        const double Sdxdx=ONE/(dX*dX),Sdydy=ONE/(dY*dY),Sdzdz=ONE/(dZ*dZ);
-        const double Fdxdx=F1o12/(dX*dX),Fdydy=F1o12/(dY*dY),Fdzdz=F1o12/(dZ*dZ);
-        const double Sdxdy=F1o4/(dX*dY),Sdxdz=F1o4/(dX*dZ),Sdydz=F1o4/(dY*dZ);
-        const double Fdxdy=F1o144/(dX*dY),Fdxdz=F1o144/(dX*dZ),Fdydz=F1o144/(dY*dZ);
-        const size_t all=(size_t)ex1*ex2*ex3;
-        for(size_t p=0;p<all;++p){fxx[p]=fyy[p]=fzz[p]=fxy[p]=fxz[p]=fyz[p]=ZEO;}
-
-        const int i2_lo=(iminF>0)?iminF:0,j2_lo=(jminF>0)?jminF:0,k2_lo=1,i2_hi=ex1-2,j2_hi=ex2-2,k2_hi=ex3-2;
-        const int i4_lo=(iminF+1>0)?iminF+1:0,j4_lo=(jminF+1>0)?jminF+1:0,k4_lo=2,i4_hi=ex1-3,j4_hi=ex2-3,k4_hi=ex3-3;
-        const int has4=(i4_lo<=i4_hi&&j4_lo<=j4_hi&&k4_lo<=k4_hi);
-        #define FH(iF,jF,kF) fh[idx_fh_stbd(iF,jF,kF,ord,ex)]
-
-        if(i2_lo<=i2_hi&&j2_lo<=j2_hi&&k2_lo<=k2_hi){
-            for(int k0=k2_lo;k0<=k2_hi;++k0){const int kF=k0+1;
-            for(int j0=j2_lo;j0<=j2_hi;++j0){const int jF=j0+1;
-            for(int i0=i2_lo;i0<=i2_hi;++i0){
-                if(has4&&i0>=i4_lo&&i0<=i4_hi&&j0>=j4_lo&&j0<=j4_hi&&k0>=k4_lo&&k0<=k4_hi)continue;
-                const int iF=i0+1;const size_t p=idx_ex(i0,j0,k0,ex);
-                fxx[p]=Sdxdx*(FH(iF-1,jF,kF)-TWO*FH(iF,jF,kF)+FH(iF+1,jF,kF));
-                fyy[p]=Sdydy*(FH(iF,jF-1,kF)-TWO*FH(iF,jF,kF)+FH(iF,jF+1,kF));
-                fzz[p]=Sdzdz*(FH(iF,jF,kF-1)-TWO*FH(iF,jF,kF)+FH(iF,jF,kF+1));
-                fxy[p]=Sdxdy*(FH(iF-1,jF-1,kF)-FH(iF+1,jF-1,kF)-FH(iF-1,jF+1,kF)+FH(iF+1,jF+1,kF));
-                fxz[p]=Sdxdz*(FH(iF-1,jF,kF-1)-FH(iF+1,jF,kF-1)-FH(iF-1,jF,kF+1)+FH(iF+1,jF,kF+1));
-                fyz[p]=Sdydz*(FH(iF,jF-1,kF-1)-FH(iF,jF+1,kF-1)-FH(iF,jF-1,kF+1)+FH(iF,jF+1,kF+1));
-            }}}
-        }
-        if(has4){
-            for(int k0=k4_lo;k0<=k4_hi;++k0){const int kF=k0+1;
-            for(int j0=j4_lo;j0<=j4_hi;++j0){const int jF=j0+1;
-            for(int i0=i4_lo;i0<=i4_hi;++i0){const int iF=i0+1;const size_t p=idx_ex(i0,j0,k0,ex);
-                fxx[p]=Fdxdx*(-FH(iF-2,jF,kF)+F16*FH(iF-1,jF,kF)-F30*FH(iF,jF,kF)-FH(iF+2,jF,kF)+F16*FH(iF+1,jF,kF));
-                fyy[p]=Fdydy*(-FH(iF,jF-2,kF)+F16*FH(iF,jF-1,kF)-F30*FH(iF,jF,kF)-FH(iF,jF+2,kF)+F16*FH(iF,jF+1,kF));
-                fzz[p]=Fdzdz*(-FH(iF,jF,kF-2)+F16*FH(iF,jF,kF-1)-F30*FH(iF,jF,kF)-FH(iF,jF,kF+2)+F16*FH(iF,jF,kF+1));
-                {const double t_jm2=(FH(iF-2,jF-2,kF)-F8*FH(iF-1,jF-2,kF)+F8*FH(iF+1,jF-2,kF)-FH(iF+2,jF-2,kF));
-                 const double t_jm1=(FH(iF-2,jF-1,kF)-F8*FH(iF-1,jF-1,kF)+F8*FH(iF+1,jF-1,kF)-FH(iF+2,jF-1,kF));
-                 const double t_jp1=(FH(iF-2,jF+1,kF)-F8*FH(iF-1,jF+1,kF)+F8*FH(iF+1,jF+1,kF)-FH(iF+2,jF+1,kF));
-                 const double t_jp2=(FH(iF-2,jF+2,kF)-F8*FH(iF-1,jF+2,kF)+F8*FH(iF+1,jF+2,kF)-FH(iF+2,jF+2,kF));
-                 fxy[p]=Fdxdy*(t_jm2-F8*t_jm1+F8*t_jp1-t_jp2);}
-                {const double t_km2=(FH(iF-2,jF,kF-2)-F8*FH(iF-1,jF,kF-2)+F8*FH(iF+1,jF,kF-2)-FH(iF+2,jF,kF-2));
-                 const double t_km1=(FH(iF-2,jF,kF-1)-F8*FH(iF-1,jF,kF-1)+F8*FH(iF+1,jF,kF-1)-FH(iF+2,jF,kF-1));
-                 const double t_kp1=(FH(iF-2,jF,kF+1)-F8*FH(iF-1,jF,kF+1)+F8*FH(iF+1,jF,kF+1)-FH(iF+2,jF,kF+1));
-                 const double t_kp2=(FH(iF-2,jF,kF+2)-F8*FH(iF-1,jF,kF+2)+F8*FH(iF+1,jF,kF+2)-FH(iF+2,jF,kF+2));
-                 fxz[p]=Fdxdz*(t_km2-F8*t_km1+F8*t_kp1-t_kp2);}
-                {const double t_km2=(FH(iF,jF-2,kF-2)-F8*FH(iF,jF-1,kF-2)+F8*FH(iF,jF+1,kF-2)-FH(iF,jF+2,kF-2));
-                 const double t_km1=(FH(iF,jF-2,kF-1)-F8*FH(iF,jF-1,kF-1)+F8*FH(iF,jF+1,kF-1)-FH(iF,jF+2,kF-1));
-                 const double t_kp1=(FH(iF,jF-2,kF+1)-F8*FH(iF,jF-1,kF+1)+F8*FH(iF,jF+1,kF+1)-FH(iF,jF+2,kF+1));
-                 const double t_kp2=(FH(iF,jF-2,kF+2)-F8*FH(iF,jF-1,kF+2)+F8*FH(iF,jF+1,kF+2)-FH(iF,jF+2,kF+2));
-                 fyz[p]=Fdydz*(t_km2-F8*t_km1+F8*t_kp1-t_kp2);}
-            }}}
-        }
-        #undef FH
-        return;
-    }
-#elif (ghost_width == 4)
-    {
-        const int ord=3;
-        int iminF=1,jminF=1,kminF=1;
-        if(Symmetry==OCTANT&&fabs(X[0])<dX)iminF=-2;
-        if(Symmetry==OCTANT&&fabs(Y[0])<dY)jminF=-2;
-        if((sst==2||sst==4)&&fabs(Y[0])<dY)jminF=-2;
-
-        const size_t nx=(size_t)ex1+2*ord,ny=(size_t)ex2+2*ord,nz=(size_t)ex3,fh_size=nx*ny*nz;
-        static double *fh_buf=NULL;static size_t cap=0;
-        if(fh_size>cap){free(fh_buf);fh_buf=(double*)aligned_alloc(64,fh_size*sizeof(double));cap=fh_size;}
-        double *fh=fh_buf;if(!fh)return;
-        symmetry_stbd(ord,ex,f,fh,SoA);
-
-        const double Sdxdx=ONE/(dX*dX),Sdydy=ONE/(dY*dY),Sdzdz=ONE/(dZ*dZ);
-        const double Fdxdx=F1o12/(dX*dX),Fdydy=F1o12/(dY*dY),Fdzdz=F1o12/(dZ*dZ);
-        const double Xdxdx=F1o180/(dX*dX),Xdydy=F1o180/(dY*dY),Xdzdz=F1o180/(dZ*dZ);
-        const double Sdxdy=F1o4/(dX*dY),Sdxdz=F1o4/(dX*dZ),Sdydz=F1o4/(dY*dZ);
-        const double Fdxdy=F1o144/(dX*dY),Fdxdz=F1o144/(dX*dZ),Fdydz=F1o144/(dY*dZ);
-        const double Xdxdy=F1o3600/(dX*dY),Xdxdz=F1o3600/(dX*dZ),Xdydz=F1o3600/(dY*dZ);
-        const size_t all=(size_t)ex1*ex2*ex3;
-        for(size_t p=0;p<all;++p){fxx[p]=fyy[p]=fzz[p]=fxy[p]=fxz[p]=fyz[p]=ZEO;}
-
-        const int i2_lo=(iminF>0)?iminF:0,j2_lo=(jminF>0)?jminF:0,k2_lo=1,i2_hi=ex1-2,j2_hi=ex2-2,k2_hi=ex3-2;
-        const int i4_lo=(iminF+1>0)?iminF+1:0,j4_lo=(jminF+1>0)?jminF+1:0,k4_lo=2,i4_hi=ex1-3,j4_hi=ex2-3,k4_hi=ex3-3;
-        const int i6_lo=(iminF+2>0)?iminF+2:0,j6_lo=(jminF+2>0)?jminF+2:0,k6_lo=3,i6_hi=ex1-4,j6_hi=ex2-4,k6_hi=ex3-4;
-        const int has4=(i4_lo<=i4_hi&&j4_lo<=j4_hi&&k4_lo<=k4_hi),has6=(i6_lo<=i6_hi&&j6_lo<=j6_hi&&k6_lo<=k6_hi);
-        #define FH(iF,jF,kF) fh[idx_fh_stbd(iF,jF,kF,ord,ex)]
-
-        if(i2_lo<=i2_hi&&j2_lo<=j2_hi&&k2_lo<=k2_hi){for(int k0=k2_lo;k0<=k2_hi;++k0){const int kF=k0+1;
-        for(int j0=j2_lo;j0<=j2_hi;++j0){const int jF=j0+1;
-        for(int i0=i2_lo;i0<=i2_hi;++i0){bool in4=has4&&i0>=i4_lo&&i0<=i4_hi&&j0>=j4_lo&&j0<=j4_hi&&k0>=k4_lo&&k0<=k4_hi;if(in4)continue;
-            const int iF=i0+1;const size_t p=idx_ex(i0,j0,k0,ex);
-            fxx[p]=Sdxdx*(FH(iF-1,jF,kF)-TWO*FH(iF,jF,kF)+FH(iF+1,jF,kF));
-            fyy[p]=Sdydy*(FH(iF,jF-1,kF)-TWO*FH(iF,jF,kF)+FH(iF,jF+1,kF));
-            fzz[p]=Sdzdz*(FH(iF,jF,kF-1)-TWO*FH(iF,jF,kF)+FH(iF,jF,kF+1));
-            fxy[p]=Sdxdy*(FH(iF-1,jF-1,kF)-FH(iF+1,jF-1,kF)-FH(iF-1,jF+1,kF)+FH(iF+1,jF+1,kF));
-            fxz[p]=Sdxdz*(FH(iF-1,jF,kF-1)-FH(iF+1,jF,kF-1)-FH(iF-1,jF,kF+1)+FH(iF+1,jF,kF+1));
-            fyz[p]=Sdydz*(FH(iF,jF-1,kF-1)-FH(iF,jF+1,kF-1)-FH(iF,jF-1,kF+1)+FH(iF,jF+1,kF+1));
-        }}}}
-        if(has4){for(int k0=k4_lo;k0<=k4_hi;++k0){const int kF=k0+1;
-        for(int j0=j4_lo;j0<=j4_hi;++j0){const int jF=j0+1;
-        for(int i0=i4_lo;i0<=i4_hi;++i0){if(has6&&i0>=i6_lo&&i0<=i6_hi&&j0>=j6_lo&&j0<=j6_hi&&k0>=k6_lo&&k0<=k6_hi)continue;
-            const int iF=i0+1;const size_t p=idx_ex(i0,j0,k0,ex);
-            fxx[p]=Fdxdx*(-FH(iF-2,jF,kF)+F16*FH(iF-1,jF,kF)-F30*FH(iF,jF,kF)-FH(iF+2,jF,kF)+F16*FH(iF+1,jF,kF));
-            fyy[p]=Fdydy*(-FH(iF,jF-2,kF)+F16*FH(iF,jF-1,kF)-F30*FH(iF,jF,kF)-FH(iF,jF+2,kF)+F16*FH(iF,jF+1,kF));
-            fzz[p]=Fdzdz*(-FH(iF,jF,kF-2)+F16*FH(iF,jF,kF-1)-F30*FH(iF,jF,kF)-FH(iF,jF,kF+2)+F16*FH(iF,jF,kF+1));
-            {const double t_jm2=(FH(iF-2,jF-2,kF)-F8*FH(iF-1,jF-2,kF)+F8*FH(iF+1,jF-2,kF)-FH(iF+2,jF-2,kF));
-             const double t_jm1=(FH(iF-2,jF-1,kF)-F8*FH(iF-1,jF-1,kF)+F8*FH(iF+1,jF-1,kF)-FH(iF+2,jF-1,kF));
-             const double t_jp1=(FH(iF-2,jF+1,kF)-F8*FH(iF-1,jF+1,kF)+F8*FH(iF+1,jF+1,kF)-FH(iF+2,jF+1,kF));
-             const double t_jp2=(FH(iF-2,jF+2,kF)-F8*FH(iF-1,jF+2,kF)+F8*FH(iF+1,jF+2,kF)-FH(iF+2,jF+2,kF));
-             fxy[p]=Fdxdy*(t_jm2-F8*t_jm1+F8*t_jp1-t_jp2);}
-            {const double t_km2=(FH(iF-2,jF,kF-2)-F8*FH(iF-1,jF,kF-2)+F8*FH(iF+1,jF,kF-2)-FH(iF+2,jF,kF-2));
-             const double t_km1=(FH(iF-2,jF,kF-1)-F8*FH(iF-1,jF,kF-1)+F8*FH(iF+1,jF,kF-1)-FH(iF+2,jF,kF-1));
-             const double t_kp1=(FH(iF-2,jF,kF+1)-F8*FH(iF-1,jF,kF+1)+F8*FH(iF+1,jF,kF+1)-FH(iF+2,jF,kF+1));
-             const double t_kp2=(FH(iF-2,jF,kF+2)-F8*FH(iF-1,jF,kF+2)+F8*FH(iF+1,jF,kF+2)-FH(iF+2,jF,kF+2));
-             fxz[p]=Fdxdz*(t_km2-F8*t_km1+F8*t_kp1-t_kp2);}
-            {const double t_km2=(FH(iF,jF-2,kF-2)-F8*FH(iF,jF-1,kF-2)+F8*FH(iF,jF+1,kF-2)-FH(iF,jF+2,kF-2));
-             const double t_km1=(FH(iF,jF-2,kF-1)-F8*FH(iF,jF-1,kF-1)+F8*FH(iF,jF+1,kF-1)-FH(iF,jF+2,kF-1));
-             const double t_kp1=(FH(iF,jF-2,kF+1)-F8*FH(iF,jF-1,kF+1)+F8*FH(iF,jF+1,kF+1)-FH(iF,jF+2,kF+1));
-             const double t_kp2=(FH(iF,jF-2,kF+2)-F8*FH(iF,jF-1,kF+2)+F8*FH(iF,jF+1,kF+2)-FH(iF,jF+2,kF+2));
-             fyz[p]=Fdydz*(t_km2-F8*t_km1+F8*t_kp1-t_kp2);}
-        }}}}
-        if(has6){for(int k0=k6_lo;k0<=k6_hi;++k0){const int kF=k0+1;
-        for(int j0=j6_lo;j0<=j6_hi;++j0){const int jF=j0+1;
-        for(int i0=i6_lo;i0<=i6_hi;++i0){const int iF=i0+1;const size_t p=idx_ex(i0,j0,k0,ex);
-            fxx[p]=Xdxdx*(TWO*FH(iF-3,jF,kF)-F27*FH(iF-2,jF,kF)+F270*FH(iF-1,jF,kF)-F490*FH(iF,jF,kF)+F270*FH(iF+1,jF,kF)-F27*FH(iF+2,jF,kF)+TWO*FH(iF+3,jF,kF));
-            fyy[p]=Xdydy*(TWO*FH(iF,jF-3,kF)-F27*FH(iF,jF-2,kF)+F270*FH(iF,jF-1,kF)-F490*FH(iF,jF,kF)+F270*FH(iF,jF+1,kF)-F27*FH(iF,jF+2,kF)+TWO*FH(iF,jF+3,kF));
-            fzz[p]=Xdzdz*(TWO*FH(iF,jF,kF-3)-F27*FH(iF,jF,kF-2)+F270*FH(iF,jF,kF-1)-F490*FH(iF,jF,kF)+F270*FH(iF,jF,kF+1)-F27*FH(iF,jF,kF+2)+TWO*FH(iF,jF,kF+3));
-            #define XS6(JF,KFDUMMY) (-FH(iF-3,JF,KFDUMMY)+F9*FH(iF-2,JF,KFDUMMY)-F45*FH(iF-1,JF,KFDUMMY)+F45*FH(iF+1,JF,KFDUMMY)-F9*FH(iF+2,JF,KFDUMMY)+FH(iF+3,JF,KFDUMMY))
-            fxy[p]=Xdxdy*(-XS6(jF-3,kF)+F9*XS6(jF-2,kF)-F45*XS6(jF-1,kF)+F45*XS6(jF+1,kF)-F9*XS6(jF+2,kF)+XS6(jF+3,kF));
-            fxz[p]=Xdxdz*(-XS6(jF,kF-3)+F9*XS6(jF,kF-2)-F45*XS6(jF,kF-1)+F45*XS6(jF,kF+1)-F9*XS6(jF,kF+2)+XS6(jF,kF+3));
-            #undef XS6
-            #define YS6(JF,KFDUMMY) (-FH(iF,JF-3,KFDUMMY)+F9*FH(iF,JF-2,KFDUMMY)-F45*FH(iF,JF-1,KFDUMMY)+F45*FH(iF,JF+1,KFDUMMY)-F9*FH(iF,JF+2,KFDUMMY)+FH(iF,JF+3,KFDUMMY))
-            fyz[p]=Xdydz*(-YS6(jF,kF-3)+F9*YS6(jF,kF-2)-F45*YS6(jF,kF-1)+F45*YS6(jF,kF+1)-F9*YS6(jF,kF+2)+YS6(jF,kF+3));
-            #undef YS6
-        }}}}
-        #undef FH
-        return;
-    }
-#elif (ghost_width == 5)
-    {
-        /* 8th-order shell second derivatives — inherits 8th-order stencil coeffs from Cartesian */
-        const int ord=4;
-        int iminF=1,jminF=1,kminF=1;
-        if(Symmetry==OCTANT&&fabs(X[0])<dX)iminF=-3;
-        if(Symmetry==OCTANT&&fabs(Y[0])<dY)jminF=-3;
-        if((sst==2||sst==4)&&fabs(Y[0])<dY)jminF=-3;
-
-        const size_t nx=(size_t)ex1+2*ord,ny=(size_t)ex2+2*ord,nz=(size_t)ex3,fh_size=nx*ny*nz;
-        static double *fh_buf=NULL;static size_t cap=0;
-        if(fh_size>cap){free(fh_buf);fh_buf=(double*)aligned_alloc(64,fh_size*sizeof(double));cap=fh_size;}
-        double *fh=fh_buf;if(!fh)return;
-        symmetry_stbd(ord,ex,f,fh,SoA);
-
-        const double Sdxdx=ONE/(dX*dX),Sdydy=ONE/(dY*dY),Sdzdz=ONE/(dZ*dZ);
-        const double Fdxdx=F1o12/(dX*dX),Fdydy=F1o12/(dY*dY),Fdzdz=F1o12/(dZ*dZ);
-        const double Xdxdx=F1o180/(dX*dX),Xdydy=F1o180/(dY*dY),Xdzdz=F1o180/(dZ*dZ);
-        const double Edxdx=F1o5040/(dX*dX),Edydy=F1o5040/(dY*dY),Edzdz=F1o5040/(dZ*dZ);
-        const double Sdxdy=F1o4/(dX*dY),Sdxdz=F1o4/(dX*dZ),Sdydz=F1o4/(dY*dZ);
-        const double Fdxdy=F1o144/(dX*dY),Fdxdz=F1o144/(dX*dZ),Fdydz=F1o144/(dY*dZ);
-        const double Xdxdy=F1o3600/(dX*dY),Xdxdz=F1o3600/(dX*dZ),Xdydz=F1o3600/(dY*dZ);
-        const double Edxdy=F1o705600/(dX*dY),Edxdz=F1o705600/(dX*dZ),Edydz=F1o705600/(dY*dZ);
-        const size_t all=(size_t)ex1*ex2*ex3;
-        for(size_t p=0;p<all;++p){fxx[p]=fyy[p]=fzz[p]=fxy[p]=fxz[p]=fyz[p]=ZEO;}
-
-        const int i2_lo=(iminF>0)?iminF:0,j2_lo=(jminF>0)?jminF:0,k2_lo=1,i2_hi=ex1-2,j2_hi=ex2-2,k2_hi=ex3-2;
-        const int i4_lo=(iminF+1>0)?iminF+1:0,j4_lo=(jminF+1>0)?jminF+1:0,k4_lo=2,i4_hi=ex1-3,j4_hi=ex2-3,k4_hi=ex3-3;
-        const int i6_lo=(iminF+2>0)?iminF+2:0,j6_lo=(jminF+2>0)?jminF+2:0,k6_lo=3,i6_hi=ex1-4,j6_hi=ex2-4,k6_hi=ex3-4;
-        const int i8_lo=(iminF+3>0)?iminF+3:0,j8_lo=(jminF+3>0)?jminF+3:0,k8_lo=4,i8_hi=ex1-5,j8_hi=ex2-5,k8_hi=ex3-5;
-        const int has4=(i4_lo<=i4_hi&&j4_lo<=j4_hi&&k4_lo<=k4_hi),has6=(i6_lo<=i6_hi&&j6_lo<=j6_hi&&k6_lo<=k6_hi),has8=(i8_lo<=i8_hi&&j8_lo<=j8_hi&&k8_lo<=k8_hi);
-        #define FH(iF,jF,kF) fh[idx_fh_stbd(iF,jF,kF,ord,ex)]
-
-        /* 2nd-order pass */
-        if(i2_lo<=i2_hi&&j2_lo<=j2_hi&&k2_lo<=k2_hi){for(int k0=k2_lo;k0<=k2_hi;++k0){const int kF=k0+1;
-        for(int j0=j2_lo;j0<=j2_hi;++j0){const int jF=j0+1;
-        for(int i0=i2_lo;i0<=i2_hi;++i0){bool in4=has4&&i0>=i4_lo&&i0<=i4_hi&&j0>=j4_lo&&j0<=j4_hi&&k0>=k4_lo&&k0<=k4_hi;if(in4)continue;
-            const int iF=i0+1;const size_t p=idx_ex(i0,j0,k0,ex);
-            fxx[p]=Sdxdx*(FH(iF-1,jF,kF)-TWO*FH(iF,jF,kF)+FH(iF+1,jF,kF));
-            fyy[p]=Sdydy*(FH(iF,jF-1,kF)-TWO*FH(iF,jF,kF)+FH(iF,jF+1,kF));
-            fzz[p]=Sdzdz*(FH(iF,jF,kF-1)-TWO*FH(iF,jF,kF)+FH(iF,jF,kF+1));
-            fxy[p]=Sdxdy*(FH(iF-1,jF-1,kF)-FH(iF+1,jF-1,kF)-FH(iF-1,jF+1,kF)+FH(iF+1,jF+1,kF));
-            fxz[p]=Sdxdz*(FH(iF-1,jF,kF-1)-FH(iF+1,jF,kF-1)-FH(iF-1,jF,kF+1)+FH(iF+1,jF,kF+1));
-            fyz[p]=Sdydz*(FH(iF,jF-1,kF-1)-FH(iF,jF+1,kF-1)-FH(iF,jF-1,kF+1)+FH(iF,jF+1,kF+1));
-        }}}}
-        /* 4th-order pass */
-        if(has4){for(int k0=k4_lo;k0<=k4_hi;++k0){const int kF=k0+1;
-        for(int j0=j4_lo;j0<=j4_hi;++j0){const int jF=j0+1;
-        for(int i0=i4_lo;i0<=i4_hi;++i0){bool in6=has6&&i0>=i6_lo&&i0<=i6_hi&&j0>=j6_lo&&j0<=j6_hi&&k0>=k6_lo&&k0<=k6_hi;if(in6)continue;
-            const int iF=i0+1;const size_t p=idx_ex(i0,j0,k0,ex);
-            fxx[p]=Fdxdx*(-FH(iF-2,jF,kF)+F16*FH(iF-1,jF,kF)-F30*FH(iF,jF,kF)-FH(iF+2,jF,kF)+F16*FH(iF+1,jF,kF));
-            fyy[p]=Fdydy*(-FH(iF,jF-2,kF)+F16*FH(iF,jF-1,kF)-F30*FH(iF,jF,kF)-FH(iF,jF+2,kF)+F16*FH(iF,jF+1,kF));
-            fzz[p]=Fdzdz*(-FH(iF,jF,kF-2)+F16*FH(iF,jF,kF-1)-F30*FH(iF,jF,kF)-FH(iF,jF,kF+2)+F16*FH(iF,jF,kF+1));
-            {const double t_jm2=(FH(iF-2,jF-2,kF)-F8*FH(iF-1,jF-2,kF)+F8*FH(iF+1,jF-2,kF)-FH(iF+2,jF-2,kF));
-             const double t_jm1=(FH(iF-2,jF-1,kF)-F8*FH(iF-1,jF-1,kF)+F8*FH(iF+1,jF-1,kF)-FH(iF+2,jF-1,kF));
-             const double t_jp1=(FH(iF-2,jF+1,kF)-F8*FH(iF-1,jF+1,kF)+F8*FH(iF+1,jF+1,kF)-FH(iF+2,jF+1,kF));
-             const double t_jp2=(FH(iF-2,jF+2,kF)-F8*FH(iF-1,jF+2,kF)+F8*FH(iF+1,jF+2,kF)-FH(iF+2,jF+2,kF));
-             fxy[p]=Fdxdy*(t_jm2-F8*t_jm1+F8*t_jp1-t_jp2);}
-            {const double t_km2=(FH(iF-2,jF,kF-2)-F8*FH(iF-1,jF,kF-2)+F8*FH(iF+1,jF,kF-2)-FH(iF+2,jF,kF-2));
-             const double t_km1=(FH(iF-2,jF,kF-1)-F8*FH(iF-1,jF,kF-1)+F8*FH(iF+1,jF,kF-1)-FH(iF+2,jF,kF-1));
-             const double t_kp1=(FH(iF-2,jF,kF+1)-F8*FH(iF-1,jF,kF+1)+F8*FH(iF+1,jF,kF+1)-FH(iF+2,jF,kF+1));
-             const double t_kp2=(FH(iF-2,jF,kF+2)-F8*FH(iF-1,jF,kF+2)+F8*FH(iF+1,jF,kF+2)-FH(iF+2,jF,kF+2));
-             fxz[p]=Fdxdz*(t_km2-F8*t_km1+F8*t_kp1-t_kp2);}
-            {const double t_km2=(FH(iF,jF-2,kF-2)-F8*FH(iF,jF-1,kF-2)+F8*FH(iF,jF+1,kF-2)-FH(iF,jF+2,kF-2));
-             const double t_km1=(FH(iF,jF-2,kF-1)-F8*FH(iF,jF-1,kF-1)+F8*FH(iF,jF+1,kF-1)-FH(iF,jF+2,kF-1));
-             const double t_kp1=(FH(iF,jF-2,kF+1)-F8*FH(iF,jF-1,kF+1)+F8*FH(iF,jF+1,kF+1)-FH(iF,jF+2,kF+1));
-             const double t_kp2=(FH(iF,jF-2,kF+2)-F8*FH(iF,jF-1,kF+2)+F8*FH(iF,jF+1,kF+2)-FH(iF,jF+2,kF+2));
-             fyz[p]=Fdydz*(t_km2-F8*t_km1+F8*t_kp1-t_kp2);}
-        }}}}
-        /* 6th-order pass */
-        if(has6){for(int k0=k6_lo;k0<=k6_hi;++k0){const int kF=k0+1;
-        for(int j0=j6_lo;j0<=j6_hi;++j0){const int jF=j0+1;
-        for(int i0=i6_lo;i0<=i6_hi;++i0){if(has8&&i0>=i8_lo&&i0<=i8_hi&&j0>=j8_lo&&j0<=j8_hi&&k0>=k8_lo&&k0<=k8_hi)continue;
-            const int iF=i0+1;const size_t p=idx_ex(i0,j0,k0,ex);
-            fxx[p]=Xdxdx*(TWO*FH(iF-3,jF,kF)-F27*FH(iF-2,jF,kF)+F270*FH(iF-1,jF,kF)-F490*FH(iF,jF,kF)+F270*FH(iF+1,jF,kF)-F27*FH(iF+2,jF,kF)+TWO*FH(iF+3,jF,kF));
-            fyy[p]=Xdydy*(TWO*FH(iF,jF-3,kF)-F27*FH(iF,jF-2,kF)+F270*FH(iF,jF-1,kF)-F490*FH(iF,jF,kF)+F270*FH(iF,jF+1,kF)-F27*FH(iF,jF+2,kF)+TWO*FH(iF,jF+3,kF));
-            fzz[p]=Xdzdz*(TWO*FH(iF,jF,kF-3)-F27*FH(iF,jF,kF-2)+F270*FH(iF,jF,kF-1)-F490*FH(iF,jF,kF)+F270*FH(iF,jF,kF+1)-F27*FH(iF,jF,kF+2)+TWO*FH(iF,jF,kF+3));
-            #define XS6_8(JF,KFDUMMY) (-FH(iF-3,JF,KFDUMMY)+F9*FH(iF-2,JF,KFDUMMY)-F45*FH(iF-1,JF,KFDUMMY)+F45*FH(iF+1,JF,KFDUMMY)-F9*FH(iF+2,JF,KFDUMMY)+FH(iF+3,JF,KFDUMMY))
-            fxy[p]=Xdxdy*(-XS6_8(jF-3,kF)+F9*XS6_8(jF-2,kF)-F45*XS6_8(jF-1,kF)+F45*XS6_8(jF+1,kF)-F9*XS6_8(jF+2,kF)+XS6_8(jF+3,kF));
-            fxz[p]=Xdxdz*(-XS6_8(jF,kF-3)+F9*XS6_8(jF,kF-2)-F45*XS6_8(jF,kF-1)+F45*XS6_8(jF,kF+1)-F9*XS6_8(jF,kF+2)+XS6_8(jF,kF+3));
-            #undef XS6_8
-            #define YS6_8(JF,KFDUMMY) (-FH(iF,JF-3,KFDUMMY)+F9*FH(iF,JF-2,KFDUMMY)-F45*FH(iF,JF-1,KFDUMMY)+F45*FH(iF,JF+1,KFDUMMY)-F9*FH(iF,JF+2,KFDUMMY)+FH(iF,JF+3,KFDUMMY))
-            fyz[p]=Xdydz*(-YS6_8(jF,kF-3)+F9*YS6_8(jF,kF-2)-F45*YS6_8(jF,kF-1)+F45*YS6_8(jF,kF+1)-F9*YS6_8(jF,kF+2)+YS6_8(jF,kF+3));
-            #undef YS6_8
-        }}}}
-        /* 8th-order pass */
-        if(has8){for(int k0=k8_lo;k0<=k8_hi;++k0){const int kF=k0+1;
-        for(int j0=j8_lo;j0<=j8_hi;++j0){const int jF=j0+1;
-        for(int i0=i8_lo;i0<=i8_hi;++i0){const int iF=i0+1;const size_t p=idx_ex(i0,j0,k0,ex);
-            fxx[p]=Edxdx*(-(double)9*FH(iF-4,jF,kF)+F128*FH(iF-3,jF,kF)-F1008*FH(iF-2,jF,kF)+F8064*FH(iF-1,jF,kF)-F14350*FH(iF,jF,kF)+F8064*FH(iF+1,jF,kF)-F1008*FH(iF+2,jF,kF)+F128*FH(iF+3,jF,kF)-(double)9*FH(iF+4,jF,kF));
-            fyy[p]=Edydy*(-(double)9*FH(iF,jF-4,kF)+F128*FH(iF,jF-3,kF)-F1008*FH(iF,jF-2,kF)+F8064*FH(iF,jF-1,kF)-F14350*FH(iF,jF,kF)+F8064*FH(iF,jF+1,kF)-F1008*FH(iF,jF+2,kF)+F128*FH(iF,jF+3,kF)-(double)9*FH(iF,jF+4,kF));
-            fzz[p]=Edzdz*(-(double)9*FH(iF,jF,kF-4)+F128*FH(iF,jF,kF-3)-F1008*FH(iF,jF,kF-2)+F8064*FH(iF,jF,kF-1)-F14350*FH(iF,jF,kF)+F8064*FH(iF,jF,kF+1)-F1008*FH(iF,jF,kF+2)+F128*FH(iF,jF,kF+3)-(double)9*FH(iF,jF,kF+4));
-            #define XS8(JF,KFDUMMY) (+(double)3*FH(iF-4,JF,KFDUMMY)-F32*FH(iF-3,JF,KFDUMMY)+F168*FH(iF-2,JF,KFDUMMY)-F672*FH(iF-1,JF,KFDUMMY)+F672*FH(iF+1,JF,KFDUMMY)-F168*FH(iF+2,JF,KFDUMMY)+F32*FH(iF+3,JF,KFDUMMY)-(double)3*FH(iF+4,JF,KFDUMMY))
-            fxy[p]=Edxdy*(+(double)3*XS8(jF-4,kF)-F32*XS8(jF-3,kF)+F168*XS8(jF-2,kF)-F672*XS8(jF-1,kF)+F672*XS8(jF+1,kF)-F168*XS8(jF+2,kF)+F32*XS8(jF+3,kF)-(double)3*XS8(jF+4,kF));
-            fxz[p]=Edxdz*(+(double)3*XS8(jF,kF-4)-F32*XS8(jF,kF-3)+F168*XS8(jF,kF-2)-F672*XS8(jF,kF-1)+F672*XS8(jF,kF+1)-F168*XS8(jF,kF+2)+F32*XS8(jF,kF+3)-(double)3*XS8(jF,kF+4));
-            #undef XS8
-            #define YS8(JF,KFDUMMY) (+(double)3*FH(iF,JF-4,KFDUMMY)-F32*FH(iF,JF-3,KFDUMMY)+F168*FH(iF,JF-2,KFDUMMY)-F672*FH(iF,JF-1,KFDUMMY)+F672*FH(iF,JF+1,KFDUMMY)-F168*FH(iF,JF+2,KFDUMMY)+F32*FH(iF,JF+3,KFDUMMY)-(double)3*FH(iF,JF+4,KFDUMMY))
-            fyz[p]=Edydz*(+(double)3*YS8(jF,kF-4)-F32*YS8(jF,kF-3)+F168*YS8(jF,kF-2)-F672*YS8(jF,kF-1)+F672*YS8(jF,kF+1)-F168*YS8(jF,kF+2)+F32*YS8(jF,kF+3)-(double)3*YS8(jF,kF+4));
-            #undef YS8
-        }}}}
-        #undef FH
-        return;
-    }
-#else
-#error "fdderivs_sh_c.C: unsupported ghost_width"
-#endif
-}

AMSS_NCKU_source/fdderivs_shc_c.C

@@ -1,107 +0,0 @@
-#include "macrodef.h"
-#include "share_func.h"
-#include <cstddef>
-
-/* Forward declarations — Fortran-mangled names from shell C kernels */
-extern "C" {
-void fderivs_sh_(const int ex[3], const double *f,
-                 double *fx, double *fy, double *fz,
-                 const double *X, const double *Y, const double *Z,
-                 double SYM1, double SYM2, double SYM3,
-                 int Symmetry, int onoff, int sst);
-
-void fdderivs_sh_(const int ex[3], const double *f,
-                  double *fxx, double *fxy, double *fxz,
-                  double *fyy, double *fyz, double *fzz,
-                  const double *X, const double *Y, const double *Z,
-                  double SYM1, double SYM2, double SYM3,
-                  int Symmetry, int onoff, int sst);
-
-void fdderivs_shc_(int *ex,
-                    double *f,
-                    double *fxx, double *fxy, double *fxz,
-                    double *fyy, double *fyz, double *fzz,
-                    double *crho, double *sigma, double *R,
-                    double &SYM1, double &SYM2, double &SYM3,
-                    int &Symmetry, int &Lev, int &sst,
-                    double *drhodx, double *drhody, double *drhodz,
-                    double *dsigmadx, double *dsigmady, double *dsigmadz,
-                    double *dRdx, double *dRdy, double *dRdz,
-                    double *drhodxx, double *drhodxy, double *drhodxz,
-                    double *drhodyy, double *drhodyz, double *drhodzz,
-                    double *dsigmadxx, double *dsigmadxy, double *dsigmadxz,
-                    double *dsigmadyy, double *dsigmadyz, double *dsigmadzz,
-                    double *dRdxx, double *dRdxy, double *dRdxz,
-                    double *dRdyy, double *dRdyz, double *dRdzz)
-{
-    const int ex3[3] = { ex[0], ex[1], ex[2] };
-    const size_t n = (size_t)ex[0] * (size_t)ex[1] * (size_t)ex[2];
-
-    double *gx  = (double*)malloc(n * sizeof(double));
-    double *gy  = (double*)malloc(n * sizeof(double));
-    double *gz  = (double*)malloc(n * sizeof(double));
-    double *gxx = (double*)malloc(n * sizeof(double));
-    double *gxy = (double*)malloc(n * sizeof(double));
-    double *gxz = (double*)malloc(n * sizeof(double));
-    double *gyy = (double*)malloc(n * sizeof(double));
-    double *gyz = (double*)malloc(n * sizeof(double));
-    double *gzz = (double*)malloc(n * sizeof(double));
-
-    if (!gx||!gy||!gz||!gxx||!gxy||!gxz||!gyy||!gyz||!gzz) {
-        free(gx);free(gy);free(gz);free(gxx);free(gxy);free(gxz);free(gyy);free(gyz);free(gzz);
-        return;
-    }
-
-    fderivs_sh_(ex3, f, gx, gy, gz, crho, sigma, R, SYM1, SYM2, SYM3, Symmetry, Lev, sst);
-    fdderivs_sh_(ex3, f, gxx, gxy, gxz, gyy, gyz, gzz, crho, sigma, R, SYM1, SYM2, SYM3, Symmetry, Lev, sst);
-
-    for (size_t i = 0; i < n; ++i) {
-        const double rx=drhodx[i], ry=drhody[i], rz=drhodz[i];
-        const double sx=dsigmadx[i], sy=dsigmady[i], sz=dsigmadz[i];
-        const double Rx=dRdx[i], Ry=dRdy[i], Rz=dRdz[i];
-        const double rxx=drhodxx[i], rxy=drhodxy[i], rxz=drhodxz[i];
-        const double ryy=drhodyy[i], ryz=drhodyz[i], rzz=drhodzz[i];
-        const double sxx=dsigmadxx[i], sxy=dsigmadxy[i], sxz=dsigmadxz[i];
-        const double syy=dsigmadyy[i], syz=dsigmadyz[i], szz=dsigmadzz[i];
-        const double Rxx=dRdxx[i], Rxy=dRdxy[i], Rxz=dRdxz[i];
-        const double Ryy=dRdyy[i], Ryz=dRdyz[i], Rzz=dRdzz[i];
-
-        const double Gr=gx[i], Gs=gy[i], GR=gz[i];
-        const double Grr=gxx[i], Grs=gxy[i], GrR=gxz[i];
-        const double Gss=gyy[i], GsR=gyz[i], GRR=gzz[i];
-
-        /* fxx */
-        fxx[i] = rx*rx*Grr + sx*sx*Gss + Rx*Rx*GRR
-               + 2.0*(rx*sx*Grs + rx*Rx*GrR + sx*Rx*GsR)
-               + rxx*Gr + sxx*Gs + Rxx*GR;
-
-        /* fxy */
-        fxy[i] = rx*ry*Grr + sx*sy*Gss + Rx*Ry*GRR
-               + rx*sy*Grs + ry*sx*Grs + rx*Ry*GrR + ry*Rx*GrR + sx*Ry*GsR + sy*Rx*GsR
-               + rxy*Gr + sxy*Gs + Rxy*GR;
-
-        /* fxz */
-        fxz[i] = rx*rz*Grr + sx*sz*Gss + Rx*Rz*GRR
-               + rx*sz*Grs + rz*sx*Grs + rx*Rz*GrR + rz*Rx*GrR + sx*Rz*GsR + sz*Rx*GsR
-               + rxz*Gr + sxz*Gs + Rxz*GR;
-
-        /* fyy */
-        fyy[i] = ry*ry*Grr + sy*sy*Gss + Ry*Ry*GRR
-               + 2.0*(ry*sy*Grs + ry*Ry*GrR + sy*Ry*GsR)
-               + ryy*Gr + syy*Gs + Ryy*GR;
-
-        /* fyz */
-        fyz[i] = ry*rz*Grr + sy*sz*Gss + Ry*Rz*GRR
-               + ry*sz*Grs + rz*sy*Grs + ry*Rz*GrR + rz*Ry*GrR + sy*Rz*GsR + sz*Ry*GsR
-               + ryz*Gr + syz*Gs + Ryz*GR;
-
-        /* fzz */
-        fzz[i] = rz*rz*Grr + sz*sz*Gss + Rz*Rz*GRR
-               + 2.0*(rz*sz*Grs + rz*Rz*GrR + sz*Rz*GsR)
-               + rzz*Gr + szz*Gs + Rzz*GR;
-    }
-
-    free(gx);free(gy);free(gz);free(gxx);free(gxy);free(gxz);free(gyy);free(gyz);free(gzz);
-}
-
-} // extern "C"

AMSS_NCKU_source/fderivs_c.C

@@ -1,18 +1,14 @@
-#include "macrodef.h"
 #include "tool.h"
 
 /*
- * C 版 fderivs — first derivatives df/dx, df/dy, df/dz.
+ * C 版 fderivs
  *
- * Finite difference order is selected at compile time via the ghost_width macro
- * (defined in macrodef.fh):
- *   ghost_width = 2 → 2nd-order
- *   ghost_width = 3 → 4th-order
- *   ghost_width = 4 → 6th-order
- *   ghost_width = 5 → 8th-order
+ * Fortran:
+ * subroutine fderivs(ex,f,fx,fy,fz,X,Y,Z,SYM1,SYM2,SYM3,symmetry,onoff)
  *
- * Multi-pass overwrite strategy: compute the widest (lowest-order) stencil first,
- * then overwrite interior regions with progressively higher-order stencils.
+ * 约定：
+ *   f, fx, fy, fz: ex1*ex2*ex3，按 idx_ex 布局
+ *   X: ex1, Y: ex2, Z: ex3
  */
 void fderivs(const int ex[3],
              const double *f,
@@ -21,596 +17,151 @@ void fderivs(const int ex[3],
              double SYM1, double SYM2, double SYM3,
              int Symmetry, int onoff)
 {
-    (void)onoff;
+    (void)onoff; // Fortran 里没用到
 
-    const double ZEO = 0.0, ONE = 1.0, TWO = 2.0, EIT = 8.0;
-    const double F9 = 9.0, F12 = 12.0, F45 = 45.0, F60 = 60.0;
-    const double F32 = 32.0, F168 = 168.0, F672 = 672.0, F840 = 840.0;
+    const double ZEO = 0.0, ONE = 1.0;
+    const double TWO = 2.0, EIT = 8.0;
+    const double F12 = 12.0;
 
-    const int NO_SYMM = 0, EQ_SYMM = 1;
+    const int NO_SYMM = 0, EQ_SYMM = 1; // OCTANT=2 在本子程序里不直接用
 
     const int ex1 = ex[0], ex2 = ex[1], ex3 = ex[2];
+
+    // dX = X(2)-X(1) -> C: X[1]-X[0]
     const double dX = X[1] - X[0];
     const double dY = Y[1] - Y[0];
     const double dZ = Z[1] - Z[0];
 
-    const int imaxF = ex1, jmaxF = ex2, kmaxF = ex3;
+    // Fortran 1-based bounds
+    const int imaxF = ex1;
+    const int jmaxF = ex2;
+    const int kmaxF = ex3;
 
-    const int gw = ghost_width; // compile-time constant
+    int iminF = 1, jminF = 1, kminF = 1;
+    if (Symmetry > NO_SYMM && fabs(Z[0]) < dZ) kminF = -1;
+    if (Symmetry > EQ_SYMM && fabs(X[0]) < dX) iminF = -1;
+    if (Symmetry > EQ_SYMM && fabs(Y[0]) < dY) jminF = -1;
 
-#if (ghost_width == 2)
-    /* ---- 2nd-order ------------------------------------------------------ */
-    {
-        const int ord = 1; // symmetry_bd ord = ghost_width - 1
+    // SoA(1:3) = SYM1,SYM2,SYM3
+    const double SoA[3] = { SYM1, SYM2, SYM3 };
 
-        int iminF = 1, jminF = 1, kminF = 1;
-        if (Symmetry > NO_SYMM && fabs(Z[0]) < dZ) kminF = 0;
-        if (Symmetry > EQ_SYMM && fabs(X[0]) < dX) iminF = 0;
-        if (Symmetry > EQ_SYMM && fabs(Y[0]) < dY) jminF = 0;
+    // fh: (ex1+2)*(ex2+2)*(ex3+2) because ord=2
+    const size_t nx = (size_t)ex1 + 2;
+    const size_t ny = (size_t)ex2 + 2;
+    const size_t nz = (size_t)ex3 + 2;
+    const size_t fh_size = nx * ny * nz;
+    static double *fh = NULL;
+    static size_t cap = 0;
 
-        const double SoA[3] = { SYM1, SYM2, SYM3 };
-
-        const size_t nx = (size_t)ex1 + ord;
-        const size_t ny = (size_t)ex2 + ord;
-        const size_t nz = (size_t)ex3 + ord;
-        const size_t fh_size = nx * ny * nz;
-
-        static double *fh_buf = NULL;
-        static size_t cap = 0;
-        if (fh_size > cap) {
-            free(fh_buf);
-            fh_buf = (double*)aligned_alloc(64, fh_size * sizeof(double));
-            cap = fh_size;
-        }
-        double *fh = fh_buf;
-        if (!fh) return;
-
-        symmetry_bd(ord, ex, f, fh, SoA);
-
-        const double d2dx = ONE / TWO / dX;
-        const double d2dy = ONE / TWO / dY;
-        const double d2dz = ONE / TWO / dZ;
-
-        const size_t all = (size_t)ex1 * (size_t)ex2 * (size_t)ex3;
-        for (size_t p = 0; p < all; ++p) {
-            fx[p] = ZEO; fy[p] = ZEO; fz[p] = ZEO;
-        }
-
-        /* 2nd-order pass: [-1, 0, +1] / (2*dx) */
-        const int i2_lo = (iminF > 0) ? iminF : 0;
-        const int j2_lo = (jminF > 0) ? jminF : 0;
-        const int k2_lo = (kminF > 0) ? kminF : 0;
-        const int i2_hi = ex1 - 2;
-        const int j2_hi = ex2 - 2;
-        const int k2_hi = ex3 - 2;
-
-        if (i2_lo <= i2_hi && j2_lo <= j2_hi && k2_lo <= k2_hi) {
-            for (int k0 = k2_lo; k0 <= k2_hi; ++k0) {
-                const int kF = k0 + 1;
-                for (int j0 = j2_lo; j0 <= j2_hi; ++j0) {
-                    const int jF = j0 + 1;
-                    for (int i0 = i2_lo; i0 <= i2_hi; ++i0) {
-                        const int iF = i0 + 1;
-                        const size_t p = idx_ex(i0, j0, k0, ex);
-
-                        fx[p] = d2dx * (
-                            -fh[idx_fh_F_ord1(iF - 1, jF,     kF,     ex)] +
-                             fh[idx_fh_F_ord1(iF + 1, jF,     kF,     ex)]
-                        );
-
-                        fy[p] = d2dy * (
-                            -fh[idx_fh_F_ord1(iF,     jF - 1, kF,     ex)] +
-                             fh[idx_fh_F_ord1(iF,     jF + 1, kF,     ex)]
-                        );
-
-                        fz[p] = d2dz * (
-                            -fh[idx_fh_F_ord1(iF,     jF,     kF - 1, ex)] +
-                             fh[idx_fh_F_ord1(iF,     jF,     kF + 1, ex)]
-                        );
-                    }
-                }
-            }
-        }
-        return;
+    if (fh_size > cap) {
+        free(fh);
+        fh = (double*)aligned_alloc(64, fh_size * sizeof(double));
+        cap = fh_size;
     }
-#elif (ghost_width == 3)
-    /* ---- 4th-order (original code) ------------------------------------ */
-    {
-        const int ord = 2; // symmetry_bd ord
+    // double *fh = (double*)malloc(fh_size * sizeof(double));
+    if (!fh) return;
 
-        int iminF = 1, jminF = 1, kminF = 1;
-        if (Symmetry > NO_SYMM && fabs(Z[0]) < dZ) kminF = -1;
-        if (Symmetry > EQ_SYMM && fabs(X[0]) < dX) iminF = -1;
-        if (Symmetry > EQ_SYMM && fabs(Y[0]) < dY) jminF = -1;
+    // call symmetry_bd(2,ex,f,fh,SoA)
+    symmetry_bd(2, ex, f, fh, SoA);
 
-        const double SoA[3] = { SYM1, SYM2, SYM3 };
+    const double d12dx = ONE / F12 / dX;
+    const double d12dy = ONE / F12 / dY;
+    const double d12dz = ONE / F12 / dZ;
 
-        const size_t nx = (size_t)ex1 + ord;
-        const size_t ny = (size_t)ex2 + ord;
-        const size_t nz = (size_t)ex3 + ord;
-        const size_t fh_size = nx * ny * nz;
+    const double d2dx  = ONE / TWO / dX;
+    const double d2dy  = ONE / TWO / dY;
+    const double d2dz  = ONE / TWO / dZ;
 
-        static double *fh_buf = NULL;
-        static size_t cap = 0;
-        if (fh_size > cap) {
-            free(fh_buf);
-            fh_buf = (double*)aligned_alloc(64, fh_size * sizeof(double));
-            cap = fh_size;
-        }
-        double *fh = fh_buf;
-        if (!fh) return;
-
-        symmetry_bd(ord, ex, f, fh, SoA);
-
-        const double d12dx = ONE / F12 / dX;
-        const double d12dy = ONE / F12 / dY;
-        const double d12dz = ONE / F12 / dZ;
-        const double d2dx  = ONE / TWO / dX;
-        const double d2dy  = ONE / TWO / dY;
-        const double d2dz  = ONE / TWO / dZ;
-
-        const size_t all = (size_t)ex1 * (size_t)ex2 * (size_t)ex3;
-        for (size_t p = 0; p < all; ++p) {
-            fx[p] = ZEO; fy[p] = ZEO; fz[p] = ZEO;
-        }
-
-        const int i2_lo = (iminF > 0) ? iminF : 0;
-        const int j2_lo = (jminF > 0) ? jminF : 0;
-        const int k2_lo = (kminF > 0) ? kminF : 0;
-        const int i2_hi = ex1 - 2;
-        const int j2_hi = ex2 - 2;
-        const int k2_hi = ex3 - 2;
-
-        const int i4_lo = (iminF + 1 > 0) ? (iminF + 1) : 0;
-        const int j4_lo = (jminF + 1 > 0) ? (jminF + 1) : 0;
-        const int k4_lo = (kminF + 1 > 0) ? (kminF + 1) : 0;
-        const int i4_hi = ex1 - 3;
-        const int j4_hi = ex2 - 3;
-        const int k4_hi = ex3 - 3;
-
-        if (i2_lo <= i2_hi && j2_lo <= j2_hi && k2_lo <= k2_hi) {
-            for (int k0 = k2_lo; k0 <= k2_hi; ++k0) {
-                const int kF = k0 + 1;
-                for (int j0 = j2_lo; j0 <= j2_hi; ++j0) {
-                    const int jF = j0 + 1;
-                    for (int i0 = i2_lo; i0 <= i2_hi; ++i0) {
-                        const int iF = i0 + 1;
-                        const size_t p = idx_ex(i0, j0, k0, ex);
-
-                        fx[p] = d2dx * (
-                            -fh[idx_fh_F_ord2(iF - 1, jF,     kF,     ex)] +
-                             fh[idx_fh_F_ord2(iF + 1, jF,     kF,     ex)]
-                        );
-
-                        fy[p] = d2dy * (
-                            -fh[idx_fh_F_ord2(iF,     jF - 1, kF,     ex)] +
-                             fh[idx_fh_F_ord2(iF,     jF + 1, kF,     ex)]
-                        );
-
-                        fz[p] = d2dz * (
-                            -fh[idx_fh_F_ord2(iF,     jF,     kF - 1, ex)] +
-                             fh[idx_fh_F_ord2(iF,     jF,     kF + 1, ex)]
-                        );
-                    }
-                }
-            }
-        }
-
-        if (i4_lo <= i4_hi && j4_lo <= j4_hi && k4_lo <= k4_hi) {
-            for (int k0 = k4_lo; k0 <= k4_hi; ++k0) {
-                const int kF = k0 + 1;
-                for (int j0 = j4_lo; j0 <= j4_hi; ++j0) {
-                    const int jF = j0 + 1;
-                    for (int i0 = i4_lo; i0 <= i4_hi; ++i0) {
-                        const int iF = i0 + 1;
-                        const size_t p = idx_ex(i0, j0, k0, ex);
-
-                        fx[p] = d12dx * (
-                            fh[idx_fh_F_ord2(iF - 2, jF,     kF,     ex)] -
-                            EIT * fh[idx_fh_F_ord2(iF - 1, jF,     kF,     ex)] +
-                            EIT * fh[idx_fh_F_ord2(iF + 1, jF,     kF,     ex)] -
-                            fh[idx_fh_F_ord2(iF + 2, jF,     kF,     ex)]
-                        );
-
-                        fy[p] = d12dy * (
-                            fh[idx_fh_F_ord2(iF,     jF - 2, kF,     ex)] -
-                            EIT * fh[idx_fh_F_ord2(iF,     jF - 1, kF,     ex)] +
-                            EIT * fh[idx_fh_F_ord2(iF,     jF + 1, kF,     ex)] -
-                            fh[idx_fh_F_ord2(iF,     jF + 2, kF,     ex)]
-                        );
-
-                        fz[p] = d12dz * (
-                            fh[idx_fh_F_ord2(iF,     jF,     kF - 2, ex)] -
-                            EIT * fh[idx_fh_F_ord2(iF,     jF,     kF - 1, ex)] +
-                            EIT * fh[idx_fh_F_ord2(iF,     jF,     kF + 1, ex)] -
-                            fh[idx_fh_F_ord2(iF,     jF,     kF + 2, ex)]
-                        );
-                    }
-                }
-            }
-        }
-        return;
+    // fx = fy = fz = 0
+    const size_t all = (size_t)ex1 * (size_t)ex2 * (size_t)ex3;
+    for (size_t p = 0; p < all; ++p) {
+        fx[p] = ZEO;
+        fy[p] = ZEO;
+        fz[p] = ZEO;
     }
-#elif (ghost_width == 4)
-    /* ---- 6th-order ----------------------------------------------------- */
-    {
-        const int ord = 3;
 
-        int iminF = 1, jminF = 1, kminF = 1;
-        if (Symmetry > NO_SYMM && fabs(Z[0]) < dZ) kminF = -2;
-        if (Symmetry > EQ_SYMM && fabs(X[0]) < dX) iminF = -2;
-        if (Symmetry > EQ_SYMM && fabs(Y[0]) < dY) jminF = -2;
+    /*
+     * 两段式：
+     * 1) 先在二阶可用区域计算二阶模板
+     * 2) 再在高阶可用区域覆盖为四阶模板
+     *
+     * 与原 if/elseif 逻辑等价，但减少逐点分支判断。
+     */
+    const int i2_lo = (iminF > 0) ? iminF : 0;
+    const int j2_lo = (jminF > 0) ? jminF : 0;
+    const int k2_lo = (kminF > 0) ? kminF : 0;
+    const int i2_hi = ex1 - 2;
+    const int j2_hi = ex2 - 2;
+    const int k2_hi = ex3 - 2;
 
-        const double SoA[3] = { SYM1, SYM2, SYM3 };
+    const int i4_lo = (iminF + 1 > 0) ? (iminF + 1) : 0;
+    const int j4_lo = (jminF + 1 > 0) ? (jminF + 1) : 0;
+    const int k4_lo = (kminF + 1 > 0) ? (kminF + 1) : 0;
+    const int i4_hi = ex1 - 3;
+    const int j4_hi = ex2 - 3;
+    const int k4_hi = ex3 - 3;
 
-        const size_t nx = (size_t)ex1 + ord;
-        const size_t ny = (size_t)ex2 + ord;
-        const size_t nz = (size_t)ex3 + ord;
-        const size_t fh_size = nx * ny * nz;
+    if (i2_lo <= i2_hi && j2_lo <= j2_hi && k2_lo <= k2_hi) {
+        for (int k0 = k2_lo; k0 <= k2_hi; ++k0) {
+            const int kF = k0 + 1;
+            for (int j0 = j2_lo; j0 <= j2_hi; ++j0) {
+                const int jF = j0 + 1;
+                for (int i0 = i2_lo; i0 <= i2_hi; ++i0) {
+                    const int iF = i0 + 1;
+                    const size_t p = idx_ex(i0, j0, k0, ex);
 
-        static double *fh_buf = NULL;
-        static size_t cap = 0;
-        if (fh_size > cap) {
-            free(fh_buf);
-            fh_buf = (double*)aligned_alloc(64, fh_size * sizeof(double));
-            cap = fh_size;
-        }
-        double *fh = fh_buf;
-        if (!fh) return;
+                    fx[p] = d2dx * (
+                        -fh[idx_fh_F_ord2(iF - 1, jF,     kF,     ex)] +
+                         fh[idx_fh_F_ord2(iF + 1, jF,     kF,     ex)]
+                    );
 
-        symmetry_bd(ord, ex, f, fh, SoA);
+                    fy[p] = d2dy * (
+                        -fh[idx_fh_F_ord2(iF,     jF - 1, kF,     ex)] +
+                         fh[idx_fh_F_ord2(iF,     jF + 1, kF,     ex)]
+                    );
 
-        /* Denominators */
-        const double d60dx = ONE / F60 / dX;
-        const double d60dy = ONE / F60 / dY;
-        const double d60dz = ONE / F60 / dZ;
-        const double d12dx = ONE / F12 / dX;
-        const double d12dy = ONE / F12 / dY;
-        const double d12dz = ONE / F12 / dZ;
-        const double d2dx  = ONE / TWO / dX;
-        const double d2dy  = ONE / TWO / dY;
-        const double d2dz  = ONE / TWO / dZ;
-
-        const size_t all = (size_t)ex1 * (size_t)ex2 * (size_t)ex3;
-        for (size_t p = 0; p < all; ++p) {
-            fx[p] = ZEO; fy[p] = ZEO; fz[p] = ZEO;
-        }
-
-        /* 2nd-order pass: 3pt, widest */
-        const int i2_lo = (iminF > 0) ? iminF : 0;
-        const int j2_lo = (jminF > 0) ? jminF : 0;
-        const int k2_lo = (kminF > 0) ? kminF : 0;
-        const int i2_hi = ex1 - 2;
-        const int j2_hi = ex2 - 2;
-        const int k2_hi = ex3 - 2;
-
-        /* 4th-order pass: 5pt */
-        const int i4_lo = (iminF + 1 > 0) ? (iminF + 1) : 0;
-        const int j4_lo = (jminF + 1 > 0) ? (jminF + 1) : 0;
-        const int k4_lo = (kminF + 1 > 0) ? (kminF + 1) : 0;
-        const int i4_hi = ex1 - 3;
-        const int j4_hi = ex2 - 3;
-        const int k4_hi = ex3 - 3;
-
-        /* 6th-order pass: 7pt, narrowest interior */
-        const int i6_lo = (iminF + 2 > 0) ? (iminF + 2) : 0;
-        const int j6_lo = (jminF + 2 > 0) ? (jminF + 2) : 0;
-        const int k6_lo = (kminF + 2 > 0) ? (kminF + 2) : 0;
-        const int i6_hi = ex1 - 4;
-        const int j6_hi = ex2 - 4;
-        const int k6_hi = ex3 - 4;
-
-        /* 2nd-order */
-        if (i2_lo <= i2_hi && j2_lo <= j2_hi && k2_lo <= k2_hi) {
-            for (int k0 = k2_lo; k0 <= k2_hi; ++k0) {
-                const int kF = k0 + 1;
-                for (int j0 = j2_lo; j0 <= j2_hi; ++j0) {
-                    const int jF = j0 + 1;
-                    for (int i0 = i2_lo; i0 <= i2_hi; ++i0) {
-                        const int iF = i0 + 1;
-                        const size_t p = idx_ex(i0, j0, k0, ex);
-
-                        fx[p] = d2dx * (
-                            -fh[idx_fh_F(iF - 1, jF,     kF,     ex)] +
-                             fh[idx_fh_F(iF + 1, jF,     kF,     ex)]);
-                        fy[p] = d2dy * (
-                            -fh[idx_fh_F(iF,     jF - 1, kF,     ex)] +
-                             fh[idx_fh_F(iF,     jF + 1, kF,     ex)]);
-                        fz[p] = d2dz * (
-                            -fh[idx_fh_F(iF,     jF,     kF - 1, ex)] +
-                             fh[idx_fh_F(iF,     jF,     kF + 1, ex)]);
-                    }
+                    fz[p] = d2dz * (
+                        -fh[idx_fh_F_ord2(iF,     jF,     kF - 1, ex)] +
+                         fh[idx_fh_F_ord2(iF,     jF,     kF + 1, ex)]
+                    );
                 }
             }
         }
-
-        /* 4th-order overwrite */
-        if (i4_lo <= i4_hi && j4_lo <= j4_hi && k4_lo <= k4_hi) {
-            for (int k0 = k4_lo; k0 <= k4_hi; ++k0) {
-                const int kF = k0 + 1;
-                for (int j0 = j4_lo; j0 <= j4_hi; ++j0) {
-                    const int jF = j0 + 1;
-                    for (int i0 = i4_lo; i0 <= i4_hi; ++i0) {
-                        const int iF = i0 + 1;
-                        const size_t p = idx_ex(i0, j0, k0, ex);
-
-                        fx[p] = d12dx * (
-                            fh[idx_fh_F(iF - 2, jF,     kF,     ex)] -
-                            EIT * fh[idx_fh_F(iF - 1, jF,     kF,     ex)] +
-                            EIT * fh[idx_fh_F(iF + 1, jF,     kF,     ex)] -
-                            fh[idx_fh_F(iF + 2, jF,     kF,     ex)]);
-
-                        fy[p] = d12dy * (
-                            fh[idx_fh_F(iF,     jF - 2, kF,     ex)] -
-                            EIT * fh[idx_fh_F(iF,     jF - 1, kF,     ex)] +
-                            EIT * fh[idx_fh_F(iF,     jF + 1, kF,     ex)] -
-                            fh[idx_fh_F(iF,     jF + 2, kF,     ex)]);
-
-                        fz[p] = d12dz * (
-                            fh[idx_fh_F(iF,     jF,     kF - 2, ex)] -
-                            EIT * fh[idx_fh_F(iF,     jF,     kF - 1, ex)] +
-                            EIT * fh[idx_fh_F(iF,     jF,     kF + 1, ex)] -
-                            fh[idx_fh_F(iF,     jF,     kF + 2, ex)]);
-                    }
-                }
-            }
-        }
-
-        /* 6th-order overwrite: [-1,+9,-45,0,+45,-9,+1] / (60*dx) */
-        if (i6_lo <= i6_hi && j6_lo <= j6_hi && k6_lo <= k6_hi) {
-            for (int k0 = k6_lo; k0 <= k6_hi; ++k0) {
-                const int kF = k0 + 1;
-                for (int j0 = j6_lo; j0 <= j6_hi; ++j0) {
-                    const int jF = j0 + 1;
-                    for (int i0 = i6_lo; i0 <= i6_hi; ++i0) {
-                        const int iF = i0 + 1;
-                        const size_t p = idx_ex(i0, j0, k0, ex);
-
-                        fx[p] = d60dx * (
-                            -fh[idx_fh_F(iF - 3, jF,     kF,     ex)] +
-                            F9 * fh[idx_fh_F(iF - 2, jF,     kF,     ex)] -
-                            F45 * fh[idx_fh_F(iF - 1, jF,     kF,     ex)] +
-                            F45 * fh[idx_fh_F(iF + 1, jF,     kF,     ex)] -
-                            F9 * fh[idx_fh_F(iF + 2, jF,     kF,     ex)] +
-                            fh[idx_fh_F(iF + 3, jF,     kF,     ex)]);
-
-                        fy[p] = d60dy * (
-                            -fh[idx_fh_F(iF,     jF - 3, kF,     ex)] +
-                            F9 * fh[idx_fh_F(iF,     jF - 2, kF,     ex)] -
-                            F45 * fh[idx_fh_F(iF,     jF - 1, kF,     ex)] +
-                            F45 * fh[idx_fh_F(iF,     jF + 1, kF,     ex)] -
-                            F9 * fh[idx_fh_F(iF,     jF + 2, kF,     ex)] +
-                            fh[idx_fh_F(iF,     jF + 3, kF,     ex)]);
-
-                        fz[p] = d60dz * (
-                            -fh[idx_fh_F(iF,     jF,     kF - 3, ex)] +
-                            F9 * fh[idx_fh_F(iF,     jF,     kF - 2, ex)] -
-                            F45 * fh[idx_fh_F(iF,     jF,     kF - 1, ex)] +
-                            F45 * fh[idx_fh_F(iF,     jF,     kF + 1, ex)] -
-                            F9 * fh[idx_fh_F(iF,     jF,     kF + 2, ex)] +
-                            fh[idx_fh_F(iF,     jF,     kF + 3, ex)]);
-                    }
-                }
-            }
-        }
-        return;
     }
-#elif (ghost_width == 5)
-    /* ---- 8th-order ----------------------------------------------------- */
-    {
-        const int ord = 5;
 
-        int iminF = 1, jminF = 1, kminF = 1;
-        if (Symmetry > NO_SYMM && fabs(Z[0]) < dZ) kminF = -3;
-        if (Symmetry > EQ_SYMM && fabs(X[0]) < dX) iminF = -3;
-        if (Symmetry > EQ_SYMM && fabs(Y[0]) < dY) jminF = -3;
+    if (i4_lo <= i4_hi && j4_lo <= j4_hi && k4_lo <= k4_hi) {
+        for (int k0 = k4_lo; k0 <= k4_hi; ++k0) {
+            const int kF = k0 + 1;
+            for (int j0 = j4_lo; j0 <= j4_hi; ++j0) {
+                const int jF = j0 + 1;
+                for (int i0 = i4_lo; i0 <= i4_hi; ++i0) {
+                    const int iF = i0 + 1;
+                    const size_t p = idx_ex(i0, j0, k0, ex);
 
-        const double SoA[3] = { SYM1, SYM2, SYM3 };
+                    fx[p] = d12dx * (
+                        fh[idx_fh_F_ord2(iF - 2, jF,     kF,     ex)] -
+                        EIT * fh[idx_fh_F_ord2(iF - 1, jF,     kF,     ex)] +
+                        EIT * fh[idx_fh_F_ord2(iF + 1, jF,     kF,     ex)] -
+                        fh[idx_fh_F_ord2(iF + 2, jF,     kF,     ex)]
+                    );
 
-        const size_t nx = (size_t)ex1 + ord;
-        const size_t ny = (size_t)ex2 + ord;
-        const size_t nz = (size_t)ex3 + ord;
-        const size_t fh_size = nx * ny * nz;
+                    fy[p] = d12dy * (
+                        fh[idx_fh_F_ord2(iF,     jF - 2, kF,     ex)] -
+                        EIT * fh[idx_fh_F_ord2(iF,     jF - 1, kF,     ex)] +
+                        EIT * fh[idx_fh_F_ord2(iF,     jF + 1, kF,     ex)] -
+                        fh[idx_fh_F_ord2(iF,     jF + 2, kF,     ex)]
+                    );
 
-        static double *fh_buf = NULL;
-        static size_t cap = 0;
-        if (fh_size > cap) {
-            free(fh_buf);
-            fh_buf = (double*)aligned_alloc(64, fh_size * sizeof(double));
-            cap = fh_size;
-        }
-        double *fh = fh_buf;
-        if (!fh) return;
-
-        symmetry_bd(ord, ex, f, fh, SoA);
-
-        const double d840dx = ONE / F840 / dX;
-        const double d840dy = ONE / F840 / dY;
-        const double d840dz = ONE / F840 / dZ;
-        const double d60dx  = ONE / F60  / dX;
-        const double d60dy  = ONE / F60  / dY;
-        const double d60dz  = ONE / F60  / dZ;
-        const double d12dx  = ONE / F12  / dX;
-        const double d12dy  = ONE / F12  / dY;
-        const double d12dz  = ONE / F12  / dZ;
-        const double d2dx   = ONE / TWO  / dX;
-        const double d2dy   = ONE / TWO  / dY;
-        const double d2dz   = ONE / TWO  / dZ;
-
-        const size_t all = (size_t)ex1 * (size_t)ex2 * (size_t)ex3;
-        for (size_t p = 0; p < all; ++p) {
-            fx[p] = ZEO; fy[p] = ZEO; fz[p] = ZEO;
-        }
-
-        /* 2nd: 3pt, widest */
-        const int i2_lo = (iminF > 0) ? iminF : 0;
-        const int j2_lo = (jminF > 0) ? jminF : 0;
-        const int k2_lo = (kminF > 0) ? kminF : 0;
-        const int i2_hi = ex1 - 2;
-        const int j2_hi = ex2 - 2;
-        const int k2_hi = ex3 - 2;
-
-        /* 4th: 5pt */
-        const int i4_lo = (iminF + 1 > 0) ? (iminF + 1) : 0;
-        const int j4_lo = (jminF + 1 > 0) ? (jminF + 1) : 0;
-        const int k4_lo = (kminF + 1 > 0) ? (kminF + 1) : 0;
-        const int i4_hi = ex1 - 3;
-        const int j4_hi = ex2 - 3;
-        const int k4_hi = ex3 - 3;
-
-        /* 6th: 7pt */
-        const int i6_lo = (iminF + 2 > 0) ? (iminF + 2) : 0;
-        const int j6_lo = (jminF + 2 > 0) ? (jminF + 2) : 0;
-        const int k6_lo = (kminF + 2 > 0) ? (kminF + 2) : 0;
-        const int i6_hi = ex1 - 4;
-        const int j6_hi = ex2 - 4;
-        const int k6_hi = ex3 - 4;
-
-        /* 8th: 9pt, narrowest */
-        const int i8_lo = (iminF + 3 > 0) ? (iminF + 3) : 0;
-        const int j8_lo = (jminF + 3 > 0) ? (jminF + 3) : 0;
-        const int k8_lo = (kminF + 3 > 0) ? (kminF + 3) : 0;
-        const int i8_hi = ex1 - 5;
-        const int j8_hi = ex2 - 5;
-        const int k8_hi = ex3 - 5;
-
-        if (i2_lo <= i2_hi && j2_lo <= j2_hi && k2_lo <= k2_hi) {
-            for (int k0 = k2_lo; k0 <= k2_hi; ++k0) {
-                const int kF = k0 + 1;
-                for (int j0 = j2_lo; j0 <= j2_hi; ++j0) {
-                    const int jF = j0 + 1;
-                    for (int i0 = i2_lo; i0 <= i2_hi; ++i0) {
-                        const int iF = i0 + 1;
-                        const size_t p = idx_ex(i0, j0, k0, ex);
-
-                        fx[p] = d2dx * (
-                            -fh[idx_fh_F_ord5(iF - 1, jF,     kF,     ex)] +
-                             fh[idx_fh_F_ord5(iF + 1, jF,     kF,     ex)]);
-                        fy[p] = d2dy * (
-                            -fh[idx_fh_F_ord5(iF,     jF - 1, kF,     ex)] +
-                             fh[idx_fh_F_ord5(iF,     jF + 1, kF,     ex)]);
-                        fz[p] = d2dz * (
-                            -fh[idx_fh_F_ord5(iF,     jF,     kF - 1, ex)] +
-                             fh[idx_fh_F_ord5(iF,     jF,     kF + 1, ex)]);
-                    }
+                    fz[p] = d12dz * (
+                        fh[idx_fh_F_ord2(iF,     jF,     kF - 2, ex)] -
+                        EIT * fh[idx_fh_F_ord2(iF,     jF,     kF - 1, ex)] +
+                        EIT * fh[idx_fh_F_ord2(iF,     jF,     kF + 1, ex)] -
+                        fh[idx_fh_F_ord2(iF,     jF,     kF + 2, ex)]
+                    );
                 }
             }
         }
-
-        if (i4_lo <= i4_hi && j4_lo <= j4_hi && k4_lo <= k4_hi) {
-            for (int k0 = k4_lo; k0 <= k4_hi; ++k0) {
-                const int kF = k0 + 1;
-                for (int j0 = j4_lo; j0 <= j4_hi; ++j0) {
-                    const int jF = j0 + 1;
-                    for (int i0 = i4_lo; i0 <= i4_hi; ++i0) {
-                        const int iF = i0 + 1;
-                        const size_t p = idx_ex(i0, j0, k0, ex);
-
-                        fx[p] = d12dx * (
-                            fh[idx_fh_F_ord5(iF - 2, jF,     kF,     ex)] -
-                            EIT * fh[idx_fh_F_ord5(iF - 1, jF,     kF,     ex)] +
-                            EIT * fh[idx_fh_F_ord5(iF + 1, jF,     kF,     ex)] -
-                            fh[idx_fh_F_ord5(iF + 2, jF,     kF,     ex)]);
-
-                        fy[p] = d12dy * (
-                            fh[idx_fh_F_ord5(iF,     jF - 2, kF,     ex)] -
-                            EIT * fh[idx_fh_F_ord5(iF,     jF - 1, kF,     ex)] +
-                            EIT * fh[idx_fh_F_ord5(iF,     jF + 1, kF,     ex)] -
-                            fh[idx_fh_F_ord5(iF,     jF + 2, kF,     ex)]);
-
-                        fz[p] = d12dz * (
-                            fh[idx_fh_F_ord5(iF,     jF,     kF - 2, ex)] -
-                            EIT * fh[idx_fh_F_ord5(iF,     jF,     kF - 1, ex)] +
-                            EIT * fh[idx_fh_F_ord5(iF,     jF,     kF + 1, ex)] -
-                            fh[idx_fh_F_ord5(iF,     jF,     kF + 2, ex)]);
-                    }
-                }
-            }
-        }
-
-        if (i6_lo <= i6_hi && j6_lo <= j6_hi && k6_lo <= k6_hi) {
-            for (int k0 = k6_lo; k0 <= k6_hi; ++k0) {
-                const int kF = k0 + 1;
-                for (int j0 = j6_lo; j0 <= j6_hi; ++j0) {
-                    const int jF = j0 + 1;
-                    for (int i0 = i6_lo; i0 <= i6_hi; ++i0) {
-                        const int iF = i0 + 1;
-                        const size_t p = idx_ex(i0, j0, k0, ex);
-
-                        fx[p] = d60dx * (
-                            -fh[idx_fh_F_ord5(iF - 3, jF,     kF,     ex)] +
-                            F9 * fh[idx_fh_F_ord5(iF - 2, jF,     kF,     ex)] -
-                            F45 * fh[idx_fh_F_ord5(iF - 1, jF,     kF,     ex)] +
-                            F45 * fh[idx_fh_F_ord5(iF + 1, jF,     kF,     ex)] -
-                            F9 * fh[idx_fh_F_ord5(iF + 2, jF,     kF,     ex)] +
-                            fh[idx_fh_F_ord5(iF + 3, jF,     kF,     ex)]);
-
-                        fy[p] = d60dy * (
-                            -fh[idx_fh_F_ord5(iF,     jF - 3, kF,     ex)] +
-                            F9 * fh[idx_fh_F_ord5(iF,     jF - 2, kF,     ex)] -
-                            F45 * fh[idx_fh_F_ord5(iF,     jF - 1, kF,     ex)] +
-                            F45 * fh[idx_fh_F_ord5(iF,     jF + 1, kF,     ex)] -
-                            F9 * fh[idx_fh_F_ord5(iF,     jF + 2, kF,     ex)] +
-                            fh[idx_fh_F_ord5(iF,     jF + 3, kF,     ex)]);
-
-                        fz[p] = d60dz * (
-                            -fh[idx_fh_F_ord5(iF,     jF,     kF - 3, ex)] +
-                            F9 * fh[idx_fh_F_ord5(iF,     jF,     kF - 2, ex)] -
-                            F45 * fh[idx_fh_F_ord5(iF,     jF,     kF - 1, ex)] +
-                            F45 * fh[idx_fh_F_ord5(iF,     jF,     kF + 1, ex)] -
-                            F9 * fh[idx_fh_F_ord5(iF,     jF,     kF + 2, ex)] +
-                            fh[idx_fh_F_ord5(iF,     jF,     kF + 3, ex)]);
-                    }
-                }
-            }
-        }
-
-        /* 8th-order overwrite: [+3,-32,+168,-672,0,+672,-168,+32,-3] / (840*dx) */
-        if (i8_lo <= i8_hi && j8_lo <= j8_hi && k8_lo <= k8_hi) {
-            for (int k0 = k8_lo; k0 <= k8_hi; ++k0) {
-                const int kF = k0 + 1;
-                for (int j0 = j8_lo; j0 <= j8_hi; ++j0) {
-                    const int jF = j0 + 1;
-                    for (int i0 = i8_lo; i0 <= i8_hi; ++i0) {
-                        const int iF = i0 + 1;
-                        const size_t p = idx_ex(i0, j0, k0, ex);
-
-                        fx[p] = d840dx * (
-                            +(double)3 * fh[idx_fh_F_ord5(iF - 4, jF,     kF,     ex)] -
-                            F32  * fh[idx_fh_F_ord5(iF - 3, jF,     kF,     ex)] +
-                            F168 * fh[idx_fh_F_ord5(iF - 2, jF,     kF,     ex)] -
-                            F672 * fh[idx_fh_F_ord5(iF - 1, jF,     kF,     ex)] +
-                            F672 * fh[idx_fh_F_ord5(iF + 1, jF,     kF,     ex)] -
-                            F168 * fh[idx_fh_F_ord5(iF + 2, jF,     kF,     ex)] +
-                            F32  * fh[idx_fh_F_ord5(iF + 3, jF,     kF,     ex)] -
-                            (double)3 * fh[idx_fh_F_ord5(iF + 4, jF,     kF,     ex)]);
-
-                        fy[p] = d840dy * (
-                            +(double)3 * fh[idx_fh_F_ord5(iF,     jF - 4, kF,     ex)] -
-                            F32  * fh[idx_fh_F_ord5(iF,     jF - 3, kF,     ex)] +
-                            F168 * fh[idx_fh_F_ord5(iF,     jF - 2, kF,     ex)] -
-                            F672 * fh[idx_fh_F_ord5(iF,     jF - 1, kF,     ex)] +
-                            F672 * fh[idx_fh_F_ord5(iF,     jF + 1, kF,     ex)] -
-                            F168 * fh[idx_fh_F_ord5(iF,     jF + 2, kF,     ex)] +
-                            F32  * fh[idx_fh_F_ord5(iF,     jF + 3, kF,     ex)] -
-                            (double)3 * fh[idx_fh_F_ord5(iF,     jF + 4, kF,     ex)]);
-
-                        fz[p] = d840dz * (
-                            +(double)3 * fh[idx_fh_F_ord5(iF,     jF,     kF - 4, ex)] -
-                            F32  * fh[idx_fh_F_ord5(iF,     jF,     kF - 3, ex)] +
-                            F168 * fh[idx_fh_F_ord5(iF,     jF,     kF - 2, ex)] -
-                            F672 * fh[idx_fh_F_ord5(iF,     jF,     kF - 1, ex)] +
-                            F672 * fh[idx_fh_F_ord5(iF,     jF,     kF + 1, ex)] -
-                            F168 * fh[idx_fh_F_ord5(iF,     jF,     kF + 2, ex)] +
-                            F32  * fh[idx_fh_F_ord5(iF,     jF,     kF + 3, ex)] -
-                            (double)3 * fh[idx_fh_F_ord5(iF,     jF,     kF + 4, ex)]);
-                    }
-                }
-            }
-        }
-        return;
     }
-#else
-#error "fderivs_c.C: unsupported ghost_width (must be 2, 3, 4, or 5)"
-#endif
+
+    // free(fh);
 }

AMSS_NCKU_source/fderivs_sh_c.C

@@ -1,234 +0,0 @@
-#include "macrodef.h"
-#include "share_func.h"
-
-/*
- * C 版 fderivs_sh — first derivatives on shell patch in (rho, sigma, R) coords.
- *
- * Same stencil coefficients as Cartesian fderivs, but:
- * - Uses symmetry_stbd (ghost on BOTH sides of x/y, none in z)
- * - fh buffer: (-ord+1:ex+ord) in x/y, (1:ex) in z
- * - SoA is 2-element only (x/y), no z-symmetry
- * - sst parameter (shell surface type, not used in stencil computation)
- */
-extern "C" void fderivs_sh_(const int ex[3],
-                const double *f,
-                double *fx, double *fy, double *fz,
-                const double *X, const double *Y, const double *Z,
-                double SYM1, double SYM2, double SYM3,
-                int Symmetry, int onoff, int sst)
-{
-    (void)SYM3; (void)onoff; (void)sst;
-
-    const double ZEO = 0.0, ONE = 1.0, TWO = 2.0, EIT = 8.0;
-    const double F9 = 9.0, F12 = 12.0, F45 = 45.0, F60 = 60.0;
-    const double F32 = 32.0, F168 = 168.0, F672 = 672.0, F840 = 840.0;
-
-    const int NO_SYMM = 0, EQ_SYMM = 1, OCTANT = 2;
-    const int ex1 = ex[0], ex2 = ex[1], ex3 = ex[2];
-    const double dX = X[1] - X[0];
-    const double dY = Y[1] - Y[0];
-    const double dZ = Z[1] - Z[0];
-    const int imaxF = ex1, jmaxF = ex2, kmaxF = ex3;
-    const double SoA[2] = { SYM1, SYM2 };
-
-#if (ghost_width == 2)
-    {
-        const int ord = 1;
-        int iminF = 1, jminF = 1, kminF = 1;
-        if (Symmetry == OCTANT && fabs(X[0]) < dX) iminF = 0;
-        if (Symmetry == OCTANT && fabs(Y[0]) < dY) jminF = 0;
-        if ((sst==2||sst==4) && fabs(Y[0]) < dY) jminF = 0; // EQ reflection
-
-        const size_t nx = (size_t)ex1 + 2 * ord;
-        const size_t ny = (size_t)ex2 + 2 * ord;
-        const size_t nz = (size_t)ex3;
-        const size_t fh_size = nx * ny * nz;
-        static double *fh_buf = NULL; static size_t cap = 0;
-        if (fh_size > cap) { free(fh_buf); fh_buf = (double*)aligned_alloc(64, fh_size*sizeof(double)); cap = fh_size; }
-        double *fh = fh_buf; if (!fh) return;
-        symmetry_stbd(ord, ex, f, fh, SoA);
-
-        const double d2dx = ONE/TWO/dX, d2dy = ONE/TWO/dY, d2dz = ONE/TWO/dZ;
-        const size_t all = (size_t)ex1*ex2*ex3;
-        for (size_t p=0;p<all;++p) { fx[p]=ZEO; fy[p]=ZEO; fz[p]=ZEO; }
-
-        const int i2_lo=(iminF>0)?iminF:0, j2_lo=(jminF>0)?jminF:0, k2_lo=1;
-        const int i2_hi=ex1-2, j2_hi=ex2-2, k2_hi=ex3-2;
-        if (i2_lo<=i2_hi&&j2_lo<=j2_hi&&k2_lo<=k2_hi) {
-            for (int k0=k2_lo;k0<=k2_hi;++k0) { const int kF=k0+1;
-            for (int j0=j2_lo;j0<=j2_hi;++j0) { const int jF=j0+1;
-            for (int i0=i2_lo;i0<=i2_hi;++i0) { const int iF=i0+1;
-                const size_t p=idx_ex(i0,j0,k0,ex);
-                fx[p]=d2dx*(-fh[idx_fh_stbd(iF-1,jF,kF,ord,ex)]+fh[idx_fh_stbd(iF+1,jF,kF,ord,ex)]);
-                fy[p]=d2dy*(-fh[idx_fh_stbd(iF,jF-1,kF,ord,ex)]+fh[idx_fh_stbd(iF,jF+1,kF,ord,ex)]);
-                fz[p]=d2dz*(-fh[idx_fh_stbd(iF,jF,kF-1,ord,ex)]+fh[idx_fh_stbd(iF,jF,kF+1,ord,ex)]);
-            }}}
-        }
-        return;
-    }
-#elif (ghost_width == 3)
-    {
-        const int ord = 2;
-        int iminF = 1, jminF = 1, kminF = 1;
-        if (Symmetry == OCTANT && fabs(X[0]) < dX) iminF = -1;
-        if (Symmetry == OCTANT && fabs(Y[0]) < dY) jminF = -1;
-        if ((sst==2||sst==4) && fabs(Y[0]) < dY) jminF = -1;
-
-        const size_t nx=(size_t)ex1+2*ord, ny=(size_t)ex2+2*ord, nz=(size_t)ex3;
-        const size_t fh_size=nx*ny*nz;
-        static double *fh_buf=NULL; static size_t cap=0;
-        if (fh_size>cap){free(fh_buf);fh_buf=(double*)aligned_alloc(64,fh_size*sizeof(double));cap=fh_size;}
-        double *fh=fh_buf; if(!fh)return;
-        symmetry_stbd(ord,ex,f,fh,SoA);
-
-        const double d12dx=ONE/F12/dX, d12dy=ONE/F12/dY, d12dz=ONE/F12/dZ;
-        const double d2dx=ONE/TWO/dX, d2dy=ONE/TWO/dY, d2dz=ONE/TWO/dZ;
-        const size_t all=(size_t)ex1*ex2*ex3;
-        for(size_t p=0;p<all;++p){fx[p]=ZEO;fy[p]=ZEO;fz[p]=ZEO;}
-
-        const int i2_lo=(iminF>0)?iminF:0, j2_lo=(jminF>0)?jminF:0, k2_lo=1;
-        const int i2_hi=ex1-2, j2_hi=ex2-2, k2_hi=ex3-2;
-        const int i4_lo=(iminF+1>0)?iminF+1:0, j4_lo=(jminF+1>0)?jminF+1:0, k4_lo=2;
-        const int i4_hi=ex1-3, j4_hi=ex2-3, k4_hi=ex3-3;
-
-        if (i2_lo<=i2_hi&&j2_lo<=j2_hi&&k2_lo<=k2_hi) {
-            for(int k0=k2_lo;k0<=k2_hi;++k0){const int kF=k0+1;
-            for(int j0=j2_lo;j0<=j2_hi;++j0){const int jF=j0+1;
-            for(int i0=i2_lo;i0<=i2_hi;++i0){const int iF=i0+1;
-                const size_t p=idx_ex(i0,j0,k0,ex);
-                fx[p]=d2dx*(-fh[idx_fh_stbd(iF-1,jF,kF,ord,ex)]+fh[idx_fh_stbd(iF+1,jF,kF,ord,ex)]);
-                fy[p]=d2dy*(-fh[idx_fh_stbd(iF,jF-1,kF,ord,ex)]+fh[idx_fh_stbd(iF,jF+1,kF,ord,ex)]);
-                fz[p]=d2dz*(-fh[idx_fh_stbd(iF,jF,kF-1,ord,ex)]+fh[idx_fh_stbd(iF,jF,kF+1,ord,ex)]);
-            }}}
-        }
-        if (i4_lo<=i4_hi&&j4_lo<=j4_hi&&k4_lo<=k4_hi) {
-            for(int k0=k4_lo;k0<=k4_hi;++k0){const int kF=k0+1;
-            for(int j0=j4_lo;j0<=j4_hi;++j0){const int jF=j0+1;
-            for(int i0=i4_lo;i0<=i4_hi;++i0){const int iF=i0+1;
-                const size_t p=idx_ex(i0,j0,k0,ex);
-                fx[p]=d12dx*(fh[idx_fh_stbd(iF-2,jF,kF,ord,ex)]-EIT*fh[idx_fh_stbd(iF-1,jF,kF,ord,ex)]+EIT*fh[idx_fh_stbd(iF+1,jF,kF,ord,ex)]-fh[idx_fh_stbd(iF+2,jF,kF,ord,ex)]);
-                fy[p]=d12dy*(fh[idx_fh_stbd(iF,jF-2,kF,ord,ex)]-EIT*fh[idx_fh_stbd(iF,jF-1,kF,ord,ex)]+EIT*fh[idx_fh_stbd(iF,jF+1,kF,ord,ex)]-fh[idx_fh_stbd(iF,jF+2,kF,ord,ex)]);
-                fz[p]=d12dz*(fh[idx_fh_stbd(iF,jF,kF-2,ord,ex)]-EIT*fh[idx_fh_stbd(iF,jF,kF-1,ord,ex)]+EIT*fh[idx_fh_stbd(iF,jF,kF+1,ord,ex)]-fh[idx_fh_stbd(iF,jF,kF+2,ord,ex)]);
-            }}}
-        }
-        return;
-    }
-#elif (ghost_width == 4)
-    {
-        const int ord = 3;
-        int iminF=1,jminF=1,kminF=1;
-        if(Symmetry==OCTANT&&fabs(X[0])<dX)iminF=-2;
-        if(Symmetry==OCTANT&&fabs(Y[0])<dY)jminF=-2;
-        if((sst==2||sst==4)&&fabs(Y[0])<dY)jminF=-2;
-
-        const size_t nx=(size_t)ex1+2*ord,ny=(size_t)ex2+2*ord,nz=(size_t)ex3;
-        const size_t fh_size=nx*ny*nz;
-        static double *fh_buf=NULL;static size_t cap=0;
-        if(fh_size>cap){free(fh_buf);fh_buf=(double*)aligned_alloc(64,fh_size*sizeof(double));cap=fh_size;}
-        double *fh=fh_buf;if(!fh)return;
-        symmetry_stbd(ord,ex,f,fh,SoA);
-
-        const double d60dx=ONE/F60/dX,d60dy=ONE/F60/dY,d60dz=ONE/F60/dZ;
-        const double d12dx=ONE/F12/dX,d12dy=ONE/F12/dY,d12dz=ONE/F12/dZ;
-        const double d2dx=ONE/TWO/dX,d2dy=ONE/TWO/dY,d2dz=ONE/TWO/dZ;
-        const size_t all=(size_t)ex1*ex2*ex3;
-        for(size_t p=0;p<all;++p){fx[p]=ZEO;fy[p]=ZEO;fz[p]=ZEO;}
-
-        const int i2_lo=(iminF>0)?iminF:0,j2_lo=(jminF>0)?jminF:0,k2_lo=1,i2_hi=ex1-2,j2_hi=ex2-2,k2_hi=ex3-2;
-        const int i4_lo=(iminF+1>0)?iminF+1:0,j4_lo=(jminF+1>0)?jminF+1:0,k4_lo=2,i4_hi=ex1-3,j4_hi=ex2-3,k4_hi=ex3-3;
-        const int i6_lo=(iminF+2>0)?iminF+2:0,j6_lo=(jminF+2>0)?jminF+2:0,k6_lo=3,i6_hi=ex1-4,j6_hi=ex2-4,k6_hi=ex3-4;
-
-        if(i2_lo<=i2_hi&&j2_lo<=j2_hi&&k2_lo<=k2_hi){
-            for(int k0=k2_lo;k0<=k2_hi;++k0){const int kF=k0+1;
-            for(int j0=j2_lo;j0<=j2_hi;++j0){const int jF=j0+1;
-            for(int i0=i2_lo;i0<=i2_hi;++i0){const int iF=i0+1;
-                const size_t p=idx_ex(i0,j0,k0,ex);
-                fx[p]=d2dx*(-fh[idx_fh_stbd(iF-1,jF,kF,ord,ex)]+fh[idx_fh_stbd(iF+1,jF,kF,ord,ex)]);
-                fy[p]=d2dy*(-fh[idx_fh_stbd(iF,jF-1,kF,ord,ex)]+fh[idx_fh_stbd(iF,jF+1,kF,ord,ex)]);
-                fz[p]=d2dz*(-fh[idx_fh_stbd(iF,jF,kF-1,ord,ex)]+fh[idx_fh_stbd(iF,jF,kF+1,ord,ex)]);
-            }}}
-        }
-        if(i4_lo<=i4_hi&&j4_lo<=j4_hi&&k4_lo<=k4_hi){
-            for(int k0=k4_lo;k0<=k4_hi;++k0){const int kF=k0+1;
-            for(int j0=j4_lo;j0<=j4_hi;++j0){const int jF=j0+1;
-            for(int i0=i4_lo;i0<=i4_hi;++i0){const int iF=i0+1;
-                const size_t p=idx_ex(i0,j0,k0,ex);
-                fx[p]=d12dx*(fh[idx_fh_stbd(iF-2,jF,kF,ord,ex)]-EIT*fh[idx_fh_stbd(iF-1,jF,kF,ord,ex)]+EIT*fh[idx_fh_stbd(iF+1,jF,kF,ord,ex)]-fh[idx_fh_stbd(iF+2,jF,kF,ord,ex)]);
-                fy[p]=d12dy*(fh[idx_fh_stbd(iF,jF-2,kF,ord,ex)]-EIT*fh[idx_fh_stbd(iF,jF-1,kF,ord,ex)]+EIT*fh[idx_fh_stbd(iF,jF+1,kF,ord,ex)]-fh[idx_fh_stbd(iF,jF+2,kF,ord,ex)]);
-                fz[p]=d12dz*(fh[idx_fh_stbd(iF,jF,kF-2,ord,ex)]-EIT*fh[idx_fh_stbd(iF,jF,kF-1,ord,ex)]+EIT*fh[idx_fh_stbd(iF,jF,kF+1,ord,ex)]-fh[idx_fh_stbd(iF,jF,kF+2,ord,ex)]);
-            }}}
-        }
-        if(i6_lo<=i6_hi&&j6_lo<=j6_hi&&k6_lo<=k6_hi){
-            for(int k0=k6_lo;k0<=k6_hi;++k0){const int kF=k0+1;
-            for(int j0=j6_lo;j0<=j6_hi;++j0){const int jF=j0+1;
-            for(int i0=i6_lo;i0<=i6_hi;++i0){const int iF=i0+1;
-                const size_t p=idx_ex(i0,j0,k0,ex);
-                fx[p]=d60dx*(-fh[idx_fh_stbd(iF-3,jF,kF,ord,ex)]+F9*fh[idx_fh_stbd(iF-2,jF,kF,ord,ex)]-F45*fh[idx_fh_stbd(iF-1,jF,kF,ord,ex)]+F45*fh[idx_fh_stbd(iF+1,jF,kF,ord,ex)]-F9*fh[idx_fh_stbd(iF+2,jF,kF,ord,ex)]+fh[idx_fh_stbd(iF+3,jF,kF,ord,ex)]);
-                fy[p]=d60dy*(-fh[idx_fh_stbd(iF,jF-3,kF,ord,ex)]+F9*fh[idx_fh_stbd(iF,jF-2,kF,ord,ex)]-F45*fh[idx_fh_stbd(iF,jF-1,kF,ord,ex)]+F45*fh[idx_fh_stbd(iF,jF+1,kF,ord,ex)]-F9*fh[idx_fh_stbd(iF,jF+2,kF,ord,ex)]+fh[idx_fh_stbd(iF,jF+3,kF,ord,ex)]);
-                fz[p]=d60dz*(-fh[idx_fh_stbd(iF,jF,kF-3,ord,ex)]+F9*fh[idx_fh_stbd(iF,jF,kF-2,ord,ex)]-F45*fh[idx_fh_stbd(iF,jF,kF-1,ord,ex)]+F45*fh[idx_fh_stbd(iF,jF,kF+1,ord,ex)]-F9*fh[idx_fh_stbd(iF,jF,kF+2,ord,ex)]+fh[idx_fh_stbd(iF,jF,kF+3,ord,ex)]);
-            }}}
-        }
-        return;
-    }
-#elif (ghost_width == 5)
-    {
-        const int ord = 4;
-        int iminF=1,jminF=1,kminF=1;
-        if(Symmetry==OCTANT&&fabs(X[0])<dX)iminF=-3;
-        if(Symmetry==OCTANT&&fabs(Y[0])<dY)jminF=-3;
-        if((sst==2||sst==4)&&fabs(Y[0])<dY)jminF=-3;
-
-        const size_t nx=(size_t)ex1+2*ord,ny=(size_t)ex2+2*ord,nz=(size_t)ex3;
-        const size_t fh_size=nx*ny*nz;
-        static double *fh_buf=NULL;static size_t cap=0;
-        if(fh_size>cap){free(fh_buf);fh_buf=(double*)aligned_alloc(64,fh_size*sizeof(double));cap=fh_size;}
-        double *fh=fh_buf;if(!fh)return;
-        symmetry_stbd(ord,ex,f,fh,SoA);
-
-        const double d840dx=ONE/F840/dX,d840dy=ONE/F840/dY,d840dz=ONE/F840/dZ;
-        const double d60dx=ONE/F60/dX,d60dy=ONE/F60/dY,d60dz=ONE/F60/dZ;
-        const double d12dx=ONE/F12/dX,d12dy=ONE/F12/dY,d12dz=ONE/F12/dZ;
-        const double d2dx=ONE/TWO/dX,d2dy=ONE/TWO/dY,d2dz=ONE/TWO/dZ;
-        const size_t all=(size_t)ex1*ex2*ex3;
-        for(size_t p=0;p<all;++p){fx[p]=ZEO;fy[p]=ZEO;fz[p]=ZEO;}
-
-        const int i2_lo=(iminF>0)?iminF:0,j2_lo=(jminF>0)?jminF:0,k2_lo=1,i2_hi=ex1-2,j2_hi=ex2-2,k2_hi=ex3-2;
-        const int i4_lo=(iminF+1>0)?iminF+1:0,j4_lo=(jminF+1>0)?jminF+1:0,k4_lo=2,i4_hi=ex1-3,j4_hi=ex2-3,k4_hi=ex3-3;
-        const int i6_lo=(iminF+2>0)?iminF+2:0,j6_lo=(jminF+2>0)?jminF+2:0,k6_lo=3,i6_hi=ex1-4,j6_hi=ex2-4,k6_hi=ex3-4;
-        const int i8_lo=(iminF+3>0)?iminF+3:0,j8_lo=(jminF+3>0)?jminF+3:0,k8_lo=4,i8_hi=ex1-5,j8_hi=ex2-5,k8_hi=ex3-5;
-
-        #define FH_S(iF,jF,kF) fh[idx_fh_stbd(iF,jF,kF,ord,ex)]
-        if(i2_lo<=i2_hi&&j2_lo<=j2_hi&&k2_lo<=k2_hi){for(int k0=k2_lo;k0<=k2_hi;++k0){const int kF=k0+1;
-        for(int j0=j2_lo;j0<=j2_hi;++j0){const int jF=j0+1;
-        for(int i0=i2_lo;i0<=i2_hi;++i0){const int iF=i0+1;const size_t p=idx_ex(i0,j0,k0,ex);
-            fx[p]=d2dx*(-FH_S(iF-1,jF,kF)+FH_S(iF+1,jF,kF));
-            fy[p]=d2dy*(-FH_S(iF,jF-1,kF)+FH_S(iF,jF+1,kF));
-            fz[p]=d2dz*(-FH_S(iF,jF,kF-1)+FH_S(iF,jF,kF+1));}}}}
-
-        if(i4_lo<=i4_hi&&j4_lo<=j4_hi&&k4_lo<=k4_hi){for(int k0=k4_lo;k0<=k4_hi;++k0){const int kF=k0+1;
-        for(int j0=j4_lo;j0<=j4_hi;++j0){const int jF=j0+1;
-        for(int i0=i4_lo;i0<=i4_hi;++i0){const int iF=i0+1;const size_t p=idx_ex(i0,j0,k0,ex);
-            fx[p]=d12dx*(FH_S(iF-2,jF,kF)-EIT*FH_S(iF-1,jF,kF)+EIT*FH_S(iF+1,jF,kF)-FH_S(iF+2,jF,kF));
-            fy[p]=d12dy*(FH_S(iF,jF-2,kF)-EIT*FH_S(iF,jF-1,kF)+EIT*FH_S(iF,jF+1,kF)-FH_S(iF,jF+2,kF));
-            fz[p]=d12dz*(FH_S(iF,jF,kF-2)-EIT*FH_S(iF,jF,kF-1)+EIT*FH_S(iF,jF,kF+1)-FH_S(iF,jF,kF+2));}}}}
-
-        if(i6_lo<=i6_hi&&j6_lo<=j6_hi&&k6_lo<=k6_hi){for(int k0=k6_lo;k0<=k6_hi;++k0){const int kF=k0+1;
-        for(int j0=j6_lo;j0<=j6_hi;++j0){const int jF=j0+1;
-        for(int i0=i6_lo;i0<=i6_hi;++i0){const int iF=i0+1;const size_t p=idx_ex(i0,j0,k0,ex);
-            fx[p]=d60dx*(-FH_S(iF-3,jF,kF)+F9*FH_S(iF-2,jF,kF)-F45*FH_S(iF-1,jF,kF)+F45*FH_S(iF+1,jF,kF)-F9*FH_S(iF+2,jF,kF)+FH_S(iF+3,jF,kF));
-            fy[p]=d60dy*(-FH_S(iF,jF-3,kF)+F9*FH_S(iF,jF-2,kF)-F45*FH_S(iF,jF-1,kF)+F45*FH_S(iF,jF+1,kF)-F9*FH_S(iF,jF+2,kF)+FH_S(iF,jF+3,kF));
-            fz[p]=d60dz*(-FH_S(iF,jF,kF-3)+F9*FH_S(iF,jF,kF-2)-F45*FH_S(iF,jF,kF-1)+F45*FH_S(iF,jF,kF+1)-F9*FH_S(iF,jF,kF+2)+FH_S(iF,jF,kF+3));}}}}
-
-        if(i8_lo<=i8_hi&&j8_lo<=j8_hi&&k8_lo<=k8_hi){for(int k0=k8_lo;k0<=k8_hi;++k0){const int kF=k0+1;
-        for(int j0=j8_lo;j0<=j8_hi;++j0){const int jF=j0+1;
-        for(int i0=i8_lo;i0<=i8_hi;++i0){const int iF=i0+1;const size_t p=idx_ex(i0,j0,k0,ex);
-            fx[p]=d840dx*(+(double)3*FH_S(iF-4,jF,kF)-F32*FH_S(iF-3,jF,kF)+F168*FH_S(iF-2,jF,kF)-F672*FH_S(iF-1,jF,kF)+F672*FH_S(iF+1,jF,kF)-F168*FH_S(iF+2,jF,kF)+F32*FH_S(iF+3,jF,kF)-(double)3*FH_S(iF+4,jF,kF));
-            fy[p]=d840dy*(+(double)3*FH_S(iF,jF-4,kF)-F32*FH_S(iF,jF-3,kF)+F168*FH_S(iF,jF-2,kF)-F672*FH_S(iF,jF-1,kF)+F672*FH_S(iF,jF+1,kF)-F168*FH_S(iF,jF+2,kF)+F32*FH_S(iF,jF+3,kF)-(double)3*FH_S(iF,jF+4,kF));
-            fz[p]=d840dz*(+(double)3*FH_S(iF,jF,kF-4)-F32*FH_S(iF,jF,kF-3)+F168*FH_S(iF,jF,kF-2)-F672*FH_S(iF,jF,kF-1)+F672*FH_S(iF,jF,kF+1)-F168*FH_S(iF,jF,kF+2)+F32*FH_S(iF,jF,kF+3)-(double)3*FH_S(iF,jF,kF+4));}}}}
-        #undef FH_S
-        return;
-    }
-#else
-#error "fderivs_sh_c.C: unsupported ghost_width"
-#endif
-}

AMSS_NCKU_source/fderivs_shc_c.C

@@ -1,54 +0,0 @@
-#include "macrodef.h"
-#include "share_func.h"
-#include <cstddef>
-
-/*
- * fderivs_shc — shell first derivatives converted to Cartesian via chain rule.
- *
- * Calls fderivs_sh internally, then:
- *   fx = drhodx * df/drho + dsigmadx * df/dsigma + dRdx * df/dR
- *   fy = drhody * df/drho + dsigmady * df/dsigma + dRdy * df/dR
- *   fz = drhodz * df/drho + dsigmadz * df/dsigma + dRdz * df/dR
- */
-
-// Forward declaration (defined in fderivs_sh_c.C with extern "C" name fderivs_sh_)
-extern "C" {
-void fderivs_sh_(const int ex[3], const double *f,
-                 double *fx, double *fy, double *fz,
-                 const double *X, const double *Y, const double *Z,
-                 double SYM1, double SYM2, double SYM3,
-                 int Symmetry, int onoff, int sst);
-
-void fderivs_shc_(int *ex,
-                   double *f,
-                   double *fx, double *fy, double *fz,
-                   double *crho, double *sigma, double *R,
-                   double &SYM1, double &SYM2, double &SYM3,
-                   int &Symmetry, int &Lev, int &sst,
-                   double *drhodx, double *drhody, double *drhodz,
-                   double *dsigmadx, double *dsigmady, double *dsigmadz,
-                   double *dRdx, double *dRdy, double *dRdz)
-{
-    const int ex3[3] = { ex[0], ex[1], ex[2] };
-    const size_t n = (size_t)ex[0] * (size_t)ex[1] * (size_t)ex[2];
-
-    // Temporary shell-coordinate derivatives
-    double *gx = (double*)malloc(n * sizeof(double));
-    double *gy = (double*)malloc(n * sizeof(double));
-    double *gz = (double*)malloc(n * sizeof(double));
-    if (!gx || !gy || !gz) { free(gx); free(gy); free(gz); return; }
-
-    // Compute shell-coordinate derivatives
-    fderivs_sh_(ex3, f, gx, gy, gz, crho, sigma, R, SYM1, SYM2, SYM3, Symmetry, Lev, sst);
-
-    // Chain rule to Cartesian
-    for (size_t i = 0; i < n; ++i) {
-        fx[i] = drhodx[i]   * gx[i] + dsigmadx[i] * gy[i] + dRdx[i] * gz[i];
-        fy[i] = drhody[i]   * gx[i] + dsigmady[i] * gy[i] + dRdy[i] * gz[i];
-        fz[i] = drhodz[i]   * gx[i] + dsigmadz[i] * gy[i] + dRdz[i] * gz[i];
-    }
-
-    free(gx); free(gy); free(gz);
-}
-
-} // extern "C"

AMSS_NCKU_source/kodiss_c.C

@@ -1,16 +1,16 @@
-#include "macrodef.h"
 #include "tool.h"
 
 /*
- * C 版 kodis — Kreiss-Oliger numerical dissipation (Cartesian patches).
+ * C 版 kodis
  *
- * The KO operator is (D₊D₋)^r applied to f_rhs with alternating sign (-1)^(r-1).
+ * Fortran signature:
+ * subroutine kodis(ex,X,Y,Z,f,f_rhs,SoA,Symmetry,eps)
  *
- * FD order → r → cof=2^(2r) mapping:
- *   ghost_width=2 (2nd) → r=2, cof=16,  sign=-
- *   ghost_width=3 (4th) → r=3, cof=64,  sign=+
- *   ghost_width=4 (6th) → r=4, cof=256, sign=-
- *   ghost_width=5 (8th) → r=5, cof=1024,sign=+
+ * 约定：
+ *   X: ex1, Y: ex2, Z: ex3
+ *   f, f_rhs: ex1*ex2*ex3 按 idx_ex 布局
+ *   SoA[3]
+ *   eps: double
  */
 void kodis(const int ex[3],
            const double *X, const double *Y, const double *Z,
@@ -18,304 +18,100 @@ void kodis(const int ex[3],
            const double SoA[3],
            int Symmetry, double eps)
 {
-    const double ZEO = 0.0;
+    const double ONE = 1.0, SIX = 6.0, FIT = 15.0, TWT = 20.0;
+    const double cof = 64.0;             // 2^6
+    const int NO_SYMM = 0, OCTANT = 2;
+
     const int ex1 = ex[0], ex2 = ex[1], ex3 = ex[2];
+
+    // Fortran: dX = X(2)-X(1) -> C: X[1]-X[0]
     const double dX = X[1] - X[0];
     const double dY = Y[1] - Y[0];
     const double dZ = Z[1] - Z[0];
+    (void)ONE; // ONE 在原 Fortran 里只是参数，这里不一定用得上
 
-    const int imaxF = ex1, jmaxF = ex2, kmaxF = ex3;
+    // Fortran: imax=ex(1) 等是 1-based 上界
+    const int imaxF = ex1;
+    const int jmaxF = ex2;
+    const int kmaxF = ex3;
 
-#if (ghost_width == 2)
-    /* ---- r=2, cof=16, sign=-, 5pt stencil ----------------------------- */
-    {
-        const int ord = 2;
-        const int r = 2;
-        const double cof = 16.0;
-        const double F4 = 4.0, F6 = 6.0;
-        const int NO_SYMM = 0, EQ_SYMM = 1;
+    // Fortran: imin=jmin=kmin=1，某些对称情况变 -2
+    int iminF = 1, jminF = 1, kminF = 1;
 
-        int iminF = 1, jminF = 1, kminF = 1;
-        if (Symmetry > NO_SYMM && fabs(Z[0]) < dZ) kminF = -1;
-        if (Symmetry > EQ_SYMM && fabs(X[0]) < dX) iminF = -1;
-        if (Symmetry > EQ_SYMM && fabs(Y[0]) < dY) jminF = -1;
+    if (Symmetry > NO_SYMM && fabs(Z[0]) < dZ) kminF = -2;
+    if (Symmetry == OCTANT && fabs(X[0]) < dX) iminF = -2;
+    if (Symmetry == OCTANT && fabs(Y[0]) < dY) jminF = -2;
 
-        const size_t nx = (size_t)ex1 + ord;
-        const size_t ny = (size_t)ex2 + ord;
-        const size_t nz = (size_t)ex3 + ord;
-        const size_t fh_size = nx * ny * nz;
+    // 分配 fh：大小 (ex1+3)*(ex2+3)*(ex3+3)，对应 ord=3
+    const size_t nx = (size_t)ex1 + 3;
+    const size_t ny = (size_t)ex2 + 3;
+    const size_t nz = (size_t)ex3 + 3;
+    const size_t fh_size = nx * ny * nz;
 
-        double *fh = (double*)malloc(fh_size * sizeof(double));
-        if (!fh) return;
+    double *fh = (double*)malloc(fh_size * sizeof(double));
+    if (!fh) return;
 
-        symmetry_bd(ord, ex, f, fh, SoA);
+    // Fortran: call symmetry_bd(3,ex,f,fh,SoA)
+    symmetry_bd(3, ex, f, fh, SoA);
 
-        /* i±2 must be valid: i-2 >= iminF && i+2 <= imaxF
-           C 0-based: i0 >= iminF+1, i0 <= ex1-3 */
-        const int i0_lo = (iminF + 1 > 0) ? (iminF + 1) : 0;
-        const int j0_lo = (jminF + 1 > 0) ? (jminF + 1) : 0;
-        const int k0_lo = (kminF + 1 > 0) ? (kminF + 1) : 0;
-        const int i0_hi = imaxF - 3;
-        const int j0_hi = jmaxF - 3;
-        const int k0_hi = kmaxF - 3;
+    /*
+     * Fortran loops:
+     * do k=1,ex3
+     * do j=1,ex2
+     * do i=1,ex1
+     *
+     * C: k0=0..ex3-1, j0=0..ex2-1, i0=0..ex1-1
+     * 并定义 Fortran index: iF=i0+1, ...
+     */
+    // 收紧循环范围：只遍历满足 iF±3/jF±3/kF±3 条件的内部点
+    // iF-3 >= iminF => iF >= iminF+3 => i0 >= iminF+2 (因为 iF=i0+1)
+    // iF+3 <= imaxF => iF <= imaxF-3 => i0 <= imaxF-4
+    const int i0_lo = (iminF + 2 > 0) ? iminF + 2 : 0;
+    const int j0_lo = (jminF + 2 > 0) ? jminF + 2 : 0;
+    const int k0_lo = (kminF + 2 > 0) ? kminF + 2 : 0;
+    const int i0_hi = imaxF - 4;  // inclusive
+    const int j0_hi = jmaxF - 4;
+    const int k0_hi = kmaxF - 4;
 
-        if (!(i0_lo > i0_hi || j0_lo > j0_hi || k0_lo > k0_hi)) {
-            for (int k0 = k0_lo; k0 <= k0_hi; ++k0) {
-                const int kF = k0 + 1;
-                for (int j0 = j0_lo; j0 <= j0_hi; ++j0) {
-                    const int jF = j0 + 1;
-                    for (int i0 = i0_lo; i0 <= i0_hi; ++i0) {
-                        const int iF = i0 + 1;
-                        const size_t p = idx_ex(i0, j0, k0, ex);
-
-                        const double Dx = (
-                            (fh[idx_fh_F_ord2(iF - 2, jF, kF, ex)] + fh[idx_fh_F_ord2(iF + 2, jF, kF, ex)]) -
-                            F4 * (fh[idx_fh_F_ord2(iF - 1, jF, kF, ex)] + fh[idx_fh_F_ord2(iF + 1, jF, kF, ex)]) +
-                            F6 *  fh[idx_fh_F_ord2(iF,     jF, kF, ex)]
-                        ) / dX;
-
-                        const double Dy = (
-                            (fh[idx_fh_F_ord2(iF, jF - 2, kF, ex)] + fh[idx_fh_F_ord2(iF, jF + 2, kF, ex)]) -
-                            F4 * (fh[idx_fh_F_ord2(iF, jF - 1, kF, ex)] + fh[idx_fh_F_ord2(iF, jF + 1, kF, ex)]) +
-                            F6 *  fh[idx_fh_F_ord2(iF, jF,     kF, ex)]
-                        ) / dY;
-
-                        const double Dz = (
-                            (fh[idx_fh_F_ord2(iF, jF, kF - 2, ex)] + fh[idx_fh_F_ord2(iF, jF, kF + 2, ex)]) -
-                            F4 * (fh[idx_fh_F_ord2(iF, jF, kF - 1, ex)] + fh[idx_fh_F_ord2(iF, jF, kF + 1, ex)]) +
-                            F6 *  fh[idx_fh_F_ord2(iF, jF, kF,     ex)]
-                        ) / dZ;
-
-                        f_rhs[p] -= (eps / cof) * (Dx + Dy + Dz); /* sign=- */
-                    }
-                }
-            }
-        }
+    if (i0_lo > i0_hi || j0_lo > j0_hi || k0_lo > k0_hi) {
         free(fh);
         return;
     }
-#elif (ghost_width == 3)
-    /* ---- r=3, cof=64, sign=+, 7pt stencil (current default) ---------- */
-    {
-        const int ord = 3;
-        const int r = 3;
-        const double cof = 64.0;
-        const double SIX = 6.0, FIT = 15.0, TWT = 20.0;
-        const int NO_SYMM = 0, OCTANT = 2;
 
-        int iminF = 1, jminF = 1, kminF = 1;
-        if (Symmetry > NO_SYMM && fabs(Z[0]) < dZ) kminF = -2;
-        if (Symmetry == OCTANT && fabs(X[0]) < dX) iminF = -2;
-        if (Symmetry == OCTANT && fabs(Y[0]) < dY) jminF = -2;
+    for (int k0 = k0_lo; k0 <= k0_hi; ++k0) {
+        const int kF = k0 + 1;
+        for (int j0 = j0_lo; j0 <= j0_hi; ++j0) {
+            const int jF = j0 + 1;
+            for (int i0 = i0_lo; i0 <= i0_hi; ++i0) {
+                const int iF = i0 + 1;
 
-        const size_t nx = (size_t)ex1 + ord;
-        const size_t ny = (size_t)ex2 + ord;
-        const size_t nz = (size_t)ex3 + ord;
-        const size_t fh_size = nx * ny * nz;
+                    const size_t p = idx_ex(i0, j0, k0, ex);
 
-        double *fh = (double*)malloc(fh_size * sizeof(double));
-        if (!fh) return;
+                    // 三个方向各一份同型的 7 点组合（实际上是对称的 6th-order dissipation/filter 核）
+                    const double Dx_term =
+                        ( (fh[idx_fh_F(iF - 3, jF, kF, ex)] + fh[idx_fh_F(iF + 3, jF, kF, ex)]) -
+                          SIX * (fh[idx_fh_F(iF - 2, jF, kF, ex)] + fh[idx_fh_F(iF + 2, jF, kF, ex)]) +
+                          FIT * (fh[idx_fh_F(iF - 1, jF, kF, ex)] + fh[idx_fh_F(iF + 1, jF, kF, ex)]) -
+                          TWT *  fh[idx_fh_F(iF    , jF, kF, ex)] ) / dX;
 
-        symmetry_bd(ord, ex, f, fh, SoA);
+                    const double Dy_term =
+                        ( (fh[idx_fh_F(iF, jF - 3, kF, ex)] + fh[idx_fh_F(iF, jF + 3, kF, ex)]) -
+                          SIX * (fh[idx_fh_F(iF, jF - 2, kF, ex)] + fh[idx_fh_F(iF, jF + 2, kF, ex)]) +
+                          FIT * (fh[idx_fh_F(iF, jF - 1, kF, ex)] + fh[idx_fh_F(iF, jF + 1, kF, ex)]) -
+                          TWT *  fh[idx_fh_F(iF, jF    , kF, ex)] ) / dY;
 
-        const int i0_lo = (iminF + 2 > 0) ? iminF + 2 : 0;
-        const int j0_lo = (jminF + 2 > 0) ? jminF + 2 : 0;
-        const int k0_lo = (kminF + 2 > 0) ? kminF + 2 : 0;
-        const int i0_hi = imaxF - 4;
-        const int j0_hi = jmaxF - 4;
-        const int k0_hi = kmaxF - 4;
+                    const double Dz_term =
+                        ( (fh[idx_fh_F(iF, jF, kF - 3, ex)] + fh[idx_fh_F(iF, jF, kF + 3, ex)]) -
+                          SIX * (fh[idx_fh_F(iF, jF, kF - 2, ex)] + fh[idx_fh_F(iF, jF, kF + 2, ex)]) +
+                          FIT * (fh[idx_fh_F(iF, jF, kF - 1, ex)] + fh[idx_fh_F(iF, jF, kF + 1, ex)]) -
+                          TWT *  fh[idx_fh_F(iF, jF, kF    , ex)] ) / dZ;
 
-        if (!(i0_lo > i0_hi || j0_lo > j0_hi || k0_lo > k0_hi)) {
-            for (int k0 = k0_lo; k0 <= k0_hi; ++k0) {
-                const int kF = k0 + 1;
-                for (int j0 = j0_lo; j0 <= j0_hi; ++j0) {
-                    const int jF = j0 + 1;
-                    for (int i0 = i0_lo; i0 <= i0_hi; ++i0) {
-                        const int iF = i0 + 1;
-                        const size_t p = idx_ex(i0, j0, k0, ex);
-
-                        const double Dx = (
-                            (fh[idx_fh_F(iF - 3, jF, kF, ex)] + fh[idx_fh_F(iF + 3, jF, kF, ex)]) -
-                            SIX * (fh[idx_fh_F(iF - 2, jF, kF, ex)] + fh[idx_fh_F(iF + 2, jF, kF, ex)]) +
-                            FIT * (fh[idx_fh_F(iF - 1, jF, kF, ex)] + fh[idx_fh_F(iF + 1, jF, kF, ex)]) -
-                            TWT *  fh[idx_fh_F(iF,     jF, kF, ex)]
-                        ) / dX;
-
-                        const double Dy = (
-                            (fh[idx_fh_F(iF, jF - 3, kF, ex)] + fh[idx_fh_F(iF, jF + 3, kF, ex)]) -
-                            SIX * (fh[idx_fh_F(iF, jF - 2, kF, ex)] + fh[idx_fh_F(iF, jF + 2, kF, ex)]) +
-                            FIT * (fh[idx_fh_F(iF, jF - 1, kF, ex)] + fh[idx_fh_F(iF, jF + 1, kF, ex)]) -
-                            TWT *  fh[idx_fh_F(iF, jF,     kF, ex)]
-                        ) / dY;
-
-                        const double Dz = (
-                            (fh[idx_fh_F(iF, jF, kF - 3, ex)] + fh[idx_fh_F(iF, jF, kF + 3, ex)]) -
-                            SIX * (fh[idx_fh_F(iF, jF, kF - 2, ex)] + fh[idx_fh_F(iF, jF, kF + 2, ex)]) +
-                            FIT * (fh[idx_fh_F(iF, jF, kF - 1, ex)] + fh[idx_fh_F(iF, jF, kF + 1, ex)]) -
-                            TWT *  fh[idx_fh_F(iF, jF, kF,     ex)]
-                        ) / dZ;
-
-                        f_rhs[p] += (eps / cof) * (Dx + Dy + Dz); /* sign=+ */
-                    }
-                }
+                    // Fortran:
+                    // f_rhs(i,j,k) = f_rhs(i,j,k) + eps/cof*(Dx_term + Dy_term + Dz_term)
+                    f_rhs[p] += (eps / cof) * (Dx_term + Dy_term + Dz_term);
             }
         }
-        free(fh);
-        return;
     }
-#elif (ghost_width == 4)
-    /* ---- r=4, cof=256, sign=-, 9pt stencil ---------------------------- */
-    {
-        const int ord = 4;
-        const int r = 4;
-        const double cof = 256.0;
-        const double F8 = 8.0, F28 = 28.0, F56 = 56.0, F70 = 70.0;
-        const int NO_SYMM = 0, EQ_SYMM = 1;
 
-        int iminF = 1, jminF = 1, kminF = 1;
-        if (Symmetry > NO_SYMM && fabs(Z[0]) < dZ) kminF = -3;
-        if (Symmetry > EQ_SYMM && fabs(X[0]) < dX) iminF = -3;
-        if (Symmetry > EQ_SYMM && fabs(Y[0]) < dY) jminF = -3;
-
-        const size_t nx = (size_t)ex1 + ord;
-        const size_t ny = (size_t)ex2 + ord;
-        const size_t nz = (size_t)ex3 + ord;
-        const size_t fh_size = nx * ny * nz;
-
-        double *fh = (double*)malloc(fh_size * sizeof(double));
-        if (!fh) return;
-
-        symmetry_bd(ord, ex, f, fh, SoA);
-
-        /* i±4 valid: i-4>=iminF → i0>=iminF+3, i+4<=imaxF → i0<=ex1-5 */
-        const int i0_lo = (iminF + 3 > 0) ? iminF + 3 : 0;
-        const int j0_lo = (jminF + 3 > 0) ? jminF + 3 : 0;
-        const int k0_lo = (kminF + 3 > 0) ? kminF + 3 : 0;
-        const int i0_hi = imaxF - 5;
-        const int j0_hi = jmaxF - 5;
-        const int k0_hi = kmaxF - 5;
-
-        if (!(i0_lo > i0_hi || j0_lo > j0_hi || k0_lo > k0_hi)) {
-            for (int k0 = k0_lo; k0 <= k0_hi; ++k0) {
-                const int kF = k0 + 1;
-                for (int j0 = j0_lo; j0 <= j0_hi; ++j0) {
-                    const int jF = j0 + 1;
-                    for (int i0 = i0_lo; i0 <= i0_hi; ++i0) {
-                        const int iF = i0 + 1;
-                        const size_t p = idx_ex(i0, j0, k0, ex);
-
-                        /* Stencil: [1,-8,28,-56,70,-56,28,-8,1] */
-                        const double Dx = (
-                            (fh[idx_fh_F_ord4(iF - 4, jF, kF, ex)] + fh[idx_fh_F_ord4(iF + 4, jF, kF, ex)]) -
-                            F8 * (fh[idx_fh_F_ord4(iF - 3, jF, kF, ex)] + fh[idx_fh_F_ord4(iF + 3, jF, kF, ex)]) +
-                            F28* (fh[idx_fh_F_ord4(iF - 2, jF, kF, ex)] + fh[idx_fh_F_ord4(iF + 2, jF, kF, ex)]) -
-                            F56* (fh[idx_fh_F_ord4(iF - 1, jF, kF, ex)] + fh[idx_fh_F_ord4(iF + 1, jF, kF, ex)]) +
-                            F70*  fh[idx_fh_F_ord4(iF,     jF, kF, ex)]
-                        ) / dX;
-
-                        const double Dy = (
-                            (fh[idx_fh_F_ord4(iF, jF - 4, kF, ex)] + fh[idx_fh_F_ord4(iF, jF + 4, kF, ex)]) -
-                            F8 * (fh[idx_fh_F_ord4(iF, jF - 3, kF, ex)] + fh[idx_fh_F_ord4(iF, jF + 3, kF, ex)]) +
-                            F28* (fh[idx_fh_F_ord4(iF, jF - 2, kF, ex)] + fh[idx_fh_F_ord4(iF, jF + 2, kF, ex)]) -
-                            F56* (fh[idx_fh_F_ord4(iF, jF - 1, kF, ex)] + fh[idx_fh_F_ord4(iF, jF + 1, kF, ex)]) +
-                            F70*  fh[idx_fh_F_ord4(iF, jF,     kF, ex)]
-                        ) / dY;
-
-                        const double Dz = (
-                            (fh[idx_fh_F_ord4(iF, jF, kF - 4, ex)] + fh[idx_fh_F_ord4(iF, jF, kF + 4, ex)]) -
-                            F8 * (fh[idx_fh_F_ord4(iF, jF, kF - 3, ex)] + fh[idx_fh_F_ord4(iF, jF, kF + 3, ex)]) +
-                            F28* (fh[idx_fh_F_ord4(iF, jF, kF - 2, ex)] + fh[idx_fh_F_ord4(iF, jF, kF + 2, ex)]) -
-                            F56* (fh[idx_fh_F_ord4(iF, jF, kF - 1, ex)] + fh[idx_fh_F_ord4(iF, jF, kF + 1, ex)]) +
-                            F70*  fh[idx_fh_F_ord4(iF, jF, kF,     ex)]
-                        ) / dZ;
-
-                        f_rhs[p] -= (eps / cof) * (Dx + Dy + Dz); /* sign=- */
-                    }
-                }
-            }
-        }
-        free(fh);
-        return;
-    }
-#elif (ghost_width == 5)
-    /* ---- r=5, cof=1024, sign=+, 11pt stencil ------------------------- */
-    {
-        const int ord = 5;
-        const int r = 5;
-        const double cof = 1024.0;
-        const double F10 = 10.0, F45 = 45.0, F120 = 120.0;
-        const double F210 = 210.0, F252 = 252.0;
-        const int NO_SYMM = 0, EQ_SYMM = 1;
-
-        int iminF = 1, jminF = 1, kminF = 1;
-        if (Symmetry > NO_SYMM && fabs(Z[0]) < dZ) kminF = -4;
-        if (Symmetry > EQ_SYMM && fabs(X[0]) < dX) iminF = -4;
-        if (Symmetry > EQ_SYMM && fabs(Y[0]) < dY) jminF = -4;
-
-        const size_t nx = (size_t)ex1 + ord;
-        const size_t ny = (size_t)ex2 + ord;
-        const size_t nz = (size_t)ex3 + ord;
-        const size_t fh_size = nx * ny * nz;
-
-        double *fh = (double*)malloc(fh_size * sizeof(double));
-        if (!fh) return;
-
-        symmetry_bd(ord, ex, f, fh, SoA);
-
-        /* i±5 valid: i0>=iminF+4, i0<=ex1-6 */
-        const int i0_lo = (iminF + 4 > 0) ? iminF + 4 : 0;
-        const int j0_lo = (jminF + 4 > 0) ? jminF + 4 : 0;
-        const int k0_lo = (kminF + 4 > 0) ? kminF + 4 : 0;
-        const int i0_hi = imaxF - 6;
-        const int j0_hi = jmaxF - 6;
-        const int k0_hi = kmaxF - 6;
-
-        if (!(i0_lo > i0_hi || j0_lo > j0_hi || k0_lo > k0_hi)) {
-            for (int k0 = k0_lo; k0 <= k0_hi; ++k0) {
-                const int kF = k0 + 1;
-                for (int j0 = j0_lo; j0 <= j0_hi; ++j0) {
-                    const int jF = j0 + 1;
-                    for (int i0 = i0_lo; i0 <= i0_hi; ++i0) {
-                        const int iF = i0 + 1;
-                        const size_t p = idx_ex(i0, j0, k0, ex);
-
-                        /* Stencil: [1,-10,45,-120,210,-252,210,-120,45,-10,1] */
-                        const double Dx = (
-                            (fh[idx_fh_F_ord5(iF - 5, jF, kF, ex)] + fh[idx_fh_F_ord5(iF + 5, jF, kF, ex)]) -
-                            F10 * (fh[idx_fh_F_ord5(iF - 4, jF, kF, ex)] + fh[idx_fh_F_ord5(iF + 4, jF, kF, ex)]) +
-                            F45 * (fh[idx_fh_F_ord5(iF - 3, jF, kF, ex)] + fh[idx_fh_F_ord5(iF + 3, jF, kF, ex)]) -
-                            F120* (fh[idx_fh_F_ord5(iF - 2, jF, kF, ex)] + fh[idx_fh_F_ord5(iF + 2, jF, kF, ex)]) +
-                            F210* (fh[idx_fh_F_ord5(iF - 1, jF, kF, ex)] + fh[idx_fh_F_ord5(iF + 1, jF, kF, ex)]) -
-                            F252*  fh[idx_fh_F_ord5(iF,     jF, kF, ex)]
-                        ) / dX;
-
-                        const double Dy = (
-                            (fh[idx_fh_F_ord5(iF, jF - 5, kF, ex)] + fh[idx_fh_F_ord5(iF, jF + 5, kF, ex)]) -
-                            F10 * (fh[idx_fh_F_ord5(iF, jF - 4, kF, ex)] + fh[idx_fh_F_ord5(iF, jF + 4, kF, ex)]) +
-                            F45 * (fh[idx_fh_F_ord5(iF, jF - 3, kF, ex)] + fh[idx_fh_F_ord5(iF, jF + 3, kF, ex)]) -
-                            F120* (fh[idx_fh_F_ord5(iF, jF - 2, kF, ex)] + fh[idx_fh_F_ord5(iF, jF + 2, kF, ex)]) +
-                            F210* (fh[idx_fh_F_ord5(iF, jF - 1, kF, ex)] + fh[idx_fh_F_ord5(iF, jF + 1, kF, ex)]) -
-                            F252*  fh[idx_fh_F_ord5(iF, jF,     kF, ex)]
-                        ) / dY;
-
-                        const double Dz = (
-                            (fh[idx_fh_F_ord5(iF, jF, kF - 5, ex)] + fh[idx_fh_F_ord5(iF, jF, kF + 5, ex)]) -
-                            F10 * (fh[idx_fh_F_ord5(iF, jF, kF - 4, ex)] + fh[idx_fh_F_ord5(iF, jF, kF + 4, ex)]) +
-                            F45 * (fh[idx_fh_F_ord5(iF, jF, kF - 3, ex)] + fh[idx_fh_F_ord5(iF, jF, kF + 3, ex)]) -
-                            F120* (fh[idx_fh_F_ord5(iF, jF, kF - 2, ex)] + fh[idx_fh_F_ord5(iF, jF, kF + 2, ex)]) +
-                            F210* (fh[idx_fh_F_ord5(iF, jF, kF - 1, ex)] + fh[idx_fh_F_ord5(iF, jF, kF + 1, ex)]) -
-                            F252*  fh[idx_fh_F_ord5(iF, jF, kF,     ex)]
-                        ) / dZ;
-
-                        f_rhs[p] += (eps / cof) * (Dx + Dy + Dz); /* sign=+ */
-                    }
-                }
-            }
-        }
-        free(fh);
-        return;
-    }
-#else
-#error "kodiss_c.C: unsupported ghost_width (must be 2, 3, 4, or 5)"
-#endif
-}
+    free(fh);
+}

AMSS_NCKU_source/kodiss_sh_c.C

@@ -1,136 +0,0 @@
-#include "macrodef.h"
-#include "share_func.h"
-
-/*
- * kodis_sh — Kreiss-Oliger dissipation on shell patches.
- * Same stencil coefficients as Cartesian kodis. Uses symmetry_stbd.
- */
-extern "C" void kodis_sh_(const int ex[3],
-              const double *X, const double *Y, const double *Z,
-              const double *f, double *f_rhs,
-              const double SoAi[2],
-              int Symmetry, double eps, int sst)
-{
-    (void)sst;
-    const double ZEO=0.0;
-    const int ex1=ex[0], ex2=ex[1], ex3=ex[2];
-    const double dX=X[1]-X[0], dY=Y[1]-Y[0], dZ=Z[1]-Z[0];
-    const int imaxF=ex1, jmaxF=ex2, kmaxF=ex3;
-    const double SoA[2]={SoAi[0],SoAi[1]};
-
-#if (ghost_width == 2)
-    {
-        const int ord=2, r=2;
-        const double cof=16.0, F4=4.0, F6=6.0;
-        const int NO_SYMM=0, OCTANT=2;
-        int iminF=1,jminF=1,kminF=1;
-        if(Symmetry==OCTANT&&fabs(X[0])<dX)iminF=-1;
-        if(Symmetry==OCTANT&&fabs(Y[0])<dY)jminF=-1;
-        if((sst==2||sst==4)&&fabs(Y[0])<dY)jminF=-1;
-
-        const size_t nx=(size_t)ex1+2*ord,ny=(size_t)ex2+2*ord,nz=(size_t)ex3,fh_size=nx*ny*nz;
-        double *fh=(double*)malloc(fh_size*sizeof(double));if(!fh)return;
-        symmetry_stbd(ord,ex,f,fh,SoA);
-
-        const int i0_lo=(iminF+1>0)?iminF+1:0,j0_lo=(jminF+1>0)?jminF+1:0,k0_lo=2;
-        const int i0_hi=imaxF-3,j0_hi=jmaxF-3,k0_hi=kmaxF-3;
-        if(!(i0_lo>i0_hi||j0_lo>j0_hi||k0_lo>k0_hi)){
-            for(int k0=k0_lo;k0<=k0_hi;++k0){const int kF=k0+1;
-            for(int j0=j0_lo;j0<=j0_hi;++j0){const int jF=j0+1;
-            for(int i0=i0_lo;i0<=i0_hi;++i0){const int iF=i0+1;const size_t p=idx_ex(i0,j0,k0,ex);
-                const double Dx=((fh[idx_fh_stbd(iF-2,jF,kF,ord,ex)]+fh[idx_fh_stbd(iF+2,jF,kF,ord,ex)])-F4*(fh[idx_fh_stbd(iF-1,jF,kF,ord,ex)]+fh[idx_fh_stbd(iF+1,jF,kF,ord,ex)])+F6*fh[idx_fh_stbd(iF,jF,kF,ord,ex)])/dX;
-                const double Dy=((fh[idx_fh_stbd(iF,jF-2,kF,ord,ex)]+fh[idx_fh_stbd(iF,jF+2,kF,ord,ex)])-F4*(fh[idx_fh_stbd(iF,jF-1,kF,ord,ex)]+fh[idx_fh_stbd(iF,jF+1,kF,ord,ex)])+F6*fh[idx_fh_stbd(iF,jF,kF,ord,ex)])/dY;
-                const double Dz=((fh[idx_fh_stbd(iF,jF,kF-2,ord,ex)]+fh[idx_fh_stbd(iF,jF,kF+2,ord,ex)])-F4*(fh[idx_fh_stbd(iF,jF,kF-1,ord,ex)]+fh[idx_fh_stbd(iF,jF,kF+1,ord,ex)])+F6*fh[idx_fh_stbd(iF,jF,kF,ord,ex)])/dZ;
-                f_rhs[p]-=(eps/cof)*(Dx+Dy+Dz);
-            }}}
-        }
-        free(fh);return;
-    }
-#elif (ghost_width == 3)
-    {
-        const int ord=3, r=3;
-        const double cof=64.0,SIX=6.0,FIT=15.0,TWT=20.0;
-        const int NO_SYMM=0,OCTANT=2;
-        int iminF=1,jminF=1,kminF=1;
-        if(Symmetry==OCTANT&&fabs(X[0])<dX)iminF=-2;
-        if(Symmetry==OCTANT&&fabs(Y[0])<dY)jminF=-2;
-        if((sst==2||sst==4)&&fabs(Y[0])<dY)jminF=-2;
-
-        const size_t nx=(size_t)ex1+2*ord,ny=(size_t)ex2+2*ord,nz=(size_t)ex3,fh_size=nx*ny*nz;
-        double *fh=(double*)malloc(fh_size*sizeof(double));if(!fh)return;
-        symmetry_stbd(ord,ex,f,fh,SoA);
-
-        const int i0_lo=(iminF+2>0)?iminF+2:0,j0_lo=(jminF+2>0)?jminF+2:0,k0_lo=3;
-        const int i0_hi=imaxF-4,j0_hi=jmaxF-4,k0_hi=kmaxF-4;
-        if(!(i0_lo>i0_hi||j0_lo>j0_hi||k0_lo>k0_hi)){
-            for(int k0=k0_lo;k0<=k0_hi;++k0){const int kF=k0+1;
-            for(int j0=j0_lo;j0<=j0_hi;++j0){const int jF=j0+1;
-            for(int i0=i0_lo;i0<=i0_hi;++i0){const int iF=i0+1;const size_t p=idx_ex(i0,j0,k0,ex);
-                const double Dx=((fh[idx_fh_stbd(iF-3,jF,kF,ord,ex)]+fh[idx_fh_stbd(iF+3,jF,kF,ord,ex)])-SIX*(fh[idx_fh_stbd(iF-2,jF,kF,ord,ex)]+fh[idx_fh_stbd(iF+2,jF,kF,ord,ex)])+FIT*(fh[idx_fh_stbd(iF-1,jF,kF,ord,ex)]+fh[idx_fh_stbd(iF+1,jF,kF,ord,ex)])-TWT*fh[idx_fh_stbd(iF,jF,kF,ord,ex)])/dX;
-                const double Dy=((fh[idx_fh_stbd(iF,jF-3,kF,ord,ex)]+fh[idx_fh_stbd(iF,jF+3,kF,ord,ex)])-SIX*(fh[idx_fh_stbd(iF,jF-2,kF,ord,ex)]+fh[idx_fh_stbd(iF,jF+2,kF,ord,ex)])+FIT*(fh[idx_fh_stbd(iF,jF-1,kF,ord,ex)]+fh[idx_fh_stbd(iF,jF+1,kF,ord,ex)])-TWT*fh[idx_fh_stbd(iF,jF,kF,ord,ex)])/dY;
-                const double Dz=((fh[idx_fh_stbd(iF,jF,kF-3,ord,ex)]+fh[idx_fh_stbd(iF,jF,kF+3,ord,ex)])-SIX*(fh[idx_fh_stbd(iF,jF,kF-2,ord,ex)]+fh[idx_fh_stbd(iF,jF,kF+2,ord,ex)])+FIT*(fh[idx_fh_stbd(iF,jF,kF-1,ord,ex)]+fh[idx_fh_stbd(iF,jF,kF+1,ord,ex)])-TWT*fh[idx_fh_stbd(iF,jF,kF,ord,ex)])/dZ;
-                f_rhs[p]+=(eps/cof)*(Dx+Dy+Dz);
-            }}}
-        }
-        free(fh);return;
-    }
-#elif (ghost_width == 4)
-    {
-        const int ord=4, r=4;
-        const double cof=256.0,F8=8.0,F28=28.0,F56=56.0,F70=70.0;
-        const int NO_SYMM=0,OCTANT=2;
-        int iminF=1,jminF=1,kminF=1;
-        if(Symmetry==OCTANT&&fabs(X[0])<dX)iminF=-3;
-        if(Symmetry==OCTANT&&fabs(Y[0])<dY)jminF=-3;
-        if((sst==2||sst==4)&&fabs(Y[0])<dY)jminF=-3;
-
-        const size_t nx=(size_t)ex1+2*ord,ny=(size_t)ex2+2*ord,nz=(size_t)ex3,fh_size=nx*ny*nz;
-        double *fh=(double*)malloc(fh_size*sizeof(double));if(!fh)return;
-        symmetry_stbd(ord,ex,f,fh,SoA);
-
-        const int i0_lo=(iminF+3>0)?iminF+3:0,j0_lo=(jminF+3>0)?jminF+3:0,k0_lo=4;
-        const int i0_hi=imaxF-5,j0_hi=jmaxF-5,k0_hi=kmaxF-5;
-        if(!(i0_lo>i0_hi||j0_lo>j0_hi||k0_lo>k0_hi)){
-            for(int k0=k0_lo;k0<=k0_hi;++k0){const int kF=k0+1;
-            for(int j0=j0_lo;j0<=j0_hi;++j0){const int jF=j0+1;
-            for(int i0=i0_lo;i0<=i0_hi;++i0){const int iF=i0+1;const size_t p=idx_ex(i0,j0,k0,ex);
-                const double Dx=((fh[idx_fh_stbd(iF-4,jF,kF,ord,ex)]+fh[idx_fh_stbd(iF+4,jF,kF,ord,ex)])-F8*(fh[idx_fh_stbd(iF-3,jF,kF,ord,ex)]+fh[idx_fh_stbd(iF+3,jF,kF,ord,ex)])+F28*(fh[idx_fh_stbd(iF-2,jF,kF,ord,ex)]+fh[idx_fh_stbd(iF+2,jF,kF,ord,ex)])-F56*(fh[idx_fh_stbd(iF-1,jF,kF,ord,ex)]+fh[idx_fh_stbd(iF+1,jF,kF,ord,ex)])+F70*fh[idx_fh_stbd(iF,jF,kF,ord,ex)])/dX;
-                const double Dy=((fh[idx_fh_stbd(iF,jF-4,kF,ord,ex)]+fh[idx_fh_stbd(iF,jF+4,kF,ord,ex)])-F8*(fh[idx_fh_stbd(iF,jF-3,kF,ord,ex)]+fh[idx_fh_stbd(iF,jF+3,kF,ord,ex)])+F28*(fh[idx_fh_stbd(iF,jF-2,kF,ord,ex)]+fh[idx_fh_stbd(iF,jF+2,kF,ord,ex)])-F56*(fh[idx_fh_stbd(iF,jF-1,kF,ord,ex)]+fh[idx_fh_stbd(iF,jF+1,kF,ord,ex)])+F70*fh[idx_fh_stbd(iF,jF,kF,ord,ex)])/dY;
-                const double Dz=((fh[idx_fh_stbd(iF,jF,kF-4,ord,ex)]+fh[idx_fh_stbd(iF,jF,kF+4,ord,ex)])-F8*(fh[idx_fh_stbd(iF,jF,kF-3,ord,ex)]+fh[idx_fh_stbd(iF,jF,kF+3,ord,ex)])+F28*(fh[idx_fh_stbd(iF,jF,kF-2,ord,ex)]+fh[idx_fh_stbd(iF,jF,kF+2,ord,ex)])-F56*(fh[idx_fh_stbd(iF,jF,kF-1,ord,ex)]+fh[idx_fh_stbd(iF,jF,kF+1,ord,ex)])+F70*fh[idx_fh_stbd(iF,jF,kF,ord,ex)])/dZ;
-                f_rhs[p]-=(eps/cof)*(Dx+Dy+Dz);
-            }}}
-        }
-        free(fh);return;
-    }
-#elif (ghost_width == 5)
-    {
-        const int ord=5, r=5;
-        const double cof=1024.0,F10=10.0,F45k=45.0,F120=120.0,F210=210.0,F252=252.0;
-        const int NO_SYMM=0,OCTANT=2;
-        int iminF=1,jminF=1,kminF=1;
-        if(Symmetry==OCTANT&&fabs(X[0])<dX)iminF=-4;
-        if(Symmetry==OCTANT&&fabs(Y[0])<dY)jminF=-4;
-        if((sst==2||sst==4)&&fabs(Y[0])<dY)jminF=-4;
-
-        const size_t nx=(size_t)ex1+2*ord,ny=(size_t)ex2+2*ord,nz=(size_t)ex3,fh_size=nx*ny*nz;
-        double *fh=(double*)malloc(fh_size*sizeof(double));if(!fh)return;
-        symmetry_stbd(ord,ex,f,fh,SoA);
-
-        const int i0_lo=(iminF+4>0)?iminF+4:0,j0_lo=(jminF+4>0)?jminF+4:0,k0_lo=5;
-        const int i0_hi=imaxF-6,j0_hi=jmaxF-6,k0_hi=kmaxF-6;
-        if(!(i0_lo>i0_hi||j0_lo>j0_hi||k0_lo>k0_hi)){
-            for(int k0=k0_lo;k0<=k0_hi;++k0){const int kF=k0+1;
-            for(int j0=j0_lo;j0<=j0_hi;++j0){const int jF=j0+1;
-            for(int i0=i0_lo;i0<=i0_hi;++i0){const int iF=i0+1;const size_t p=idx_ex(i0,j0,k0,ex);
-                const double Dx=((fh[idx_fh_stbd(iF-5,jF,kF,ord,ex)]+fh[idx_fh_stbd(iF+5,jF,kF,ord,ex)])-F10*(fh[idx_fh_stbd(iF-4,jF,kF,ord,ex)]+fh[idx_fh_stbd(iF+4,jF,kF,ord,ex)])+F45k*(fh[idx_fh_stbd(iF-3,jF,kF,ord,ex)]+fh[idx_fh_stbd(iF+3,jF,kF,ord,ex)])-F120*(fh[idx_fh_stbd(iF-2,jF,kF,ord,ex)]+fh[idx_fh_stbd(iF+2,jF,kF,ord,ex)])+F210*(fh[idx_fh_stbd(iF-1,jF,kF,ord,ex)]+fh[idx_fh_stbd(iF+1,jF,kF,ord,ex)])-F252*fh[idx_fh_stbd(iF,jF,kF,ord,ex)])/dX;
-                const double Dy=((fh[idx_fh_stbd(iF,jF-5,kF,ord,ex)]+fh[idx_fh_stbd(iF,jF+5,kF,ord,ex)])-F10*(fh[idx_fh_stbd(iF,jF-4,kF,ord,ex)]+fh[idx_fh_stbd(iF,jF+4,kF,ord,ex)])+F45k*(fh[idx_fh_stbd(iF,jF-3,kF,ord,ex)]+fh[idx_fh_stbd(iF,jF+3,kF,ord,ex)])-F120*(fh[idx_fh_stbd(iF,jF-2,kF,ord,ex)]+fh[idx_fh_stbd(iF,jF+2,kF,ord,ex)])+F210*(fh[idx_fh_stbd(iF,jF-1,kF,ord,ex)]+fh[idx_fh_stbd(iF,jF+1,kF,ord,ex)])-F252*fh[idx_fh_stbd(iF,jF,kF,ord,ex)])/dY;
-                const double Dz=((fh[idx_fh_stbd(iF,jF,kF-5,ord,ex)]+fh[idx_fh_stbd(iF,jF,kF+5,ord,ex)])-F10*(fh[idx_fh_stbd(iF,jF,kF-4,ord,ex)]+fh[idx_fh_stbd(iF,jF,kF+4,ord,ex)])+F45k*(fh[idx_fh_stbd(iF,jF,kF-3,ord,ex)]+fh[idx_fh_stbd(iF,jF,kF+3,ord,ex)])-F120*(fh[idx_fh_stbd(iF,jF,kF-2,ord,ex)]+fh[idx_fh_stbd(iF,jF,kF+2,ord,ex)])+F210*(fh[idx_fh_stbd(iF,jF,kF-1,ord,ex)]+fh[idx_fh_stbd(iF,jF,kF+1,ord,ex)])-F252*fh[idx_fh_stbd(iF,jF,kF,ord,ex)])/dZ;
-                f_rhs[p]+=(eps/cof)*(Dx+Dy+Dz);
-            }}}
-        }
-        free(fh);return;
-    }
-#else
-#error "kodiss_sh_c.C: unsupported ghost_width"
-#endif
-}

AMSS_NCKU_source/lopsided_c.C

@@ -1,13 +1,14 @@
-#include "macrodef.h"
 #include "tool.h"
-
 /*
- * C 版 lopsided — upwind (lopsided) advection derivatives.
+ * 你需要提供 symmetry_bd 的 C 版本（或 Fortran 绑到 C 的接口）。
+ * Fortran: call symmetry_bd(3,ex,f,fh,SoA)
  *
- * Adds advection terms to f_rhs for all three spatial directions.
- * Uses sign-biased (one-sided) stencils with centered fallbacks.
- *
- * For lopsided, symmetry_bd ord = ghost_width (same as kodiss).
+ * 约定：
+ *   nghost = 3
+ *   ex[3]  = {ex1,ex2,ex3}
+ *   f      = 原始网格 (ex1*ex2*ex3)
+ *   fh     = 扩展网格 ((ex1+3)*(ex2+3)*(ex3+3))，对应 Fortran 的 (-2:ex1, ...)
+ *   SoA[3] = 输入参数
  */
 void lopsided(const int ex[3],
               const double *X, const double *Y, const double *Z,
@@ -15,577 +16,240 @@ void lopsided(const int ex[3],
               const double *Sfx, const double *Sfy, const double *Sfz,
               int Symmetry, const double SoA[3])
 {
-    const double ZEO = 0.0, ONE = 1.0;
-    const double TWO = 2.0, F6 = 6.0, EIT = 8.0;
-    const double F3 = 3.0, F4 = 4.0, F5 = 5.0, F10 = 10.0, F12 = 12.0, F18 = 18.0;
-    const double F9 = 9.0, F45 = 45.0, F60 = 60.0;
-    const double F2 = 2.0, F15 = 15.0, F24 = 24.0, F30 = 30.0, F35 = 35.0;
-    const double F50 = 50.0, F77 = 77.0, F80 = 80.0, F100 = 100.0, F150 = 150.0;
-    const double F32 = 32.0, F168 = 168.0, F672 = 672.0, F840 = 840.0;
-    const double F140=140.0, F378=378.0, F420=420.0, F1050=1050.0;
+    const double ZEO = 0.0, ONE = 1.0, F3 = 3.0;
+    const double TWO = 2.0, F6 = 6.0, F18 = 18.0;
+    const double F12 = 12.0, F10 = 10.0, EIT = 8.0;
+
+    const int NO_SYMM = 0, EQ_SYMM = 1, OCTANT = 2;
+    (void)OCTANT; // 这里和 Fortran 一样只是定义了不用也没关系
 
-    const int NO_SYMM = 0, EQ_SYMM = 1;
     const int ex1 = ex[0], ex2 = ex[1], ex3 = ex[2];
 
+    // 对应 Fortran: dX = X(2)-X(1)  （Fortran 1-based）
+    // C: X[1]-X[0]
     const double dX = X[1] - X[0];
     const double dY = Y[1] - Y[0];
     const double dZ = Z[1] - Z[0];
 
-#if (ghost_width == 2)
-    /* ---- 2nd-order lopsided --------------------------------------------- */
-    {
-        const int ord = 2;
-        int iminF = 1, jminF = 1, kminF = 1;
-        if (Symmetry > NO_SYMM && fabs(Z[0]) < dZ) kminF = -1;
-        if (Symmetry > EQ_SYMM && fabs(X[0]) < dX) iminF = -1;
-        if (Symmetry > EQ_SYMM && fabs(Y[0]) < dY) jminF = -1;
+    const double d12dx = ONE / F12 / dX;
+    const double d12dy = ONE / F12 / dY;
+    const double d12dz = ONE / F12 / dZ;
 
-        const size_t nx = (size_t)ex1 + ord;
-        const size_t ny = (size_t)ex2 + ord;
-        const size_t nz = (size_t)ex3 + ord;
-        const size_t fh_size = nx * ny * nz;
+    // Fortran 里算了 d2dx/d2dy/d2dz 但本 subroutine 里没用到（保持一致也算出来）
+    const double d2dx  = ONE / TWO / dX;
+    const double d2dy  = ONE / TWO / dY;
+    const double d2dz  = ONE / TWO / dZ;
+    (void)d2dx; (void)d2dy; (void)d2dz;
 
-        double *fh = (double*)malloc(fh_size * sizeof(double));
-        if (!fh) return;
-        symmetry_bd(ord, ex, f, fh, SoA);
+    // Fortran:
+    // imax = ex(1); jmax = ex(2); kmax = ex(3)
+    const int imaxF = ex1;
+    const int jmaxF = ex2;
+    const int kmaxF = ex3;
 
-        const double d2dx  = ONE / TWO / dX;
-        const double d2dy  = ONE / TWO / dY;
-        const double d2dz  = ONE / TWO / dZ;
+    // Fortran:
+    // imin=jmin=kmin=1; 若满足对称条件则设为 -2
+    int iminF = 1, jminF = 1, kminF = 1;
+    if (Symmetry > NO_SYMM && fabs(Z[0]) < dZ) kminF = -2;
+    if (Symmetry > EQ_SYMM && fabs(X[0]) < dX) iminF = -2;
+    if (Symmetry > EQ_SYMM && fabs(Y[0]) < dY) jminF = -2;
 
-        const int imaxF = ex1, jmaxF = ex2, kmaxF = ex3;
+    // 分配 fh：大小 (ex1+3)*(ex2+3)*(ex3+3)
+    const size_t nx = (size_t)ex1 + 3;
+    const size_t ny = (size_t)ex2 + 3;
+    const size_t nz = (size_t)ex3 + 3;
+    const size_t fh_size = nx * ny * nz;
 
-        for (int k0 = 0; k0 <= ex3 - 2; ++k0) {
-            const int kF = k0 + 1;
-            for (int j0 = 0; j0 <= ex2 - 2; ++j0) {
-                const int jF = j0 + 1;
-                for (int i0 = 0; i0 <= ex1 - 2; ++i0) {
-                    const int iF = i0 + 1;
-                    const size_t p = idx_ex(i0, j0, k0, ex);
+    double *fh = (double*)malloc(fh_size * sizeof(double));
+    if (!fh) return; // 内存不足：直接返回（你也可以改成 abort/报错）
 
-                    /* x-direction */
-                    const double sfx = Sfx[p];
-                    if (sfx > ZEO) {
-                        if (i0 <= ex1 - 3) // i+2 <= imax
-                            f_rhs[p] += sfx * d2dx * (
-                                -F3*fh[idx_fh_F_ord2(iF,   jF, kF, ex)] +
-                                 F4*fh[idx_fh_F_ord2(iF+1, jF, kF, ex)] -
-                                    fh[idx_fh_F_ord2(iF+2, jF, kF, ex)]);
-                        else if (i0 <= ex1 - 2) // i+1 <= imax
-                            f_rhs[p] += sfx * d2dx * (
-                                -fh[idx_fh_F_ord2(iF,   jF, kF, ex)] +
-                                 fh[idx_fh_F_ord2(iF+1, jF, kF, ex)]);
-                    } else if (sfx < ZEO) {
-                        if ((i0 - 1) >= iminF) // i-2 >= imin
-                            f_rhs[p] -= sfx * d2dx * (
-                                -F3*fh[idx_fh_F_ord2(iF,   jF, kF, ex)] +
-                                 F4*fh[idx_fh_F_ord2(iF-1, jF, kF, ex)] -
-                                    fh[idx_fh_F_ord2(iF-2, jF, kF, ex)]);
-                        else if (i0 >= iminF) // i-1 >= imin
-                            f_rhs[p] -= sfx * d2dx * (
-                                -fh[idx_fh_F_ord2(iF,   jF, kF, ex)] +
-                                 fh[idx_fh_F_ord2(iF-1, jF, kF, ex)]);
+    // Fortran: call symmetry_bd(3,ex,f,fh,SoA)
+    symmetry_bd(3, ex, f, fh, SoA);
+
+    /*
+     * Fortran 主循环：
+     * do k=1,ex(3)-1
+     * do j=1,ex(2)-1
+     * do i=1,ex(1)-1
+     *
+     * 转成 C 0-based：
+     * k0 = 0..ex3-2, j0 = 0..ex2-2, i0 = 0..ex1-2
+     *
+     * 并且 Fortran 里的 i/j/k 在 fh 访问时，仍然是 Fortran 索引值：
+     * iF=i0+1, jF=j0+1, kF=k0+1
+     */
+    for (int k0 = 0; k0 <= ex3 - 2; ++k0) {
+        const int kF = k0 + 1;
+        for (int j0 = 0; j0 <= ex2 - 2; ++j0) {
+            const int jF = j0 + 1;
+            for (int i0 = 0; i0 <= ex1 - 2; ++i0) {
+                const int iF = i0 + 1;
+
+                const size_t p = idx_ex(i0, j0, k0, ex);
+
+                // ---------------- x direction ----------------
+                const double sfx = Sfx[p];
+                if (sfx > ZEO) {
+                    // Fortran: if(i+3 <= imax)
+                    // iF+3 <= ex1  <=> i0+4 <= ex1 <=> i0 <= ex1-4
+                    if (i0 <= ex1 - 4) {
+                        f_rhs[p] += sfx * d12dx *
+                            (-F3  * fh[idx_fh_F(iF - 1, jF, kF, ex)]
+                             -F10 * fh[idx_fh_F(iF    , jF, kF, ex)]
+                             +F18 * fh[idx_fh_F(iF + 1, jF, kF, ex)]
+                             -F6  * fh[idx_fh_F(iF + 2, jF, kF, ex)]
+                             +      fh[idx_fh_F(iF + 3, jF, kF, ex)]);
                     }
-
-                    /* y-direction */
-                    const double sfy = Sfy[p];
-                    if (sfy > ZEO) {
-                        if (j0 <= ex2-3)
-                            f_rhs[p] += sfy * d2dy * (
-                                -F3*fh[idx_fh_F_ord2(iF, jF,   kF, ex)] +
-                                 F4*fh[idx_fh_F_ord2(iF, jF+1, kF, ex)] -
-                                    fh[idx_fh_F_ord2(iF, jF+2, kF, ex)]);
-                        else if (j0 <= ex2-2)
-                            f_rhs[p] += sfy * d2dy * (
-                                -fh[idx_fh_F_ord2(iF, jF,   kF, ex)] +
-                                 fh[idx_fh_F_ord2(iF, jF+1, kF, ex)]);
-                    } else if (sfy < ZEO) {
-                        if ((j0-1) >= jminF)
-                            f_rhs[p] -= sfy * d2dy * (
-                                -F3*fh[idx_fh_F_ord2(iF, jF,   kF, ex)] +
-                                 F4*fh[idx_fh_F_ord2(iF, jF-1, kF, ex)] -
-                                    fh[idx_fh_F_ord2(iF, jF-2, kF, ex)]);
-                        else if (j0 >= jminF)
-                            f_rhs[p] -= sfy * d2dy * (
-                                -fh[idx_fh_F_ord2(iF, jF,   kF, ex)] +
-                                 fh[idx_fh_F_ord2(iF, jF-1, kF, ex)]);
+                    // elseif(i+2 <= imax)  <=> i0 <= ex1-3
+                    else if (i0 <= ex1 - 3) {
+                        f_rhs[p] += sfx * d12dx *
+                            ( fh[idx_fh_F(iF - 2, jF, kF, ex)]
+                             -EIT * fh[idx_fh_F(iF - 1, jF, kF, ex)]
+                             +EIT * fh[idx_fh_F(iF + 1, jF, kF, ex)]
+                             -      fh[idx_fh_F(iF + 2, jF, kF, ex)]);
                     }
+                    // elseif(i+1 <= imax)  <=> i0 <= ex1-2（循环里总成立）
+                    else if (i0 <= ex1 - 2) {
+                        f_rhs[p] -= sfx * d12dx *
+                            (-F3  * fh[idx_fh_F(iF + 1, jF, kF, ex)]
+                             -F10 * fh[idx_fh_F(iF    , jF, kF, ex)]
+                             +F18 * fh[idx_fh_F(iF - 1, jF, kF, ex)]
+                             -F6  * fh[idx_fh_F(iF - 2, jF, kF, ex)]
+                             +      fh[idx_fh_F(iF - 3, jF, kF, ex)]);
+                    }
+                } else if (sfx < ZEO) {
+                    // Fortran: if(i-3 >= imin)
+                    // (iF-3) >= iminF  <=> (i0-2) >= iminF
+                    if ((i0 - 2) >= iminF) {
+                        f_rhs[p] -= sfx * d12dx *
+                            (-F3  * fh[idx_fh_F(iF + 1, jF, kF, ex)]
+                             -F10 * fh[idx_fh_F(iF    , jF, kF, ex)]
+                             +F18 * fh[idx_fh_F(iF - 1, jF, kF, ex)]
+                             -F6  * fh[idx_fh_F(iF - 2, jF, kF, ex)]
+                             +      fh[idx_fh_F(iF - 3, jF, kF, ex)]);
+                    }
+                    // elseif(i-2 >= imin) <=> (i0-1) >= iminF
+                    else if ((i0 - 1) >= iminF) {
+                        f_rhs[p] += sfx * d12dx *
+                            ( fh[idx_fh_F(iF - 2, jF, kF, ex)]
+                             -EIT * fh[idx_fh_F(iF - 1, jF, kF, ex)]
+                             +EIT * fh[idx_fh_F(iF + 1, jF, kF, ex)]
+                             -      fh[idx_fh_F(iF + 2, jF, kF, ex)]);
+                    }
+                    // elseif(i-1 >= imin) <=> i0 >= iminF
+                    else if (i0 >= iminF) {
+                        f_rhs[p] += sfx * d12dx *
+                            (-F3  * fh[idx_fh_F(iF - 1, jF, kF, ex)]
+                             -F10 * fh[idx_fh_F(iF    , jF, kF, ex)]
+                             +F18 * fh[idx_fh_F(iF + 1, jF, kF, ex)]
+                             -F6  * fh[idx_fh_F(iF + 2, jF, kF, ex)]
+                             +      fh[idx_fh_F(iF + 3, jF, kF, ex)]);
+                    }
+                }
 
-                    /* z-direction */
-                    const double sfz = Sfz[p];
-                    if (sfz > ZEO) {
-                        if (k0 <= ex3-3)
-                            f_rhs[p] += sfz * d2dz * (
-                                -F3*fh[idx_fh_F_ord2(iF, jF, kF,   ex)] +
-                                 F4*fh[idx_fh_F_ord2(iF, jF, kF+1, ex)] -
-                                    fh[idx_fh_F_ord2(iF, jF, kF+2, ex)]);
-                        else if (k0 <= ex3-2)
-                            f_rhs[p] += sfz * d2dz * (
-                                -fh[idx_fh_F_ord2(iF, jF, kF,   ex)] +
-                                 fh[idx_fh_F_ord2(iF, jF, kF+1, ex)]);
-                    } else if (sfz < ZEO) {
-                        if ((k0-1) >= kminF)
-                            f_rhs[p] -= sfz * d2dz * (
-                                -F3*fh[idx_fh_F_ord2(iF, jF, kF,   ex)] +
-                                 F4*fh[idx_fh_F_ord2(iF, jF, kF-1, ex)] -
-                                    fh[idx_fh_F_ord2(iF, jF, kF-2, ex)]);
-                        else if (k0 >= kminF)
-                            f_rhs[p] -= sfz * d2dz * (
-                                -fh[idx_fh_F_ord2(iF, jF, kF,   ex)] +
-                                 fh[idx_fh_F_ord2(iF, jF, kF-1, ex)]);
+                // ---------------- y direction ----------------
+                const double sfy = Sfy[p];
+                if (sfy > ZEO) {
+                    // jF+3 <= ex2 <=> j0+4 <= ex2 <=> j0 <= ex2-4
+                    if (j0 <= ex2 - 4) {
+                        f_rhs[p] += sfy * d12dy *
+                            (-F3  * fh[idx_fh_F(iF, jF - 1, kF, ex)]
+                             -F10 * fh[idx_fh_F(iF, jF    , kF, ex)]
+                             +F18 * fh[idx_fh_F(iF, jF + 1, kF, ex)]
+                             -F6  * fh[idx_fh_F(iF, jF + 2, kF, ex)]
+                             +      fh[idx_fh_F(iF, jF + 3, kF, ex)]);
+                    } else if (j0 <= ex2 - 3) {
+                        f_rhs[p] += sfy * d12dy *
+                            ( fh[idx_fh_F(iF, jF - 2, kF, ex)]
+                             -EIT * fh[idx_fh_F(iF, jF - 1, kF, ex)]
+                             +EIT * fh[idx_fh_F(iF, jF + 1, kF, ex)]
+                             -      fh[idx_fh_F(iF, jF + 2, kF, ex)]);
+                    } else if (j0 <= ex2 - 2) {
+                        f_rhs[p] -= sfy * d12dy *
+                            (-F3  * fh[idx_fh_F(iF, jF + 1, kF, ex)]
+                             -F10 * fh[idx_fh_F(iF, jF    , kF, ex)]
+                             +F18 * fh[idx_fh_F(iF, jF - 1, kF, ex)]
+                             -F6  * fh[idx_fh_F(iF, jF - 2, kF, ex)]
+                             +      fh[idx_fh_F(iF, jF - 3, kF, ex)]);
+                    }
+                } else if (sfy < ZEO) {
+                    if ((j0 - 2) >= jminF) {
+                        f_rhs[p] -= sfy * d12dy *
+                            (-F3  * fh[idx_fh_F(iF, jF + 1, kF, ex)]
+                             -F10 * fh[idx_fh_F(iF, jF    , kF, ex)]
+                             +F18 * fh[idx_fh_F(iF, jF - 1, kF, ex)]
+                             -F6  * fh[idx_fh_F(iF, jF - 2, kF, ex)]
+                             +      fh[idx_fh_F(iF, jF - 3, kF, ex)]);
+                    } else if ((j0 - 1) >= jminF) {
+                        f_rhs[p] += sfy * d12dy *
+                            ( fh[idx_fh_F(iF, jF - 2, kF, ex)]
+                             -EIT * fh[idx_fh_F(iF, jF - 1, kF, ex)]
+                             +EIT * fh[idx_fh_F(iF, jF + 1, kF, ex)]
+                             -      fh[idx_fh_F(iF, jF + 2, kF, ex)]);
+                    } else if (j0 >= jminF) {
+                        f_rhs[p] += sfy * d12dy *
+                            (-F3  * fh[idx_fh_F(iF, jF - 1, kF, ex)]
+                             -F10 * fh[idx_fh_F(iF, jF    , kF, ex)]
+                             +F18 * fh[idx_fh_F(iF, jF + 1, kF, ex)]
+                             -F6  * fh[idx_fh_F(iF, jF + 2, kF, ex)]
+                             +      fh[idx_fh_F(iF, jF + 3, kF, ex)]);
+                    }
+                }
+
+                // ---------------- z direction ----------------
+                const double sfz = Sfz[p];
+                if (sfz > ZEO) {
+                    if (k0 <= ex3 - 4) {
+                        f_rhs[p] += sfz * d12dz *
+                            (-F3  * fh[idx_fh_F(iF, jF, kF - 1, ex)]
+                             -F10 * fh[idx_fh_F(iF, jF, kF    , ex)]
+                             +F18 * fh[idx_fh_F(iF, jF, kF + 1, ex)]
+                             -F6  * fh[idx_fh_F(iF, jF, kF + 2, ex)]
+                             +      fh[idx_fh_F(iF, jF, kF + 3, ex)]);
+                    } else if (k0 <= ex3 - 3) {
+                        f_rhs[p] += sfz * d12dz *
+                            ( fh[idx_fh_F(iF, jF, kF - 2, ex)]
+                             -EIT * fh[idx_fh_F(iF, jF, kF - 1, ex)]
+                             +EIT * fh[idx_fh_F(iF, jF, kF + 1, ex)]
+                             -      fh[idx_fh_F(iF, jF, kF + 2, ex)]);
+                    } else if (k0 <= ex3 - 2) {
+                        f_rhs[p] -= sfz * d12dz *
+                            (-F3  * fh[idx_fh_F(iF, jF, kF + 1, ex)]
+                             -F10 * fh[idx_fh_F(iF, jF, kF    , ex)]
+                             +F18 * fh[idx_fh_F(iF, jF, kF - 1, ex)]
+                             -F6  * fh[idx_fh_F(iF, jF, kF - 2, ex)]
+                             +      fh[idx_fh_F(iF, jF, kF - 3, ex)]);
+                    }
+                } else if (sfz < ZEO) {
+                    if ((k0 - 2) >= kminF) {
+                        f_rhs[p] -= sfz * d12dz *
+                            (-F3  * fh[idx_fh_F(iF, jF, kF + 1, ex)]
+                             -F10 * fh[idx_fh_F(iF, jF, kF    , ex)]
+                             +F18 * fh[idx_fh_F(iF, jF, kF - 1, ex)]
+                             -F6  * fh[idx_fh_F(iF, jF, kF - 2, ex)]
+                             +      fh[idx_fh_F(iF, jF, kF - 3, ex)]);
+                    } else if ((k0 - 1) >= kminF) {
+                        f_rhs[p] += sfz * d12dz *
+                            ( fh[idx_fh_F(iF, jF, kF - 2, ex)]
+                             -EIT * fh[idx_fh_F(iF, jF, kF - 1, ex)]
+                             +EIT * fh[idx_fh_F(iF, jF, kF + 1, ex)]
+                             -      fh[idx_fh_F(iF, jF, kF + 2, ex)]);
+                    } else if (k0 >= kminF) {
+                        f_rhs[p] += sfz * d12dz *
+                            (-F3  * fh[idx_fh_F(iF, jF, kF - 1, ex)]
+                             -F10 * fh[idx_fh_F(iF, jF, kF    , ex)]
+                             +F18 * fh[idx_fh_F(iF, jF, kF + 1, ex)]
+                             -F6  * fh[idx_fh_F(iF, jF, kF + 2, ex)]
+                             +      fh[idx_fh_F(iF, jF, kF + 3, ex)]);
                     }
                 }
             }
         }
-        free(fh);
-        return;
     }
-#elif (ghost_width == 3)
-    /* ---- 4th-order lopsided (original code) ---------------------------- */
-    {
-        const int ord = 3;
-        int iminF = 1, jminF = 1, kminF = 1;
-        if (Symmetry > NO_SYMM && fabs(Z[0]) < dZ) kminF = -2;
-        if (Symmetry > EQ_SYMM && fabs(X[0]) < dX) iminF = -2;
-        if (Symmetry > EQ_SYMM && fabs(Y[0]) < dY) jminF = -2;
-
-        const size_t nx = (size_t)ex1 + ord;
-        const size_t ny = (size_t)ex2 + ord;
-        const size_t nz = (size_t)ex3 + ord;
-        const size_t fh_size = nx * ny * nz;
-
-        double *fh = (double*)malloc(fh_size * sizeof(double));
-        if (!fh) return;
-        symmetry_bd(ord, ex, f, fh, SoA);
-
-        const double d12dx = ONE / F12 / dX;
-        const double d12dy = ONE / F12 / dY;
-        const double d12dz = ONE / F12 / dZ;
-
-        const int imaxF = ex1, jmaxF = ex2, kmaxF = ex3;
-
-        for (int k0 = 0; k0 <= ex3 - 2; ++k0) {
-            const int kF = k0 + 1;
-            for (int j0 = 0; j0 <= ex2 - 2; ++j0) {
-                const int jF = j0 + 1;
-                for (int i0 = 0; i0 <= ex1 - 2; ++i0) {
-                    const int iF = i0 + 1;
-                    const size_t p = idx_ex(i0, j0, k0, ex);
-
-                    const double sfx = Sfx[p];
-                    if (sfx > ZEO) {
-                        if (i0 <= ex1 - 4) // i+3 <= imax
-                            f_rhs[p] += sfx * d12dx * (
-                                -F3 *fh[idx_fh_F(iF-1, jF, kF, ex)]
-                                -F10*fh[idx_fh_F(iF,   jF, kF, ex)]
-                                +F18*fh[idx_fh_F(iF+1, jF, kF, ex)]
-                                -F6 *fh[idx_fh_F(iF+2, jF, kF, ex)]
-                                +    fh[idx_fh_F(iF+3, jF, kF, ex)]);
-                        else if (i0 <= ex1 - 3) // i+2 <= imax
-                            f_rhs[p] += sfx * d12dx * (
-                                fh[idx_fh_F(iF-2, jF, kF, ex)]
-                                -EIT*fh[idx_fh_F(iF-1, jF, kF, ex)]
-                                +EIT*fh[idx_fh_F(iF+1, jF, kF, ex)]
-                                -    fh[idx_fh_F(iF+2, jF, kF, ex)]);
-                        else if (i0 <= ex1 - 2) // i+1 <= imax → mirrored
-                            f_rhs[p] -= sfx * d12dx * (
-                                -F3 *fh[idx_fh_F(iF+1, jF, kF, ex)]
-                                -F10*fh[idx_fh_F(iF,   jF, kF, ex)]
-                                +F18*fh[idx_fh_F(iF-1, jF, kF, ex)]
-                                -F6 *fh[idx_fh_F(iF-2, jF, kF, ex)]
-                                +    fh[idx_fh_F(iF-3, jF, kF, ex)]);
-                    } else if (sfx < ZEO) {
-                        if ((i0 - 2) >= iminF) // i-3 >= imin
-                            f_rhs[p] -= sfx * d12dx * (
-                                -F3 *fh[idx_fh_F(iF+1, jF, kF, ex)]
-                                -F10*fh[idx_fh_F(iF,   jF, kF, ex)]
-                                +F18*fh[idx_fh_F(iF-1, jF, kF, ex)]
-                                -F6 *fh[idx_fh_F(iF-2, jF, kF, ex)]
-                                +    fh[idx_fh_F(iF-3, jF, kF, ex)]);
-                        else if ((i0 - 1) >= iminF) // i-2 >= imin
-                            f_rhs[p] += sfx * d12dx * (
-                                fh[idx_fh_F(iF-2, jF, kF, ex)]
-                                -EIT*fh[idx_fh_F(iF-1, jF, kF, ex)]
-                                +EIT*fh[idx_fh_F(iF+1, jF, kF, ex)]
-                                -    fh[idx_fh_F(iF+2, jF, kF, ex)]);
-                        else if (i0 >= iminF) // i-1 >= imin → mirrored
-                            f_rhs[p] += sfx * d12dx * (
-                                -F3 *fh[idx_fh_F(iF-1, jF, kF, ex)]
-                                -F10*fh[idx_fh_F(iF,   jF, kF, ex)]
-                                +F18*fh[idx_fh_F(iF+1, jF, kF, ex)]
-                                -F6 *fh[idx_fh_F(iF+2, jF, kF, ex)]
-                                +    fh[idx_fh_F(iF+3, jF, kF, ex)]);
-                    }
-
-                    const double sfy = Sfy[p];
-                    if (sfy > ZEO) {
-                        if (j0 <= ex2-4)
-                            f_rhs[p] += sfy * d12dy * (
-                                -F3*fh[idx_fh_F(iF,jF-1,kF,ex)]-F10*fh[idx_fh_F(iF,jF,kF,ex)]
-                                +F18*fh[idx_fh_F(iF,jF+1,kF,ex)]-F6*fh[idx_fh_F(iF,jF+2,kF,ex)]
-                                +fh[idx_fh_F(iF,jF+3,kF,ex)]);
-                        else if (j0 <= ex2-3)
-                            f_rhs[p] += sfy * d12dy * (fh[idx_fh_F(iF,jF-2,kF,ex)]-EIT*fh[idx_fh_F(iF,jF-1,kF,ex)]+EIT*fh[idx_fh_F(iF,jF+1,kF,ex)]-fh[idx_fh_F(iF,jF+2,kF,ex)]);
-                        else if (j0 <= ex2-2)
-                            f_rhs[p] -= sfy * d12dy * (
-                                -F3*fh[idx_fh_F(iF,jF+1,kF,ex)]-F10*fh[idx_fh_F(iF,jF,kF,ex)]
-                                +F18*fh[idx_fh_F(iF,jF-1,kF,ex)]-F6*fh[idx_fh_F(iF,jF-2,kF,ex)]
-                                +fh[idx_fh_F(iF,jF-3,kF,ex)]);
-                    } else if (sfy < ZEO) {
-                        if ((j0-2) >= jminF)
-                            f_rhs[p] -= sfy * d12dy * (
-                                -F3*fh[idx_fh_F(iF,jF+1,kF,ex)]-F10*fh[idx_fh_F(iF,jF,kF,ex)]
-                                +F18*fh[idx_fh_F(iF,jF-1,kF,ex)]-F6*fh[idx_fh_F(iF,jF-2,kF,ex)]
-                                +fh[idx_fh_F(iF,jF-3,kF,ex)]);
-                        else if ((j0-1) >= jminF)
-                            f_rhs[p] += sfy * d12dy * (fh[idx_fh_F(iF,jF-2,kF,ex)]-EIT*fh[idx_fh_F(iF,jF-1,kF,ex)]+EIT*fh[idx_fh_F(iF,jF+1,kF,ex)]-fh[idx_fh_F(iF,jF+2,kF,ex)]);
-                        else if (j0 >= jminF)
-                            f_rhs[p] += sfy * d12dy * (
-                                -F3*fh[idx_fh_F(iF,jF-1,kF,ex)]-F10*fh[idx_fh_F(iF,jF,kF,ex)]
-                                +F18*fh[idx_fh_F(iF,jF+1,kF,ex)]-F6*fh[idx_fh_F(iF,jF+2,kF,ex)]
-                                +fh[idx_fh_F(iF,jF+3,kF,ex)]);
-                    }
-
-                    const double sfz = Sfz[p];
-                    if (sfz > ZEO) {
-                        if (k0 <= ex3-4)
-                            f_rhs[p] += sfz * d12dz * (
-                                -F3*fh[idx_fh_F(iF,jF,kF-1,ex)]-F10*fh[idx_fh_F(iF,jF,kF,ex)]
-                                +F18*fh[idx_fh_F(iF,jF,kF+1,ex)]-F6*fh[idx_fh_F(iF,jF,kF+2,ex)]
-                                +fh[idx_fh_F(iF,jF,kF+3,ex)]);
-                        else if (k0 <= ex3-3)
-                            f_rhs[p] += sfz * d12dz * (fh[idx_fh_F(iF,jF,kF-2,ex)]-EIT*fh[idx_fh_F(iF,jF,kF-1,ex)]+EIT*fh[idx_fh_F(iF,jF,kF+1,ex)]-fh[idx_fh_F(iF,jF,kF+2,ex)]);
-                        else if (k0 <= ex3-2)
-                            f_rhs[p] -= sfz * d12dz * (
-                                -F3*fh[idx_fh_F(iF,jF,kF+1,ex)]-F10*fh[idx_fh_F(iF,jF,kF,ex)]
-                                +F18*fh[idx_fh_F(iF,jF,kF-1,ex)]-F6*fh[idx_fh_F(iF,jF,kF-2,ex)]
-                                +fh[idx_fh_F(iF,jF,kF-3,ex)]);
-                    } else if (sfz < ZEO) {
-                        if ((k0-2) >= kminF)
-                            f_rhs[p] -= sfz * d12dz * (
-                                -F3*fh[idx_fh_F(iF,jF,kF+1,ex)]-F10*fh[idx_fh_F(iF,jF,kF,ex)]
-                                +F18*fh[idx_fh_F(iF,jF,kF-1,ex)]-F6*fh[idx_fh_F(iF,jF,kF-2,ex)]
-                                +fh[idx_fh_F(iF,jF,kF-3,ex)]);
-                        else if ((k0-1) >= kminF)
-                            f_rhs[p] += sfz * d12dz * (fh[idx_fh_F(iF,jF,kF-2,ex)]-EIT*fh[idx_fh_F(iF,jF,kF-1,ex)]+EIT*fh[idx_fh_F(iF,jF,kF+1,ex)]-fh[idx_fh_F(iF,jF,kF+2,ex)]);
-                        else if (k0 >= kminF)
-                            f_rhs[p] += sfz * d12dz * (
-                                -F3*fh[idx_fh_F(iF,jF,kF-1,ex)]-F10*fh[idx_fh_F(iF,jF,kF,ex)]
-                                +F18*fh[idx_fh_F(iF,jF,kF+1,ex)]-F6*fh[idx_fh_F(iF,jF,kF+2,ex)]
-                                +fh[idx_fh_F(iF,jF,kF+3,ex)]);
-                    }
-                }
-            }
-        }
-        free(fh);
-        return;
-    }
-#elif (ghost_width == 4)
-    /* ---- 6th-order lopsided --------------------------------------------- */
-    {
-        const int ord = 4;
-        int iminF = 1, jminF = 1, kminF = 1;
-        if (Symmetry > NO_SYMM && fabs(Z[0]) < dZ) kminF = -3;
-        if (Symmetry > EQ_SYMM && fabs(X[0]) < dX) iminF = -3;
-        if (Symmetry > EQ_SYMM && fabs(Y[0]) < dY) jminF = -3;
-
-        const size_t nx = (size_t)ex1 + ord;
-        const size_t ny = (size_t)ex2 + ord;
-        const size_t nz = (size_t)ex3 + ord;
-        const size_t fh_size = nx * ny * nz;
-
-        double *fh = (double*)malloc(fh_size * sizeof(double));
-        if (!fh) return;
-        symmetry_bd(ord, ex, f, fh, SoA);
-
-        const double d60dx = ONE / F60 / dX;
-        const double d60dy = ONE / F60 / dY;
-        const double d60dz = ONE / F60 / dZ;
-        const double d12dx = ONE / F12 / dX;
-        const double d12dy = ONE / F12 / dY;
-        const double d12dz = ONE / F12 / dZ;
-        const double d2dx  = ONE / TWO / dX;
-        const double d2dy  = ONE / TWO / dY;
-        const double d2dz  = ONE / TWO / dZ;
-
-        const int imaxF = ex1, jmaxF = ex2, kmaxF = ex3;
-
-        for (int k0 = 0; k0 <= ex3 - 2; ++k0) {
-            const int kF = k0 + 1;
-            for (int j0 = 0; j0 <= ex2 - 2; ++j0) {
-                const int jF = j0 + 1;
-                for (int i0 = 0; i0 <= ex1 - 2; ++i0) {
-                    const int iF = i0 + 1;
-                    const size_t p = idx_ex(i0, j0, k0, ex);
-
-                    /* ---- x-direction ---- */
-                    const double sfx = Sfx[p];
-                    if (sfx > ZEO) {
-                        /* Primary biased: 2*f(i-2)-24*f(i-1)-35*f(i)+80*f(i+1)-30*f(i+2)+8*f(i+3)-f(i+4) */
-                        if (i0 <= ex1-5 && (i0-1)>=iminF) // i+4<=imax && i-2>=imin
-                            f_rhs[p] += sfx * d60dx * (
-                                +F2*fh[idx_fh_F_ord4(iF-2,jF,kF,ex)]-F24*fh[idx_fh_F_ord4(iF-1,jF,kF,ex)]
-                                -F35*fh[idx_fh_F_ord4(iF,jF,kF,ex)]+F80*fh[idx_fh_F_ord4(iF+1,jF,kF,ex)]
-                                -F30*fh[idx_fh_F_ord4(iF+2,jF,kF,ex)]+EIT*fh[idx_fh_F_ord4(iF+3,jF,kF,ex)]
-                                -fh[idx_fh_F_ord4(iF+4,jF,kF,ex)]);
-                        /* Boundary-adapted: -10*f(i-1)-77*f(i)+150*f(i+1)-100*f(i+2)+50*f(i+3)-15*f(i+4)+2*f(i+5) */
-                        else if (i0 <= ex1-6 && i0 >= iminF) // i+5<=imax && i-1>=imin
-                            f_rhs[p] += sfx * d60dx * (
-                                -F10*fh[idx_fh_F_ord4(iF-1,jF,kF,ex)]-F77*fh[idx_fh_F_ord4(iF,jF,kF,ex)]
-                                +F150*fh[idx_fh_F_ord4(iF+1,jF,kF,ex)]-F100*fh[idx_fh_F_ord4(iF+2,jF,kF,ex)]
-                                +F50*fh[idx_fh_F_ord4(iF+3,jF,kF,ex)]-F15*fh[idx_fh_F_ord4(iF+4,jF,kF,ex)]
-                                +F2*fh[idx_fh_F_ord4(iF+5,jF,kF,ex)]);
-                        /* Centered fallbacks */
-                        else if (i0 <= ex1-4 && (i0-2)>=iminF) // 6th: i+3<=imax && i-3>=imin
-                            f_rhs[p] += sfx * d60dx * (
-                                -fh[idx_fh_F_ord4(iF-3,jF,kF,ex)]+F9*fh[idx_fh_F_ord4(iF-2,jF,kF,ex)]
-                                -F45*fh[idx_fh_F_ord4(iF-1,jF,kF,ex)]+F45*fh[idx_fh_F_ord4(iF+1,jF,kF,ex)]
-                                -F9*fh[idx_fh_F_ord4(iF+2,jF,kF,ex)]+fh[idx_fh_F_ord4(iF+3,jF,kF,ex)]);
-                        else if (i0 <= ex1-3 && (i0-1)>=iminF) // 4th
-                            f_rhs[p] += sfx * d12dx * (
-                                fh[idx_fh_F_ord4(iF-2,jF,kF,ex)]-EIT*fh[idx_fh_F_ord4(iF-1,jF,kF,ex)]
-                                +EIT*fh[idx_fh_F_ord4(iF+1,jF,kF,ex)]-fh[idx_fh_F_ord4(iF+2,jF,kF,ex)]);
-                        else if (i0 <= ex1-2 && i0>=iminF) // 2nd
-                            f_rhs[p] += sfx * d2dx * (
-                                -fh[idx_fh_F_ord4(iF-1,jF,kF,ex)]+fh[idx_fh_F_ord4(iF+1,jF,kF,ex)]);
-                    } else if (sfx < ZEO) {
-                        if ((i0-3)>=iminF && i0<=ex1-3) // i-4>=imin && i+2<=imax
-                            f_rhs[p] -= sfx * d60dx * (
-                                +F2*fh[idx_fh_F_ord4(iF+2,jF,kF,ex)]-F24*fh[idx_fh_F_ord4(iF+1,jF,kF,ex)]
-                                -F35*fh[idx_fh_F_ord4(iF,jF,kF,ex)]+F80*fh[idx_fh_F_ord4(iF-1,jF,kF,ex)]
-                                -F30*fh[idx_fh_F_ord4(iF-2,jF,kF,ex)]+EIT*fh[idx_fh_F_ord4(iF-3,jF,kF,ex)]
-                                -fh[idx_fh_F_ord4(iF-4,jF,kF,ex)]);
-                        else if ((i0-4)>=iminF && i0<=ex1-2) // i-5>=imin && i+1<=imax
-                            f_rhs[p] -= sfx * d60dx * (
-                                -F10*fh[idx_fh_F_ord4(iF+1,jF,kF,ex)]-F77*fh[idx_fh_F_ord4(iF,jF,kF,ex)]
-                                +F150*fh[idx_fh_F_ord4(iF-1,jF,kF,ex)]-F100*fh[idx_fh_F_ord4(iF-2,jF,kF,ex)]
-                                +F50*fh[idx_fh_F_ord4(iF-3,jF,kF,ex)]-F15*fh[idx_fh_F_ord4(iF-4,jF,kF,ex)]
-                                +F2*fh[idx_fh_F_ord4(iF-5,jF,kF,ex)]);
-                        else if ((i0-2)>=iminF && i0<=ex1-4) // 6th centered
-                            f_rhs[p] += sfx * d60dx * (
-                                -fh[idx_fh_F_ord4(iF-3,jF,kF,ex)]+F9*fh[idx_fh_F_ord4(iF-2,jF,kF,ex)]
-                                -F45*fh[idx_fh_F_ord4(iF-1,jF,kF,ex)]+F45*fh[idx_fh_F_ord4(iF+1,jF,kF,ex)]
-                                -F9*fh[idx_fh_F_ord4(iF+2,jF,kF,ex)]+fh[idx_fh_F_ord4(iF+3,jF,kF,ex)]);
-                        else if ((i0-1)>=iminF && i0<=ex1-3) // 4th
-                            f_rhs[p] += sfx * d12dx * (
-                                fh[idx_fh_F_ord4(iF-2,jF,kF,ex)]-EIT*fh[idx_fh_F_ord4(iF-1,jF,kF,ex)]
-                                +EIT*fh[idx_fh_F_ord4(iF+1,jF,kF,ex)]-fh[idx_fh_F_ord4(iF+2,jF,kF,ex)]);
-                        else if (i0>=iminF && i0<=ex1-2) // 2nd
-                            f_rhs[p] += sfx * d2dx * (
-                                -fh[idx_fh_F_ord4(iF-1,jF,kF,ex)]+fh[idx_fh_F_ord4(iF+1,jF,kF,ex)]);
-                    }
-
-                    /* ---- y-direction ---- */
-                    const double sfy = Sfy[p];
-                    if (sfy > ZEO) {
-                        if (j0<=ex2-5 && (j0-1)>=jminF)
-                            f_rhs[p] += sfy * d60dy*(F2*fh[idx_fh_F_ord4(iF,jF-2,kF,ex)]-F24*fh[idx_fh_F_ord4(iF,jF-1,kF,ex)]-F35*fh[idx_fh_F_ord4(iF,jF,kF,ex)]+F80*fh[idx_fh_F_ord4(iF,jF+1,kF,ex)]-F30*fh[idx_fh_F_ord4(iF,jF+2,kF,ex)]+EIT*fh[idx_fh_F_ord4(iF,jF+3,kF,ex)]-fh[idx_fh_F_ord4(iF,jF+4,kF,ex)]);
-                        else if (j0<=ex2-6 && j0>=jminF)
-                            f_rhs[p] += sfy * d60dy*(-F10*fh[idx_fh_F_ord4(iF,jF-1,kF,ex)]-F77*fh[idx_fh_F_ord4(iF,jF,kF,ex)]+F150*fh[idx_fh_F_ord4(iF,jF+1,kF,ex)]-F100*fh[idx_fh_F_ord4(iF,jF+2,kF,ex)]+F50*fh[idx_fh_F_ord4(iF,jF+3,kF,ex)]-F15*fh[idx_fh_F_ord4(iF,jF+4,kF,ex)]+F2*fh[idx_fh_F_ord4(iF,jF+5,kF,ex)]);
-                        else if (j0<=ex2-4 && (j0-2)>=jminF)
-                            f_rhs[p] += sfy * d60dy*(-fh[idx_fh_F_ord4(iF,jF-3,kF,ex)]+F9*fh[idx_fh_F_ord4(iF,jF-2,kF,ex)]-F45*fh[idx_fh_F_ord4(iF,jF-1,kF,ex)]+F45*fh[idx_fh_F_ord4(iF,jF+1,kF,ex)]-F9*fh[idx_fh_F_ord4(iF,jF+2,kF,ex)]+fh[idx_fh_F_ord4(iF,jF+3,kF,ex)]);
-                        else if (j0<=ex2-3 && (j0-1)>=jminF)
-                            f_rhs[p] += sfy * d12dy*(fh[idx_fh_F_ord4(iF,jF-2,kF,ex)]-EIT*fh[idx_fh_F_ord4(iF,jF-1,kF,ex)]+EIT*fh[idx_fh_F_ord4(iF,jF+1,kF,ex)]-fh[idx_fh_F_ord4(iF,jF+2,kF,ex)]);
-                        else if (j0<=ex2-2 && j0>=jminF)
-                            f_rhs[p] += sfy * d2dy*(-fh[idx_fh_F_ord4(iF,jF-1,kF,ex)]+fh[idx_fh_F_ord4(iF,jF+1,kF,ex)]);
-                    } else if (sfy < ZEO) {
-                        if ((j0-3)>=jminF && j0<=ex2-3)
-                            f_rhs[p] -= sfy * d60dy*(F2*fh[idx_fh_F_ord4(iF,jF+2,kF,ex)]-F24*fh[idx_fh_F_ord4(iF,jF+1,kF,ex)]-F35*fh[idx_fh_F_ord4(iF,jF,kF,ex)]+F80*fh[idx_fh_F_ord4(iF,jF-1,kF,ex)]-F30*fh[idx_fh_F_ord4(iF,jF-2,kF,ex)]+EIT*fh[idx_fh_F_ord4(iF,jF-3,kF,ex)]-fh[idx_fh_F_ord4(iF,jF-4,kF,ex)]);
-                        else if ((j0-4)>=jminF && j0<=ex2-2)
-                            f_rhs[p] -= sfy * d60dy*(-F10*fh[idx_fh_F_ord4(iF,jF+1,kF,ex)]-F77*fh[idx_fh_F_ord4(iF,jF,kF,ex)]+F150*fh[idx_fh_F_ord4(iF,jF-1,kF,ex)]-F100*fh[idx_fh_F_ord4(iF,jF-2,kF,ex)]+F50*fh[idx_fh_F_ord4(iF,jF-3,kF,ex)]-F15*fh[idx_fh_F_ord4(iF,jF-4,kF,ex)]+F2*fh[idx_fh_F_ord4(iF,jF-5,kF,ex)]);
-                        else if ((j0-2)>=jminF && j0<=ex2-4)
-                            f_rhs[p] += sfy * d60dy*(-fh[idx_fh_F_ord4(iF,jF-3,kF,ex)]+F9*fh[idx_fh_F_ord4(iF,jF-2,kF,ex)]-F45*fh[idx_fh_F_ord4(iF,jF-1,kF,ex)]+F45*fh[idx_fh_F_ord4(iF,jF+1,kF,ex)]-F9*fh[idx_fh_F_ord4(iF,jF+2,kF,ex)]+fh[idx_fh_F_ord4(iF,jF+3,kF,ex)]);
-                        else if ((j0-1)>=jminF && j0<=ex2-3)
-                            f_rhs[p] += sfy * d12dy*(fh[idx_fh_F_ord4(iF,jF-2,kF,ex)]-EIT*fh[idx_fh_F_ord4(iF,jF-1,kF,ex)]+EIT*fh[idx_fh_F_ord4(iF,jF+1,kF,ex)]-fh[idx_fh_F_ord4(iF,jF+2,kF,ex)]);
-                        else if (j0>=jminF && j0<=ex2-2)
-                            f_rhs[p] += sfy * d2dy*(-fh[idx_fh_F_ord4(iF,jF-1,kF,ex)]+fh[idx_fh_F_ord4(iF,jF+1,kF,ex)]);
-                    }
-
-                    /* ---- z-direction ---- */
-                    const double sfz = Sfz[p];
-                    if (sfz > ZEO) {
-                        if (k0<=ex3-5 && (k0-1)>=kminF)
-                            f_rhs[p] += sfz * d60dz*(F2*fh[idx_fh_F_ord4(iF,jF,kF-2,ex)]-F24*fh[idx_fh_F_ord4(iF,jF,kF-1,ex)]-F35*fh[idx_fh_F_ord4(iF,jF,kF,ex)]+F80*fh[idx_fh_F_ord4(iF,jF,kF+1,ex)]-F30*fh[idx_fh_F_ord4(iF,jF,kF+2,ex)]+EIT*fh[idx_fh_F_ord4(iF,jF,kF+3,ex)]-fh[idx_fh_F_ord4(iF,jF,kF+4,ex)]);
-                        else if (k0<=ex3-6 && k0>=kminF)
-                            f_rhs[p] += sfz * d60dz*(-F10*fh[idx_fh_F_ord4(iF,jF,kF-1,ex)]-F77*fh[idx_fh_F_ord4(iF,jF,kF,ex)]+F150*fh[idx_fh_F_ord4(iF,jF,kF+1,ex)]-F100*fh[idx_fh_F_ord4(iF,jF,kF+2,ex)]+F50*fh[idx_fh_F_ord4(iF,jF,kF+3,ex)]-F15*fh[idx_fh_F_ord4(iF,jF,kF+4,ex)]+F2*fh[idx_fh_F_ord4(iF,jF,kF+5,ex)]);
-                        else if (k0<=ex3-4 && (k0-2)>=kminF)
-                            f_rhs[p] += sfz * d60dz*(-fh[idx_fh_F_ord4(iF,jF,kF-3,ex)]+F9*fh[idx_fh_F_ord4(iF,jF,kF-2,ex)]-F45*fh[idx_fh_F_ord4(iF,jF,kF-1,ex)]+F45*fh[idx_fh_F_ord4(iF,jF,kF+1,ex)]-F9*fh[idx_fh_F_ord4(iF,jF,kF+2,ex)]+fh[idx_fh_F_ord4(iF,jF,kF+3,ex)]);
-                        else if (k0<=ex3-3 && (k0-1)>=kminF)
-                            f_rhs[p] += sfz * d12dz*(fh[idx_fh_F_ord4(iF,jF,kF-2,ex)]-EIT*fh[idx_fh_F_ord4(iF,jF,kF-1,ex)]+EIT*fh[idx_fh_F_ord4(iF,jF,kF+1,ex)]-fh[idx_fh_F_ord4(iF,jF,kF+2,ex)]);
-                        else if (k0<=ex3-2 && k0>=kminF)
-                            f_rhs[p] += sfz * d2dz*(-fh[idx_fh_F_ord4(iF,jF,kF-1,ex)]+fh[idx_fh_F_ord4(iF,jF,kF+1,ex)]);
-                    } else if (sfz < ZEO) {
-                        if ((k0-3)>=kminF && k0<=ex3-3)
-                            f_rhs[p] -= sfz * d60dz*(F2*fh[idx_fh_F_ord4(iF,jF,kF+2,ex)]-F24*fh[idx_fh_F_ord4(iF,jF,kF+1,ex)]-F35*fh[idx_fh_F_ord4(iF,jF,kF,ex)]+F80*fh[idx_fh_F_ord4(iF,jF,kF-1,ex)]-F30*fh[idx_fh_F_ord4(iF,jF,kF-2,ex)]+EIT*fh[idx_fh_F_ord4(iF,jF,kF-3,ex)]-fh[idx_fh_F_ord4(iF,jF,kF-4,ex)]);
-                        else if ((k0-4)>=kminF && k0<=ex3-2)
-                            f_rhs[p] -= sfz * d60dz*(-F10*fh[idx_fh_F_ord4(iF,jF,kF+1,ex)]-F77*fh[idx_fh_F_ord4(iF,jF,kF,ex)]+F150*fh[idx_fh_F_ord4(iF,jF,kF-1,ex)]-F100*fh[idx_fh_F_ord4(iF,jF,kF-2,ex)]+F50*fh[idx_fh_F_ord4(iF,jF,kF-3,ex)]-F15*fh[idx_fh_F_ord4(iF,jF,kF-4,ex)]+F2*fh[idx_fh_F_ord4(iF,jF,kF-5,ex)]);
-                        else if ((k0-2)>=kminF && k0<=ex3-4)
-                            f_rhs[p] += sfz * d60dz*(-fh[idx_fh_F_ord4(iF,jF,kF-3,ex)]+F9*fh[idx_fh_F_ord4(iF,jF,kF-2,ex)]-F45*fh[idx_fh_F_ord4(iF,jF,kF-1,ex)]+F45*fh[idx_fh_F_ord4(iF,jF,kF+1,ex)]-F9*fh[idx_fh_F_ord4(iF,jF,kF+2,ex)]+fh[idx_fh_F_ord4(iF,jF,kF+3,ex)]);
-                        else if ((k0-1)>=kminF && k0<=ex3-3)
-                            f_rhs[p] += sfz * d12dz*(fh[idx_fh_F_ord4(iF,jF,kF-2,ex)]-EIT*fh[idx_fh_F_ord4(iF,jF,kF-1,ex)]+EIT*fh[idx_fh_F_ord4(iF,jF,kF+1,ex)]-fh[idx_fh_F_ord4(iF,jF,kF+2,ex)]);
-                        else if (k0>=kminF && k0<=ex3-2)
-                            f_rhs[p] += sfz * d2dz*(-fh[idx_fh_F_ord4(iF,jF,kF-1,ex)]+fh[idx_fh_F_ord4(iF,jF,kF+1,ex)]);
-                    }
-                }
-            }
-        }
-        free(fh);
-        return;
-    }
-#elif (ghost_width == 5)
-    /* ---- 8th-order lopsided --------------------------------------------- */
-    {
-        const int ord = 5;
-        int iminF = 1, jminF = 1, kminF = 1;
-        if (Symmetry > NO_SYMM && fabs(Z[0]) < dZ) kminF = -4;
-        if (Symmetry > EQ_SYMM && fabs(X[0]) < dX) iminF = -4;
-        if (Symmetry > EQ_SYMM && fabs(Y[0]) < dY) jminF = -4;
-
-        const size_t nx = (size_t)ex1 + ord;
-        const size_t ny = (size_t)ex2 + ord;
-        const size_t nz = (size_t)ex3 + ord;
-        const size_t fh_size = nx * ny * nz;
-
-        double *fh = (double*)malloc(fh_size * sizeof(double));
-        if (!fh) return;
-        symmetry_bd(ord, ex, f, fh, SoA);
-
-        const double d840dx = ONE / F840 / dX;
-        const double d840dy = ONE / F840 / dY;
-        const double d840dz = ONE / F840 / dZ;
-        const double d60dx = ONE / F60 / dX;
-        const double d60dy = ONE / F60 / dY;
-        const double d60dz = ONE / F60 / dZ;
-        const double d12dx = ONE / F12 / dX;
-        const double d12dy = ONE / F12 / dY;
-        const double d12dz = ONE / F12 / dZ;
-        const double d2dx  = ONE / TWO / dX;
-        const double d2dy  = ONE / TWO / dY;
-        const double d2dz  = ONE / TWO / dZ;
-
-        const int imaxF = ex1, jmaxF = ex2, kmaxF = ex3;
-
-        for (int k0 = 0; k0 <= ex3 - 2; ++k0) {
-            const int kF = k0 + 1;
-            for (int j0 = 0; j0 <= ex2 - 2; ++j0) {
-                const int jF = j0 + 1;
-                for (int i0 = 0; i0 <= ex1 - 2; ++i0) {
-                    const int iF = i0 + 1;
-                    const size_t p = idx_ex(i0, j0, k0, ex);
-
-                    const double sfx = Sfx[p];
-                    if (sfx > ZEO) {
-                        /* 8th biased: -5*f(i-3)+60*f(i-2)-420*f(i-1)-378*f(i)+1050*f(i+1)-420*f(i+2)+140*f(i+3)-30*f(i+4)+3*f(i+5) */
-                        if (i0 <= ex1-6 && (i0-2)>=iminF) // i+5<=imax && i-3>=imin
-                            f_rhs[p] += sfx * d840dx * (
-                                -F5*fh[idx_fh_F_ord5(iF-3,jF,kF,ex)]+F60*fh[idx_fh_F_ord5(iF-2,jF,kF,ex)]
-                                -F420*fh[idx_fh_F_ord5(iF-1,jF,kF,ex)]-F378*fh[idx_fh_F_ord5(iF,jF,kF,ex)]
-                                +F1050*fh[idx_fh_F_ord5(iF+1,jF,kF,ex)]-F420*fh[idx_fh_F_ord5(iF+2,jF,kF,ex)]
-                                +F140*fh[idx_fh_F_ord5(iF+3,jF,kF,ex)]-F30*fh[idx_fh_F_ord5(iF+4,jF,kF,ex)]
-                                +F3*fh[idx_fh_F_ord5(iF+5,jF,kF,ex)]);
-                        /* 8th centered: +3*f(i-4)-32*f(i-3)+168*f(i-2)-672*f(i-1)+672*f(i+1)-168*f(i+2)+32*f(i+3)-3*f(i+4) */
-                        else if (i0 <= ex1-5 && (i0-3)>=iminF)
-                            f_rhs[p] += sfx * d840dx * (
-                                +F3*fh[idx_fh_F_ord5(iF-4,jF,kF,ex)]-F32*fh[idx_fh_F_ord5(iF-3,jF,kF,ex)]
-                                +F168*fh[idx_fh_F_ord5(iF-2,jF,kF,ex)]-F672*fh[idx_fh_F_ord5(iF-1,jF,kF,ex)]
-                                +F672*fh[idx_fh_F_ord5(iF+1,jF,kF,ex)]-F168*fh[idx_fh_F_ord5(iF+2,jF,kF,ex)]
-                                +F32*fh[idx_fh_F_ord5(iF+3,jF,kF,ex)]-F3*fh[idx_fh_F_ord5(iF+4,jF,kF,ex)]);
-                        else if (i0 <= ex1-4 && (i0-2)>=iminF) // 6th centered
-                            f_rhs[p] += sfx * d60dx * (
-                                -fh[idx_fh_F_ord5(iF-3,jF,kF,ex)]+F9*fh[idx_fh_F_ord5(iF-2,jF,kF,ex)]
-                                -F45*fh[idx_fh_F_ord5(iF-1,jF,kF,ex)]+F45*fh[idx_fh_F_ord5(iF+1,jF,kF,ex)]
-                                -F9*fh[idx_fh_F_ord5(iF+2,jF,kF,ex)]+fh[idx_fh_F_ord5(iF+3,jF,kF,ex)]);
-                        else if (i0 <= ex1-3 && (i0-1)>=iminF) // 4th centered
-                            f_rhs[p] += sfx * d12dx * (
-                                fh[idx_fh_F_ord5(iF-2,jF,kF,ex)]-EIT*fh[idx_fh_F_ord5(iF-1,jF,kF,ex)]
-                                +EIT*fh[idx_fh_F_ord5(iF+1,jF,kF,ex)]-fh[idx_fh_F_ord5(iF+2,jF,kF,ex)]);
-                        else if (i0 <= ex1-2 && i0>=iminF) // 2nd centered
-                            f_rhs[p] += sfx * d2dx * (
-                                -fh[idx_fh_F_ord5(iF-1,jF,kF,ex)]+fh[idx_fh_F_ord5(iF+1,jF,kF,ex)]);
-                    } else if (sfx < ZEO) {
-                        if ((i0-4)>=iminF && i0<=ex1-4)
-                            f_rhs[p] -= sfx * d840dx * (
-                                -F5*fh[idx_fh_F_ord5(iF+3,jF,kF,ex)]+F60*fh[idx_fh_F_ord5(iF+2,jF,kF,ex)]
-                                -F420*fh[idx_fh_F_ord5(iF+1,jF,kF,ex)]-F378*fh[idx_fh_F_ord5(iF,jF,kF,ex)]
-                                +F1050*fh[idx_fh_F_ord5(iF-1,jF,kF,ex)]-F420*fh[idx_fh_F_ord5(iF-2,jF,kF,ex)]
-                                +F140*fh[idx_fh_F_ord5(iF-3,jF,kF,ex)]-F30*fh[idx_fh_F_ord5(iF-4,jF,kF,ex)]
-                                +F3*fh[idx_fh_F_ord5(iF-5,jF,kF,ex)]);
-                        else if ((i0-3)>=iminF && i0<=ex1-5) // 8th centered
-                            f_rhs[p] += sfx * d840dx * (
-                                +F3*fh[idx_fh_F_ord5(iF-4,jF,kF,ex)]-F32*fh[idx_fh_F_ord5(iF-3,jF,kF,ex)]
-                                +F168*fh[idx_fh_F_ord5(iF-2,jF,kF,ex)]-F672*fh[idx_fh_F_ord5(iF-1,jF,kF,ex)]
-                                +F672*fh[idx_fh_F_ord5(iF+1,jF,kF,ex)]-F168*fh[idx_fh_F_ord5(iF+2,jF,kF,ex)]
-                                +F32*fh[idx_fh_F_ord5(iF+3,jF,kF,ex)]-F3*fh[idx_fh_F_ord5(iF+4,jF,kF,ex)]);
-                        else if ((i0-2)>=iminF && i0<=ex1-4) // 6th centered
-                            f_rhs[p] += sfx * d60dx * (
-                                -fh[idx_fh_F_ord5(iF-3,jF,kF,ex)]+F9*fh[idx_fh_F_ord5(iF-2,jF,kF,ex)]
-                                -F45*fh[idx_fh_F_ord5(iF-1,jF,kF,ex)]+F45*fh[idx_fh_F_ord5(iF+1,jF,kF,ex)]
-                                -F9*fh[idx_fh_F_ord5(iF+2,jF,kF,ex)]+fh[idx_fh_F_ord5(iF+3,jF,kF,ex)]);
-                        else if ((i0-1)>=iminF && i0<=ex1-3) // 4th centered
-                            f_rhs[p] += sfx * d12dx * (
-                                fh[idx_fh_F_ord5(iF-2,jF,kF,ex)]-EIT*fh[idx_fh_F_ord5(iF-1,jF,kF,ex)]
-                                +EIT*fh[idx_fh_F_ord5(iF+1,jF,kF,ex)]-fh[idx_fh_F_ord5(iF+2,jF,kF,ex)]);
-                        else if (i0>=iminF && i0<=ex1-2) // 2nd centered
-                            f_rhs[p] += sfx * d2dx * (
-                                -fh[idx_fh_F_ord5(iF-1,jF,kF,ex)]+fh[idx_fh_F_ord5(iF+1,jF,kF,ex)]);
-                    }
-
-                    const double sfy = Sfy[p];
-                    if (sfy > ZEO) {
-                        if (j0<=ex2-6 && (j0-2)>=jminF)
-                            f_rhs[p] += sfy * d840dy*(-F5*fh[idx_fh_F_ord5(iF,jF-3,kF,ex)]+F60*fh[idx_fh_F_ord5(iF,jF-2,kF,ex)]-F420*fh[idx_fh_F_ord5(iF,jF-1,kF,ex)]-F378*fh[idx_fh_F_ord5(iF,jF,kF,ex)]+F1050*fh[idx_fh_F_ord5(iF,jF+1,kF,ex)]-F420*fh[idx_fh_F_ord5(iF,jF+2,kF,ex)]+F140*fh[idx_fh_F_ord5(iF,jF+3,kF,ex)]-F30*fh[idx_fh_F_ord5(iF,jF+4,kF,ex)]+F3*fh[idx_fh_F_ord5(iF,jF+5,kF,ex)]);
-                        else if (j0<=ex2-5 && (j0-3)>=jminF)
-                            f_rhs[p] += sfy * d840dy*(+F3*fh[idx_fh_F_ord5(iF,jF-4,kF,ex)]-F32*fh[idx_fh_F_ord5(iF,jF-3,kF,ex)]+F168*fh[idx_fh_F_ord5(iF,jF-2,kF,ex)]-F672*fh[idx_fh_F_ord5(iF,jF-1,kF,ex)]+F672*fh[idx_fh_F_ord5(iF,jF+1,kF,ex)]-F168*fh[idx_fh_F_ord5(iF,jF+2,kF,ex)]+F32*fh[idx_fh_F_ord5(iF,jF+3,kF,ex)]-F3*fh[idx_fh_F_ord5(iF,jF+4,kF,ex)]);
-                        else if (j0<=ex2-4 && (j0-2)>=jminF)
-                            f_rhs[p] += sfy * d60dy*(-fh[idx_fh_F_ord5(iF,jF-3,kF,ex)]+F9*fh[idx_fh_F_ord5(iF,jF-2,kF,ex)]-F45*fh[idx_fh_F_ord5(iF,jF-1,kF,ex)]+F45*fh[idx_fh_F_ord5(iF,jF+1,kF,ex)]-F9*fh[idx_fh_F_ord5(iF,jF+2,kF,ex)]+fh[idx_fh_F_ord5(iF,jF+3,kF,ex)]);
-                        else if (j0<=ex2-3 && (j0-1)>=jminF)
-                            f_rhs[p] += sfy * d12dy*(fh[idx_fh_F_ord5(iF,jF-2,kF,ex)]-EIT*fh[idx_fh_F_ord5(iF,jF-1,kF,ex)]+EIT*fh[idx_fh_F_ord5(iF,jF+1,kF,ex)]-fh[idx_fh_F_ord5(iF,jF+2,kF,ex)]);
-                        else if (j0<=ex2-2 && j0>=jminF)
-                            f_rhs[p] += sfy * d2dy*(-fh[idx_fh_F_ord5(iF,jF-1,kF,ex)]+fh[idx_fh_F_ord5(iF,jF+1,kF,ex)]);
-                    } else if (sfy < ZEO) {
-                        if ((j0-4)>=jminF && j0<=ex2-4)
-                            f_rhs[p] -= sfy * d840dy*(-F5*fh[idx_fh_F_ord5(iF,jF+3,kF,ex)]+F60*fh[idx_fh_F_ord5(iF,jF+2,kF,ex)]-F420*fh[idx_fh_F_ord5(iF,jF+1,kF,ex)]-F378*fh[idx_fh_F_ord5(iF,jF,kF,ex)]+F1050*fh[idx_fh_F_ord5(iF,jF-1,kF,ex)]-F420*fh[idx_fh_F_ord5(iF,jF-2,kF,ex)]+F140*fh[idx_fh_F_ord5(iF,jF-3,kF,ex)]-F30*fh[idx_fh_F_ord5(iF,jF-4,kF,ex)]+F3*fh[idx_fh_F_ord5(iF,jF-5,kF,ex)]);
-                        else if ((j0-3)>=jminF && j0<=ex2-5)
-                            f_rhs[p] += sfy * d840dy*(+F3*fh[idx_fh_F_ord5(iF,jF-4,kF,ex)]-F32*fh[idx_fh_F_ord5(iF,jF-3,kF,ex)]+F168*fh[idx_fh_F_ord5(iF,jF-2,kF,ex)]-F672*fh[idx_fh_F_ord5(iF,jF-1,kF,ex)]+F672*fh[idx_fh_F_ord5(iF,jF+1,kF,ex)]-F168*fh[idx_fh_F_ord5(iF,jF+2,kF,ex)]+F32*fh[idx_fh_F_ord5(iF,jF+3,kF,ex)]-F3*fh[idx_fh_F_ord5(iF,jF+4,kF,ex)]);
-                        else if ((j0-2)>=jminF && j0<=ex2-4)
-                            f_rhs[p] += sfy * d60dy*(-fh[idx_fh_F_ord5(iF,jF-3,kF,ex)]+F9*fh[idx_fh_F_ord5(iF,jF-2,kF,ex)]-F45*fh[idx_fh_F_ord5(iF,jF-1,kF,ex)]+F45*fh[idx_fh_F_ord5(iF,jF+1,kF,ex)]-F9*fh[idx_fh_F_ord5(iF,jF+2,kF,ex)]+fh[idx_fh_F_ord5(iF,jF+3,kF,ex)]);
-                        else if ((j0-1)>=jminF && j0<=ex2-3)
-                            f_rhs[p] += sfy * d12dy*(fh[idx_fh_F_ord5(iF,jF-2,kF,ex)]-EIT*fh[idx_fh_F_ord5(iF,jF-1,kF,ex)]+EIT*fh[idx_fh_F_ord5(iF,jF+1,kF,ex)]-fh[idx_fh_F_ord5(iF,jF+2,kF,ex)]);
-                        else if (j0>=jminF && j0<=ex2-2)
-                            f_rhs[p] += sfy * d2dy*(-fh[idx_fh_F_ord5(iF,jF-1,kF,ex)]+fh[idx_fh_F_ord5(iF,jF+1,kF,ex)]);
-                    }
-
-                    const double sfz = Sfz[p];
-                    if (sfz > ZEO) {
-                        if (k0<=ex3-6 && (k0-2)>=kminF)
-                            f_rhs[p] += sfz * d840dz*(-F5*fh[idx_fh_F_ord5(iF,jF,kF-3,ex)]+F60*fh[idx_fh_F_ord5(iF,jF,kF-2,ex)]-F420*fh[idx_fh_F_ord5(iF,jF,kF-1,ex)]-F378*fh[idx_fh_F_ord5(iF,jF,kF,ex)]+F1050*fh[idx_fh_F_ord5(iF,jF,kF+1,ex)]-F420*fh[idx_fh_F_ord5(iF,jF,kF+2,ex)]+F140*fh[idx_fh_F_ord5(iF,jF,kF+3,ex)]-F30*fh[idx_fh_F_ord5(iF,jF,kF+4,ex)]+F3*fh[idx_fh_F_ord5(iF,jF,kF+5,ex)]);
-                        else if (k0<=ex3-5 && (k0-3)>=kminF)
-                            f_rhs[p] += sfz * d840dz*(+F3*fh[idx_fh_F_ord5(iF,jF,kF-4,ex)]-F32*fh[idx_fh_F_ord5(iF,jF,kF-3,ex)]+F168*fh[idx_fh_F_ord5(iF,jF,kF-2,ex)]-F672*fh[idx_fh_F_ord5(iF,jF,kF-1,ex)]+F672*fh[idx_fh_F_ord5(iF,jF,kF+1,ex)]-F168*fh[idx_fh_F_ord5(iF,jF,kF+2,ex)]+F32*fh[idx_fh_F_ord5(iF,jF,kF+3,ex)]-F3*fh[idx_fh_F_ord5(iF,jF,kF+4,ex)]);
-                        else if (k0<=ex3-4 && (k0-2)>=kminF)
-                            f_rhs[p] += sfz * d60dz*(-fh[idx_fh_F_ord5(iF,jF,kF-3,ex)]+F9*fh[idx_fh_F_ord5(iF,jF,kF-2,ex)]-F45*fh[idx_fh_F_ord5(iF,jF,kF-1,ex)]+F45*fh[idx_fh_F_ord5(iF,jF,kF+1,ex)]-F9*fh[idx_fh_F_ord5(iF,jF,kF+2,ex)]+fh[idx_fh_F_ord5(iF,jF,kF+3,ex)]);
-                        else if (k0<=ex3-3 && (k0-1)>=kminF)
-                            f_rhs[p] += sfz * d12dz*(fh[idx_fh_F_ord5(iF,jF,kF-2,ex)]-EIT*fh[idx_fh_F_ord5(iF,jF,kF-1,ex)]+EIT*fh[idx_fh_F_ord5(iF,jF,kF+1,ex)]-fh[idx_fh_F_ord5(iF,jF,kF+2,ex)]);
-                        else if (k0<=ex3-2 && k0>=kminF)
-                            f_rhs[p] += sfz * d2dz*(-fh[idx_fh_F_ord5(iF,jF,kF-1,ex)]+fh[idx_fh_F_ord5(iF,jF,kF+1,ex)]);
-                    } else if (sfz < ZEO) {
-                        if ((k0-4)>=kminF && k0<=ex3-4)
-                            f_rhs[p] -= sfz * d840dz*(-F5*fh[idx_fh_F_ord5(iF,jF,kF+3,ex)]+F60*fh[idx_fh_F_ord5(iF,jF,kF+2,ex)]-F420*fh[idx_fh_F_ord5(iF,jF,kF+1,ex)]-F378*fh[idx_fh_F_ord5(iF,jF,kF,ex)]+F1050*fh[idx_fh_F_ord5(iF,jF,kF-1,ex)]-F420*fh[idx_fh_F_ord5(iF,jF,kF-2,ex)]+F140*fh[idx_fh_F_ord5(iF,jF,kF-3,ex)]-F30*fh[idx_fh_F_ord5(iF,jF,kF-4,ex)]+F3*fh[idx_fh_F_ord5(iF,jF,kF-5,ex)]);
-                        else if ((k0-3)>=kminF && k0<=ex3-5)
-                            f_rhs[p] += sfz * d840dz*(+F3*fh[idx_fh_F_ord5(iF,jF,kF-4,ex)]-F32*fh[idx_fh_F_ord5(iF,jF,kF-3,ex)]+F168*fh[idx_fh_F_ord5(iF,jF,kF-2,ex)]-F672*fh[idx_fh_F_ord5(iF,jF,kF-1,ex)]+F672*fh[idx_fh_F_ord5(iF,jF,kF+1,ex)]-F168*fh[idx_fh_F_ord5(iF,jF,kF+2,ex)]+F32*fh[idx_fh_F_ord5(iF,jF,kF+3,ex)]-F3*fh[idx_fh_F_ord5(iF,jF,kF+4,ex)]);
-                        else if ((k0-2)>=kminF && k0<=ex3-4)
-                            f_rhs[p] += sfz * d60dz*(-fh[idx_fh_F_ord5(iF,jF,kF-3,ex)]+F9*fh[idx_fh_F_ord5(iF,jF,kF-2,ex)]-F45*fh[idx_fh_F_ord5(iF,jF,kF-1,ex)]+F45*fh[idx_fh_F_ord5(iF,jF,kF+1,ex)]-F9*fh[idx_fh_F_ord5(iF,jF,kF+2,ex)]+fh[idx_fh_F_ord5(iF,jF,kF+3,ex)]);
-                        else if ((k0-1)>=kminF && k0<=ex3-3)
-                            f_rhs[p] += sfz * d12dz*(fh[idx_fh_F_ord5(iF,jF,kF-2,ex)]-EIT*fh[idx_fh_F_ord5(iF,jF,kF-1,ex)]+EIT*fh[idx_fh_F_ord5(iF,jF,kF+1,ex)]-fh[idx_fh_F_ord5(iF,jF,kF+2,ex)]);
-                        else if (k0>=kminF && k0<=ex3-2)
-                            f_rhs[p] += sfz * d2dz*(-fh[idx_fh_F_ord5(iF,jF,kF-1,ex)]+fh[idx_fh_F_ord5(iF,jF,kF+1,ex)]);
-                    }
-                }
-            }
-        }
-        free(fh);
-        return;
-    }
-#else
-#error "lopsided_c.C: unsupported ghost_width (must be 2, 3, 4, or 5)"
-#endif
+    free(fh);
 }
+
+
+
+
+

AMSS_NCKU_source/lopsided_kodis_c.C

@@ -1,17 +1,8 @@
-#include "macrodef.h"
 #include "tool.h"
 
 /*
- * C 版 lopsided_kodis — combined upwind advection + KO dissipation.
- * Uses one shared symmetry_bd buffer (ord = ghost_width for both components)
- * where a stable merged stencil is available. The 8th-order path delegates to
- * the separate lopsided + kodis kernels, matching the original Fortran flow.
- *
- * FD order selection via ghost_width:
- *   2 → 2nd-order advection + r=2 KO (cof=16, sign=-)
- *   3 → 4th-order advection + r=3 KO (cof=64, sign=+)
- *   4 → 6th-order advection + r=4 KO (cof=256, sign=-)
- *   5 → 8th-order advection + r=5 KO (cof=1024, sign=+)
+ * Combined advection (lopsided) + KO dissipation (kodis).
+ * Uses one shared symmetry_bd buffer per call.
  */
 void lopsided_kodis(const int ex[3],
                     const double *X, const double *Y, const double *Z,
@@ -19,286 +10,239 @@ void lopsided_kodis(const int ex[3],
                     const double *Sfx, const double *Sfy, const double *Sfz,
                     int Symmetry, const double SoA[3], double eps)
 {
-    const double ZEO = 0.0, ONE = 1.0;
-    const double TWO = 2.0, F6 = 6.0, EIT = 8.0;
-    const double F3 = 3.0, F4 = 4.0, F5 = 5.0, F10 = 10.0, F12 = 12.0, F18 = 18.0;
-    const double F9 = 9.0, F45 = 45.0, F60 = 60.0;
-    const double F2 = 2.0, F15 = 15.0, F24 = 24.0, F30 = 30.0, F35 = 35.0;
-    const double F50 = 50.0, F77 = 77.0, F80 = 80.0, F100 = 100.0, F150 = 150.0;
-    const double F32 = 32.0, F168 = 168.0, F672 = 672.0, F840 = 840.0;
-    const double F140=140.0, F378=378.0, F420=420.0, F1050=1050.0;
+    const double ZEO = 0.0, ONE = 1.0, F3 = 3.0;
+    const double F6 = 6.0, F18 = 18.0;
+    const double F12 = 12.0, F10 = 10.0, EIT = 8.0;
+    const double SIX = 6.0, FIT = 15.0, TWT = 20.0;
+    const double cof = 64.0; // 2^6
 
     const int NO_SYMM = 0, EQ_SYMM = 1;
+
     const int ex1 = ex[0], ex2 = ex[1], ex3 = ex[2];
 
     const double dX = X[1] - X[0];
     const double dY = Y[1] - Y[0];
     const double dZ = Z[1] - Z[0];
 
-    const int imaxF = ex1, jmaxF = ex2, kmaxF = ex3;
+    const double d12dx = ONE / F12 / dX;
+    const double d12dy = ONE / F12 / dY;
+    const double d12dz = ONE / F12 / dZ;
 
-#if (ghost_width == 2)
-    {
-        const int ord = 2;
-        int iminF = 1, jminF = 1, kminF = 1;
-        if (Symmetry > NO_SYMM && fabs(Z[0]) < dZ) kminF = -1;
-        if (Symmetry > EQ_SYMM && fabs(X[0]) < dX) iminF = -1;
-        if (Symmetry > EQ_SYMM && fabs(Y[0]) < dY) jminF = -1;
+    const int imaxF = ex1;
+    const int jmaxF = ex2;
+    const int kmaxF = ex3;
 
-        const size_t nx = (size_t)ex1 + ord;
-        const size_t ny = (size_t)ex2 + ord;
-        const size_t nz = (size_t)ex3 + ord;
-        double *fh = (double*)malloc(nx*ny*nz*sizeof(double));
-        if (!fh) return;
-        symmetry_bd(ord, ex, f, fh, SoA);
+    int iminF = 1, jminF = 1, kminF = 1;
+    if (Symmetry > NO_SYMM && fabs(Z[0]) < dZ) kminF = -2;
+    if (Symmetry > EQ_SYMM && fabs(X[0]) < dX) iminF = -2;
+    if (Symmetry > EQ_SYMM && fabs(Y[0]) < dY) jminF = -2;
 
-        const double d2dx = ONE/TWO/dX, d2dy = ONE/TWO/dY, d2dz = ONE/TWO/dZ;
+    // fh for Fortran-style domain (-2:ex1,-2:ex2,-2:ex3)
+    const size_t nx = (size_t)ex1 + 3;
+    const size_t ny = (size_t)ex2 + 3;
+    const size_t nz = (size_t)ex3 + 3;
+    const size_t fh_size = nx * ny * nz;
 
-        /* ---- advection (2nd-order) ---- */
-        for (int k0 = 0; k0 <= ex3-2; ++k0) {
-            const int kF = k0+1;
-            for (int j0 = 0; j0 <= ex2-2; ++j0) {
-                const int jF = j0+1;
-                for (int i0 = 0; i0 <= ex1-2; ++i0) {
-                    const int iF = i0+1;
-                    const size_t p = idx_ex(i0,j0,k0,ex);
+    double *fh = (double*)malloc(fh_size * sizeof(double));
+    if (!fh) return;
 
-                    const double sfx = Sfx[p];
-                    if (sfx > ZEO) {
-                        if (i0<=ex1-3) f_rhs[p] += sfx*d2dx*(-F3*fh[idx_fh_F_ord2(iF,jF,kF,ex)]+F4*fh[idx_fh_F_ord2(iF+1,jF,kF,ex)]-fh[idx_fh_F_ord2(iF+2,jF,kF,ex)]);
-                        else if (i0<=ex1-2) f_rhs[p] += sfx*d2dx*(-fh[idx_fh_F_ord2(iF,jF,kF,ex)]+fh[idx_fh_F_ord2(iF+1,jF,kF,ex)]);
-                    } else if (sfx < ZEO) {
-                        if ((i0-1)>=iminF) f_rhs[p] -= sfx*d2dx*(-F3*fh[idx_fh_F_ord2(iF,jF,kF,ex)]+F4*fh[idx_fh_F_ord2(iF-1,jF,kF,ex)]-fh[idx_fh_F_ord2(iF-2,jF,kF,ex)]);
-                        else if (i0>=iminF) f_rhs[p] -= sfx*d2dx*(-fh[idx_fh_F_ord2(iF,jF,kF,ex)]+fh[idx_fh_F_ord2(iF-1,jF,kF,ex)]);
+    symmetry_bd(3, ex, f, fh, SoA);
+
+    // Advection (same stencil logic as lopsided_c.C)
+    for (int k0 = 0; k0 <= ex3 - 2; ++k0) {
+        const int kF = k0 + 1;
+        for (int j0 = 0; j0 <= ex2 - 2; ++j0) {
+            const int jF = j0 + 1;
+            for (int i0 = 0; i0 <= ex1 - 2; ++i0) {
+                const int iF = i0 + 1;
+                const size_t p = idx_ex(i0, j0, k0, ex);
+
+                const double sfx = Sfx[p];
+                if (sfx > ZEO) {
+                    if (i0 <= ex1 - 4) {
+                        f_rhs[p] += sfx * d12dx *
+                            (-F3  * fh[idx_fh_F(iF - 1, jF, kF, ex)]
+                             -F10 * fh[idx_fh_F(iF    , jF, kF, ex)]
+                             +F18 * fh[idx_fh_F(iF + 1, jF, kF, ex)]
+                             -F6  * fh[idx_fh_F(iF + 2, jF, kF, ex)]
+                             +      fh[idx_fh_F(iF + 3, jF, kF, ex)]);
+                    } else if (i0 <= ex1 - 3) {
+                        f_rhs[p] += sfx * d12dx *
+                            ( fh[idx_fh_F(iF - 2, jF, kF, ex)]
+                             -EIT * fh[idx_fh_F(iF - 1, jF, kF, ex)]
+                             +EIT * fh[idx_fh_F(iF + 1, jF, kF, ex)]
+                             -      fh[idx_fh_F(iF + 2, jF, kF, ex)]);
+                    } else if (i0 <= ex1 - 2) {
+                        f_rhs[p] -= sfx * d12dx *
+                            (-F3  * fh[idx_fh_F(iF + 1, jF, kF, ex)]
+                             -F10 * fh[idx_fh_F(iF    , jF, kF, ex)]
+                             +F18 * fh[idx_fh_F(iF - 1, jF, kF, ex)]
+                             -F6  * fh[idx_fh_F(iF - 2, jF, kF, ex)]
+                             +      fh[idx_fh_F(iF - 3, jF, kF, ex)]);
                     }
-                    const double sfy = Sfy[p];
-                    if (sfy > ZEO) {
-                        if (j0<=ex2-3) f_rhs[p] += sfy*d2dy*(-F3*fh[idx_fh_F_ord2(iF,jF,kF,ex)]+F4*fh[idx_fh_F_ord2(iF,jF+1,kF,ex)]-fh[idx_fh_F_ord2(iF,jF+2,kF,ex)]);
-                        else if (j0<=ex2-2) f_rhs[p] += sfy*d2dy*(-fh[idx_fh_F_ord2(iF,jF,kF,ex)]+fh[idx_fh_F_ord2(iF,jF+1,kF,ex)]);
-                    } else if (sfy < ZEO) {
-                        if ((j0-1)>=jminF) f_rhs[p] -= sfy*d2dy*(-F3*fh[idx_fh_F_ord2(iF,jF,kF,ex)]+F4*fh[idx_fh_F_ord2(iF,jF-1,kF,ex)]-fh[idx_fh_F_ord2(iF,jF-2,kF,ex)]);
-                        else if (j0>=jminF) f_rhs[p] -= sfy*d2dy*(-fh[idx_fh_F_ord2(iF,jF,kF,ex)]+fh[idx_fh_F_ord2(iF,jF-1,kF,ex)]);
+                } else if (sfx < ZEO) {
+                    if ((i0 - 2) >= iminF) {
+                        f_rhs[p] -= sfx * d12dx *
+                            (-F3  * fh[idx_fh_F(iF + 1, jF, kF, ex)]
+                             -F10 * fh[idx_fh_F(iF    , jF, kF, ex)]
+                             +F18 * fh[idx_fh_F(iF - 1, jF, kF, ex)]
+                             -F6  * fh[idx_fh_F(iF - 2, jF, kF, ex)]
+                             +      fh[idx_fh_F(iF - 3, jF, kF, ex)]);
+                    } else if ((i0 - 1) >= iminF) {
+                        f_rhs[p] += sfx * d12dx *
+                            ( fh[idx_fh_F(iF - 2, jF, kF, ex)]
+                             -EIT * fh[idx_fh_F(iF - 1, jF, kF, ex)]
+                             +EIT * fh[idx_fh_F(iF + 1, jF, kF, ex)]
+                             -      fh[idx_fh_F(iF + 2, jF, kF, ex)]);
+                    } else if (i0 >= iminF) {
+                        f_rhs[p] += sfx * d12dx *
+                            (-F3  * fh[idx_fh_F(iF - 1, jF, kF, ex)]
+                             -F10 * fh[idx_fh_F(iF    , jF, kF, ex)]
+                             +F18 * fh[idx_fh_F(iF + 1, jF, kF, ex)]
+                             -F6  * fh[idx_fh_F(iF + 2, jF, kF, ex)]
+                             +      fh[idx_fh_F(iF + 3, jF, kF, ex)]);
                     }
-                    const double sfz = Sfz[p];
-                    if (sfz > ZEO) {
-                        if (k0<=ex3-3) f_rhs[p] += sfz*d2dz*(-F3*fh[idx_fh_F_ord2(iF,jF,kF,ex)]+F4*fh[idx_fh_F_ord2(iF,jF,kF+1,ex)]-fh[idx_fh_F_ord2(iF,jF,kF+2,ex)]);
-                        else if (k0<=ex3-2) f_rhs[p] += sfz*d2dz*(-fh[idx_fh_F_ord2(iF,jF,kF,ex)]+fh[idx_fh_F_ord2(iF,jF,kF+1,ex)]);
-                    } else if (sfz < ZEO) {
-                        if ((k0-1)>=kminF) f_rhs[p] -= sfz*d2dz*(-F3*fh[idx_fh_F_ord2(iF,jF,kF,ex)]+F4*fh[idx_fh_F_ord2(iF,jF,kF-1,ex)]-fh[idx_fh_F_ord2(iF,jF,kF-2,ex)]);
-                        else if (k0>=kminF) f_rhs[p] -= sfz*d2dz*(-fh[idx_fh_F_ord2(iF,jF,kF,ex)]+fh[idx_fh_F_ord2(iF,jF,kF-1,ex)]);
+                }
+
+                const double sfy = Sfy[p];
+                if (sfy > ZEO) {
+                    if (j0 <= ex2 - 4) {
+                        f_rhs[p] += sfy * d12dy *
+                            (-F3  * fh[idx_fh_F(iF, jF - 1, kF, ex)]
+                             -F10 * fh[idx_fh_F(iF, jF    , kF, ex)]
+                             +F18 * fh[idx_fh_F(iF, jF + 1, kF, ex)]
+                             -F6  * fh[idx_fh_F(iF, jF + 2, kF, ex)]
+                             +      fh[idx_fh_F(iF, jF + 3, kF, ex)]);
+                    } else if (j0 <= ex2 - 3) {
+                        f_rhs[p] += sfy * d12dy *
+                            ( fh[idx_fh_F(iF, jF - 2, kF, ex)]
+                             -EIT * fh[idx_fh_F(iF, jF - 1, kF, ex)]
+                             +EIT * fh[idx_fh_F(iF, jF + 1, kF, ex)]
+                             -      fh[idx_fh_F(iF, jF + 2, kF, ex)]);
+                    } else if (j0 <= ex2 - 2) {
+                        f_rhs[p] -= sfy * d12dy *
+                            (-F3  * fh[idx_fh_F(iF, jF + 1, kF, ex)]
+                             -F10 * fh[idx_fh_F(iF, jF    , kF, ex)]
+                             +F18 * fh[idx_fh_F(iF, jF - 1, kF, ex)]
+                             -F6  * fh[idx_fh_F(iF, jF - 2, kF, ex)]
+                             +      fh[idx_fh_F(iF, jF - 3, kF, ex)]);
+                    }
+                } else if (sfy < ZEO) {
+                    if ((j0 - 2) >= jminF) {
+                        f_rhs[p] -= sfy * d12dy *
+                            (-F3  * fh[idx_fh_F(iF, jF + 1, kF, ex)]
+                             -F10 * fh[idx_fh_F(iF, jF    , kF, ex)]
+                             +F18 * fh[idx_fh_F(iF, jF - 1, kF, ex)]
+                             -F6  * fh[idx_fh_F(iF, jF - 2, kF, ex)]
+                             +      fh[idx_fh_F(iF, jF - 3, kF, ex)]);
+                    } else if ((j0 - 1) >= jminF) {
+                        f_rhs[p] += sfy * d12dy *
+                            ( fh[idx_fh_F(iF, jF - 2, kF, ex)]
+                             -EIT * fh[idx_fh_F(iF, jF - 1, kF, ex)]
+                             +EIT * fh[idx_fh_F(iF, jF + 1, kF, ex)]
+                             -      fh[idx_fh_F(iF, jF + 2, kF, ex)]);
+                    } else if (j0 >= jminF) {
+                        f_rhs[p] += sfy * d12dy *
+                            (-F3  * fh[idx_fh_F(iF, jF - 1, kF, ex)]
+                             -F10 * fh[idx_fh_F(iF, jF    , kF, ex)]
+                             +F18 * fh[idx_fh_F(iF, jF + 1, kF, ex)]
+                             -F6  * fh[idx_fh_F(iF, jF + 2, kF, ex)]
+                             +      fh[idx_fh_F(iF, jF + 3, kF, ex)]);
+                    }
+                }
+
+                const double sfz = Sfz[p];
+                if (sfz > ZEO) {
+                    if (k0 <= ex3 - 4) {
+                        f_rhs[p] += sfz * d12dz *
+                            (-F3  * fh[idx_fh_F(iF, jF, kF - 1, ex)]
+                             -F10 * fh[idx_fh_F(iF, jF, kF    , ex)]
+                             +F18 * fh[idx_fh_F(iF, jF, kF + 1, ex)]
+                             -F6  * fh[idx_fh_F(iF, jF, kF + 2, ex)]
+                             +      fh[idx_fh_F(iF, jF, kF + 3, ex)]);
+                    } else if (k0 <= ex3 - 3) {
+                        f_rhs[p] += sfz * d12dz *
+                            ( fh[idx_fh_F(iF, jF, kF - 2, ex)]
+                             -EIT * fh[idx_fh_F(iF, jF, kF - 1, ex)]
+                             +EIT * fh[idx_fh_F(iF, jF, kF + 1, ex)]
+                             -      fh[idx_fh_F(iF, jF, kF + 2, ex)]);
+                    } else if (k0 <= ex3 - 2) {
+                        f_rhs[p] -= sfz * d12dz *
+                            (-F3  * fh[idx_fh_F(iF, jF, kF + 1, ex)]
+                             -F10 * fh[idx_fh_F(iF, jF, kF    , ex)]
+                             +F18 * fh[idx_fh_F(iF, jF, kF - 1, ex)]
+                             -F6  * fh[idx_fh_F(iF, jF, kF - 2, ex)]
+                             +      fh[idx_fh_F(iF, jF, kF - 3, ex)]);
+                    }
+                } else if (sfz < ZEO) {
+                    if ((k0 - 2) >= kminF) {
+                        f_rhs[p] -= sfz * d12dz *
+                            (-F3  * fh[idx_fh_F(iF, jF, kF + 1, ex)]
+                             -F10 * fh[idx_fh_F(iF, jF, kF    , ex)]
+                             +F18 * fh[idx_fh_F(iF, jF, kF - 1, ex)]
+                             -F6  * fh[idx_fh_F(iF, jF, kF - 2, ex)]
+                             +      fh[idx_fh_F(iF, jF, kF - 3, ex)]);
+                    } else if ((k0 - 1) >= kminF) {
+                        f_rhs[p] += sfz * d12dz *
+                            ( fh[idx_fh_F(iF, jF, kF - 2, ex)]
+                             -EIT * fh[idx_fh_F(iF, jF, kF - 1, ex)]
+                             +EIT * fh[idx_fh_F(iF, jF, kF + 1, ex)]
+                             -      fh[idx_fh_F(iF, jF, kF + 2, ex)]);
+                    } else if (k0 >= kminF) {
+                        f_rhs[p] += sfz * d12dz *
+                            (-F3  * fh[idx_fh_F(iF, jF, kF - 1, ex)]
+                             -F10 * fh[idx_fh_F(iF, jF, kF    , ex)]
+                             +F18 * fh[idx_fh_F(iF, jF, kF + 1, ex)]
+                             -F6  * fh[idx_fh_F(iF, jF, kF + 2, ex)]
+                             +      fh[idx_fh_F(iF, jF, kF + 3, ex)]);
                     }
                 }
             }
         }
-
-        /* ---- KO dissipation (r=2, cof=16, sign=-) ---- */
-        if (eps > ZEO) {
-            const double cof = 16.0;
-            const double F4k = 4.0, F6k = 6.0;
-            const int i0_lo = (iminF+1>0)?iminF+1:0, j0_lo=(jminF+1>0)?jminF+1:0, k0_lo=(kminF+1>0)?kminF+1:0;
-            const int i0_hi=imaxF-3, j0_hi=jmaxF-3, k0_hi=kmaxF-3;
-            if (!(i0_lo>i0_hi||j0_lo>j0_hi||k0_lo>k0_hi)) {
-                for (int k0=k0_lo;k0<=k0_hi;++k0) { const int kF=k0+1;
-                for (int j0=j0_lo;j0<=j0_hi;++j0) { const int jF=j0+1;
-                for (int i0=i0_lo;i0<=i0_hi;++i0) { const int iF=i0+1;
-                    const size_t p=idx_ex(i0,j0,k0,ex);
-                    const double Dx=((fh[idx_fh_F_ord2(iF-2,jF,kF,ex)]+fh[idx_fh_F_ord2(iF+2,jF,kF,ex)])-F4k*(fh[idx_fh_F_ord2(iF-1,jF,kF,ex)]+fh[idx_fh_F_ord2(iF+1,jF,kF,ex)])+F6k*fh[idx_fh_F_ord2(iF,jF,kF,ex)])/dX;
-                    const double Dy=((fh[idx_fh_F_ord2(iF,jF-2,kF,ex)]+fh[idx_fh_F_ord2(iF,jF+2,kF,ex)])-F4k*(fh[idx_fh_F_ord2(iF,jF-1,kF,ex)]+fh[idx_fh_F_ord2(iF,jF+1,kF,ex)])+F6k*fh[idx_fh_F_ord2(iF,jF,kF,ex)])/dY;
-                    const double Dz=((fh[idx_fh_F_ord2(iF,jF,kF-2,ex)]+fh[idx_fh_F_ord2(iF,jF,kF+2,ex)])-F4k*(fh[idx_fh_F_ord2(iF,jF,kF-1,ex)]+fh[idx_fh_F_ord2(iF,jF,kF+1,ex)])+F6k*fh[idx_fh_F_ord2(iF,jF,kF,ex)])/dZ;
-                    f_rhs[p] -= (eps/cof)*(Dx+Dy+Dz);
-                }}}
-            }
-        }
-        free(fh);
-        return;
     }
-#elif (ghost_width == 3)
-    /* ---- 4th-order advection + r=3 KO (original code) ----------------- */
-    {
-        const int ord = 3;
-        int iminF = 1, jminF = 1, kminF = 1;
-        if (Symmetry > NO_SYMM && fabs(Z[0]) < dZ) kminF = -2;
-        if (Symmetry > EQ_SYMM && fabs(X[0]) < dX) iminF = -2;
-        if (Symmetry > EQ_SYMM && fabs(Y[0]) < dY) jminF = -2;
 
-        const size_t nx = (size_t)ex1 + ord;
-        const size_t ny = (size_t)ex2 + ord;
-        const size_t nz = (size_t)ex3 + ord;
-        double *fh = (double*)malloc(nx*ny*nz*sizeof(double));
-        if (!fh) return;
-        symmetry_bd(ord, ex, f, fh, SoA);
+    // KO dissipation (same domain restriction as kodiss_c.C)
+    if (eps > ZEO) {
+        const int i0_lo = (iminF + 2 > 0) ? iminF + 2 : 0;
+        const int j0_lo = (jminF + 2 > 0) ? jminF + 2 : 0;
+        const int k0_lo = (kminF + 2 > 0) ? kminF + 2 : 0;
+        const int i0_hi = imaxF - 4; // inclusive
+        const int j0_hi = jmaxF - 4;
+        const int k0_hi = kmaxF - 4;
 
-        const double d12dx = ONE/F12/dX, d12dy = ONE/F12/dY, d12dz = ONE/F12/dZ;
+        if (!(i0_lo > i0_hi || j0_lo > j0_hi || k0_lo > k0_hi)) {
+            for (int k0 = k0_lo; k0 <= k0_hi; ++k0) {
+                const int kF = k0 + 1;
+                for (int j0 = j0_lo; j0 <= j0_hi; ++j0) {
+                    const int jF = j0 + 1;
+                    for (int i0 = i0_lo; i0 <= i0_hi; ++i0) {
+                        const int iF = i0 + 1;
+                        const size_t p = idx_ex(i0, j0, k0, ex);
 
-        /* ---- advection ---- */
-        for (int k0 = 0; k0 <= ex3-2; ++k0) {
-            const int kF = k0+1;
-            for (int j0 = 0; j0 <= ex2-2; ++j0) {
-                const int jF = j0+1;
-                for (int i0 = 0; i0 <= ex1-2; ++i0) {
-                    const int iF = i0+1;
-                    const size_t p = idx_ex(i0,j0,k0,ex);
+                        const double Dx_term =
+                            ((fh[idx_fh_F(iF - 3, jF, kF, ex)] + fh[idx_fh_F(iF + 3, jF, kF, ex)]) -
+                             SIX * (fh[idx_fh_F(iF - 2, jF, kF, ex)] + fh[idx_fh_F(iF + 2, jF, kF, ex)]) +
+                             FIT * (fh[idx_fh_F(iF - 1, jF, kF, ex)] + fh[idx_fh_F(iF + 1, jF, kF, ex)]) -
+                             TWT *  fh[idx_fh_F(iF,     jF, kF, ex)]) / dX;
 
-                    const double sfx = Sfx[p];
-                    if (sfx > ZEO) {
-                        if (i0 <= ex1-4)
-                            f_rhs[p] += sfx*d12dx*(-F3*fh[idx_fh_F(iF-1,jF,kF,ex)]-F10*fh[idx_fh_F(iF,jF,kF,ex)]+F18*fh[idx_fh_F(iF+1,jF,kF,ex)]-F6*fh[idx_fh_F(iF+2,jF,kF,ex)]+fh[idx_fh_F(iF+3,jF,kF,ex)]);
-                        else if (i0 <= ex1-3)
-                            f_rhs[p] += sfx*d12dx*(fh[idx_fh_F(iF-2,jF,kF,ex)]-EIT*fh[idx_fh_F(iF-1,jF,kF,ex)]+EIT*fh[idx_fh_F(iF+1,jF,kF,ex)]-fh[idx_fh_F(iF+2,jF,kF,ex)]);
-                        else if (i0 <= ex1-2)
-                            f_rhs[p] -= sfx*d12dx*(-F3*fh[idx_fh_F(iF+1,jF,kF,ex)]-F10*fh[idx_fh_F(iF,jF,kF,ex)]+F18*fh[idx_fh_F(iF-1,jF,kF,ex)]-F6*fh[idx_fh_F(iF-2,jF,kF,ex)]+fh[idx_fh_F(iF-3,jF,kF,ex)]);
-                    } else if (sfx < ZEO) {
-                        if ((i0-2) >= iminF)
-                            f_rhs[p] -= sfx*d12dx*(-F3*fh[idx_fh_F(iF+1,jF,kF,ex)]-F10*fh[idx_fh_F(iF,jF,kF,ex)]+F18*fh[idx_fh_F(iF-1,jF,kF,ex)]-F6*fh[idx_fh_F(iF-2,jF,kF,ex)]+fh[idx_fh_F(iF-3,jF,kF,ex)]);
-                        else if ((i0-1) >= iminF)
-                            f_rhs[p] += sfx*d12dx*(fh[idx_fh_F(iF-2,jF,kF,ex)]-EIT*fh[idx_fh_F(iF-1,jF,kF,ex)]+EIT*fh[idx_fh_F(iF+1,jF,kF,ex)]-fh[idx_fh_F(iF+2,jF,kF,ex)]);
-                        else if (i0 >= iminF)
-                            f_rhs[p] += sfx*d12dx*(-F3*fh[idx_fh_F(iF-1,jF,kF,ex)]-F10*fh[idx_fh_F(iF,jF,kF,ex)]+F18*fh[idx_fh_F(iF+1,jF,kF,ex)]-F6*fh[idx_fh_F(iF+2,jF,kF,ex)]+fh[idx_fh_F(iF+3,jF,kF,ex)]);
-                    }
-                    const double sfy = Sfy[p];
-                    if (sfy > ZEO) {
-                        if (j0<=ex2-4) f_rhs[p] += sfy*d12dy*(-F3*fh[idx_fh_F(iF,jF-1,kF,ex)]-F10*fh[idx_fh_F(iF,jF,kF,ex)]+F18*fh[idx_fh_F(iF,jF+1,kF,ex)]-F6*fh[idx_fh_F(iF,jF+2,kF,ex)]+fh[idx_fh_F(iF,jF+3,kF,ex)]);
-                        else if (j0<=ex2-3) f_rhs[p] += sfy*d12dy*(fh[idx_fh_F(iF,jF-2,kF,ex)]-EIT*fh[idx_fh_F(iF,jF-1,kF,ex)]+EIT*fh[idx_fh_F(iF,jF+1,kF,ex)]-fh[idx_fh_F(iF,jF+2,kF,ex)]);
-                        else if (j0<=ex2-2) f_rhs[p] -= sfy*d12dy*(-F3*fh[idx_fh_F(iF,jF+1,kF,ex)]-F10*fh[idx_fh_F(iF,jF,kF,ex)]+F18*fh[idx_fh_F(iF,jF-1,kF,ex)]-F6*fh[idx_fh_F(iF,jF-2,kF,ex)]+fh[idx_fh_F(iF,jF-3,kF,ex)]);
-                    } else if (sfy < ZEO) {
-                        if ((j0-2)>=jminF) f_rhs[p] -= sfy*d12dy*(-F3*fh[idx_fh_F(iF,jF+1,kF,ex)]-F10*fh[idx_fh_F(iF,jF,kF,ex)]+F18*fh[idx_fh_F(iF,jF-1,kF,ex)]-F6*fh[idx_fh_F(iF,jF-2,kF,ex)]+fh[idx_fh_F(iF,jF-3,kF,ex)]);
-                        else if ((j0-1)>=jminF) f_rhs[p] += sfy*d12dy*(fh[idx_fh_F(iF,jF-2,kF,ex)]-EIT*fh[idx_fh_F(iF,jF-1,kF,ex)]+EIT*fh[idx_fh_F(iF,jF+1,kF,ex)]-fh[idx_fh_F(iF,jF+2,kF,ex)]);
-                        else if (j0>=jminF) f_rhs[p] += sfy*d12dy*(-F3*fh[idx_fh_F(iF,jF-1,kF,ex)]-F10*fh[idx_fh_F(iF,jF,kF,ex)]+F18*fh[idx_fh_F(iF,jF+1,kF,ex)]-F6*fh[idx_fh_F(iF,jF+2,kF,ex)]+fh[idx_fh_F(iF,jF+3,kF,ex)]);
-                    }
-                    const double sfz = Sfz[p];
-                    if (sfz > ZEO) {
-                        if (k0<=ex3-4) f_rhs[p] += sfz*d12dz*(-F3*fh[idx_fh_F(iF,jF,kF-1,ex)]-F10*fh[idx_fh_F(iF,jF,kF,ex)]+F18*fh[idx_fh_F(iF,jF,kF+1,ex)]-F6*fh[idx_fh_F(iF,jF,kF+2,ex)]+fh[idx_fh_F(iF,jF,kF+3,ex)]);
-                        else if (k0<=ex3-3) f_rhs[p] += sfz*d12dz*(fh[idx_fh_F(iF,jF,kF-2,ex)]-EIT*fh[idx_fh_F(iF,jF,kF-1,ex)]+EIT*fh[idx_fh_F(iF,jF,kF+1,ex)]-fh[idx_fh_F(iF,jF,kF+2,ex)]);
-                        else if (k0<=ex3-2) f_rhs[p] -= sfz*d12dz*(-F3*fh[idx_fh_F(iF,jF,kF+1,ex)]-F10*fh[idx_fh_F(iF,jF,kF,ex)]+F18*fh[idx_fh_F(iF,jF,kF-1,ex)]-F6*fh[idx_fh_F(iF,jF,kF-2,ex)]+fh[idx_fh_F(iF,jF,kF-3,ex)]);
-                    } else if (sfz < ZEO) {
-                        if ((k0-2)>=kminF) f_rhs[p] -= sfz*d12dz*(-F3*fh[idx_fh_F(iF,jF,kF+1,ex)]-F10*fh[idx_fh_F(iF,jF,kF,ex)]+F18*fh[idx_fh_F(iF,jF,kF-1,ex)]-F6*fh[idx_fh_F(iF,jF,kF-2,ex)]+fh[idx_fh_F(iF,jF,kF-3,ex)]);
-                        else if ((k0-1)>=kminF) f_rhs[p] += sfz*d12dz*(fh[idx_fh_F(iF,jF,kF-2,ex)]-EIT*fh[idx_fh_F(iF,jF,kF-1,ex)]+EIT*fh[idx_fh_F(iF,jF,kF+1,ex)]-fh[idx_fh_F(iF,jF,kF+2,ex)]);
-                        else if (k0>=kminF) f_rhs[p] += sfz*d12dz*(-F3*fh[idx_fh_F(iF,jF,kF-1,ex)]-F10*fh[idx_fh_F(iF,jF,kF,ex)]+F18*fh[idx_fh_F(iF,jF,kF+1,ex)]-F6*fh[idx_fh_F(iF,jF,kF+2,ex)]+fh[idx_fh_F(iF,jF,kF+3,ex)]);
+                        const double Dy_term =
+                            ((fh[idx_fh_F(iF, jF - 3, kF, ex)] + fh[idx_fh_F(iF, jF + 3, kF, ex)]) -
+                             SIX * (fh[idx_fh_F(iF, jF - 2, kF, ex)] + fh[idx_fh_F(iF, jF + 2, kF, ex)]) +
+                             FIT * (fh[idx_fh_F(iF, jF - 1, kF, ex)] + fh[idx_fh_F(iF, jF + 1, kF, ex)]) -
+                             TWT *  fh[idx_fh_F(iF, jF,     kF, ex)]) / dY;
+
+                        const double Dz_term =
+                            ((fh[idx_fh_F(iF, jF, kF - 3, ex)] + fh[idx_fh_F(iF, jF, kF + 3, ex)]) -
+                             SIX * (fh[idx_fh_F(iF, jF, kF - 2, ex)] + fh[idx_fh_F(iF, jF, kF + 2, ex)]) +
+                             FIT * (fh[idx_fh_F(iF, jF, kF - 1, ex)] + fh[idx_fh_F(iF, jF, kF + 1, ex)]) -
+                             TWT *  fh[idx_fh_F(iF, jF, kF,     ex)]) / dZ;
+
+                        f_rhs[p] += (eps / cof) * (Dx_term + Dy_term + Dz_term);
                     }
                 }
             }
         }
-
-        /* ---- KO dissipation (r=3, cof=64, sign=+) ---- */
-        if (eps > ZEO) {
-            const double cof = 64.0;
-            const double SIX = 6.0, FIT = 15.0, TWT = 20.0;
-            const int i0_lo=(iminF+2>0)?iminF+2:0, j0_lo=(jminF+2>0)?jminF+2:0, k0_lo=(kminF+2>0)?kminF+2:0;
-            const int i0_hi=imaxF-4, j0_hi=jmaxF-4, k0_hi=kmaxF-4;
-            if (!(i0_lo>i0_hi||j0_lo>j0_hi||k0_lo>k0_hi)) {
-                for (int k0=k0_lo;k0<=k0_hi;++k0) { const int kF=k0+1;
-                for (int j0=j0_lo;j0<=j0_hi;++j0) { const int jF=j0+1;
-                for (int i0=i0_lo;i0<=i0_hi;++i0) { const int iF=i0+1;
-                    const size_t p=idx_ex(i0,j0,k0,ex);
-                    const double Dx=((fh[idx_fh_F(iF-3,jF,kF,ex)]+fh[idx_fh_F(iF+3,jF,kF,ex)])-SIX*(fh[idx_fh_F(iF-2,jF,kF,ex)]+fh[idx_fh_F(iF+2,jF,kF,ex)])+FIT*(fh[idx_fh_F(iF-1,jF,kF,ex)]+fh[idx_fh_F(iF+1,jF,kF,ex)])-TWT*fh[idx_fh_F(iF,jF,kF,ex)])/dX;
-                    const double Dy=((fh[idx_fh_F(iF,jF-3,kF,ex)]+fh[idx_fh_F(iF,jF+3,kF,ex)])-SIX*(fh[idx_fh_F(iF,jF-2,kF,ex)]+fh[idx_fh_F(iF,jF+2,kF,ex)])+FIT*(fh[idx_fh_F(iF,jF-1,kF,ex)]+fh[idx_fh_F(iF,jF+1,kF,ex)])-TWT*fh[idx_fh_F(iF,jF,kF,ex)])/dY;
-                    const double Dz=((fh[idx_fh_F(iF,jF,kF-3,ex)]+fh[idx_fh_F(iF,jF,kF+3,ex)])-SIX*(fh[idx_fh_F(iF,jF,kF-2,ex)]+fh[idx_fh_F(iF,jF,kF+2,ex)])+FIT*(fh[idx_fh_F(iF,jF,kF-1,ex)]+fh[idx_fh_F(iF,jF,kF+1,ex)])-TWT*fh[idx_fh_F(iF,jF,kF,ex)])/dZ;
-                    f_rhs[p] += (eps/cof)*(Dx+Dy+Dz);
-                }}}
-            }
-        }
-        free(fh);
-        return;
     }
-#elif (ghost_width == 4)
-    {
-        const int ord = 4;
-        int iminF = 1, jminF = 1, kminF = 1;
-        if (Symmetry > NO_SYMM && fabs(Z[0]) < dZ) kminF = -3;
-        if (Symmetry > EQ_SYMM && fabs(X[0]) < dX) iminF = -3;
-        if (Symmetry > EQ_SYMM && fabs(Y[0]) < dY) jminF = -3;
 
-        const size_t nx = (size_t)ex1 + ord;
-        const size_t ny = (size_t)ex2 + ord;
-        const size_t nz = (size_t)ex3 + ord;
-        double *fh = (double*)malloc(nx*ny*nz*sizeof(double));
-        if (!fh) return;
-        symmetry_bd(ord, ex, f, fh, SoA);
-
-        const double d60dx=ONE/F60/dX, d60dy=ONE/F60/dY, d60dz=ONE/F60/dZ;
-        const double d12dx=ONE/F12/dX, d12dy=ONE/F12/dY, d12dz=ONE/F12/dZ;
-        const double d2dx=ONE/TWO/dX, d2dy=ONE/TWO/dY, d2dz=ONE/TWO/dZ;
-
-        /* ---- advection (6th-order lopsided) ---- */
-        for (int k0=0;k0<=ex3-2;++k0) { const int kF=k0+1;
-        for (int j0=0;j0<=ex2-2;++j0) { const int jF=j0+1;
-        for (int i0=0;i0<=ex1-2;++i0) { const int iF=i0+1;
-            const size_t p=idx_ex(i0,j0,k0,ex);
-            /* x */
-            const double sfx=Sfx[p];
-            if (sfx>ZEO) {
-                if (i0<=ex1-5&&(i0-1)>=iminF) f_rhs[p]+=sfx*d60dx*(+F2*fh[idx_fh_F_ord4(iF-2,jF,kF,ex)]-F24*fh[idx_fh_F_ord4(iF-1,jF,kF,ex)]-F35*fh[idx_fh_F_ord4(iF,jF,kF,ex)]+F80*fh[idx_fh_F_ord4(iF+1,jF,kF,ex)]-F30*fh[idx_fh_F_ord4(iF+2,jF,kF,ex)]+EIT*fh[idx_fh_F_ord4(iF+3,jF,kF,ex)]-fh[idx_fh_F_ord4(iF+4,jF,kF,ex)]);
-                else if (i0<=ex1-6&&i0>=iminF) f_rhs[p]+=sfx*d60dx*(-F10*fh[idx_fh_F_ord4(iF-1,jF,kF,ex)]-F77*fh[idx_fh_F_ord4(iF,jF,kF,ex)]+F150*fh[idx_fh_F_ord4(iF+1,jF,kF,ex)]-F100*fh[idx_fh_F_ord4(iF+2,jF,kF,ex)]+F50*fh[idx_fh_F_ord4(iF+3,jF,kF,ex)]-F15*fh[idx_fh_F_ord4(iF+4,jF,kF,ex)]+F2*fh[idx_fh_F_ord4(iF+5,jF,kF,ex)]);
-                else if (i0<=ex1-4&&(i0-2)>=iminF) f_rhs[p]+=sfx*d60dx*(-fh[idx_fh_F_ord4(iF-3,jF,kF,ex)]+F9*fh[idx_fh_F_ord4(iF-2,jF,kF,ex)]-F45*fh[idx_fh_F_ord4(iF-1,jF,kF,ex)]+F45*fh[idx_fh_F_ord4(iF+1,jF,kF,ex)]-F9*fh[idx_fh_F_ord4(iF+2,jF,kF,ex)]+fh[idx_fh_F_ord4(iF+3,jF,kF,ex)]);
-                else if (i0<=ex1-3&&(i0-1)>=iminF) f_rhs[p]+=sfx*d12dx*(fh[idx_fh_F_ord4(iF-2,jF,kF,ex)]-EIT*fh[idx_fh_F_ord4(iF-1,jF,kF,ex)]+EIT*fh[idx_fh_F_ord4(iF+1,jF,kF,ex)]-fh[idx_fh_F_ord4(iF+2,jF,kF,ex)]);
-                else if (i0<=ex1-2&&i0>=iminF) f_rhs[p]+=sfx*d2dx*(-fh[idx_fh_F_ord4(iF-1,jF,kF,ex)]+fh[idx_fh_F_ord4(iF+1,jF,kF,ex)]);
-            } else if (sfx<ZEO) {
-                if ((i0-3)>=iminF&&i0<=ex1-3) f_rhs[p]-=sfx*d60dx*(+F2*fh[idx_fh_F_ord4(iF+2,jF,kF,ex)]-F24*fh[idx_fh_F_ord4(iF+1,jF,kF,ex)]-F35*fh[idx_fh_F_ord4(iF,jF,kF,ex)]+F80*fh[idx_fh_F_ord4(iF-1,jF,kF,ex)]-F30*fh[idx_fh_F_ord4(iF-2,jF,kF,ex)]+EIT*fh[idx_fh_F_ord4(iF-3,jF,kF,ex)]-fh[idx_fh_F_ord4(iF-4,jF,kF,ex)]);
-                else if ((i0-4)>=iminF&&i0<=ex1-2) f_rhs[p]-=sfx*d60dx*(-F10*fh[idx_fh_F_ord4(iF+1,jF,kF,ex)]-F77*fh[idx_fh_F_ord4(iF,jF,kF,ex)]+F150*fh[idx_fh_F_ord4(iF-1,jF,kF,ex)]-F100*fh[idx_fh_F_ord4(iF-2,jF,kF,ex)]+F50*fh[idx_fh_F_ord4(iF-3,jF,kF,ex)]-F15*fh[idx_fh_F_ord4(iF-4,jF,kF,ex)]+F2*fh[idx_fh_F_ord4(iF-5,jF,kF,ex)]);
-                else if ((i0-2)>=iminF&&i0<=ex1-4) f_rhs[p]+=sfx*d60dx*(-fh[idx_fh_F_ord4(iF-3,jF,kF,ex)]+F9*fh[idx_fh_F_ord4(iF-2,jF,kF,ex)]-F45*fh[idx_fh_F_ord4(iF-1,jF,kF,ex)]+F45*fh[idx_fh_F_ord4(iF+1,jF,kF,ex)]-F9*fh[idx_fh_F_ord4(iF+2,jF,kF,ex)]+fh[idx_fh_F_ord4(iF+3,jF,kF,ex)]);
-                else if ((i0-1)>=iminF&&i0<=ex1-3) f_rhs[p]+=sfx*d12dx*(fh[idx_fh_F_ord4(iF-2,jF,kF,ex)]-EIT*fh[idx_fh_F_ord4(iF-1,jF,kF,ex)]+EIT*fh[idx_fh_F_ord4(iF+1,jF,kF,ex)]-fh[idx_fh_F_ord4(iF+2,jF,kF,ex)]);
-                else if (i0>=iminF&&i0<=ex1-2) f_rhs[p]+=sfx*d2dx*(-fh[idx_fh_F_ord4(iF-1,jF,kF,ex)]+fh[idx_fh_F_ord4(iF+1,jF,kF,ex)]);
-            }
-            /* y */
-            const double sfy=Sfy[p];
-            if (sfy>ZEO) {
-                if (j0<=ex2-5&&(j0-1)>=jminF) f_rhs[p]+=sfy*d60dy*(F2*fh[idx_fh_F_ord4(iF,jF-2,kF,ex)]-F24*fh[idx_fh_F_ord4(iF,jF-1,kF,ex)]-F35*fh[idx_fh_F_ord4(iF,jF,kF,ex)]+F80*fh[idx_fh_F_ord4(iF,jF+1,kF,ex)]-F30*fh[idx_fh_F_ord4(iF,jF+2,kF,ex)]+EIT*fh[idx_fh_F_ord4(iF,jF+3,kF,ex)]-fh[idx_fh_F_ord4(iF,jF+4,kF,ex)]);
-                else if (j0<=ex2-6&&j0>=jminF) f_rhs[p]+=sfy*d60dy*(-F10*fh[idx_fh_F_ord4(iF,jF-1,kF,ex)]-F77*fh[idx_fh_F_ord4(iF,jF,kF,ex)]+F150*fh[idx_fh_F_ord4(iF,jF+1,kF,ex)]-F100*fh[idx_fh_F_ord4(iF,jF+2,kF,ex)]+F50*fh[idx_fh_F_ord4(iF,jF+3,kF,ex)]-F15*fh[idx_fh_F_ord4(iF,jF+4,kF,ex)]+F2*fh[idx_fh_F_ord4(iF,jF+5,kF,ex)]);
-                else if (j0<=ex2-4&&(j0-2)>=jminF) f_rhs[p]+=sfy*d60dy*(-fh[idx_fh_F_ord4(iF,jF-3,kF,ex)]+F9*fh[idx_fh_F_ord4(iF,jF-2,kF,ex)]-F45*fh[idx_fh_F_ord4(iF,jF-1,kF,ex)]+F45*fh[idx_fh_F_ord4(iF,jF+1,kF,ex)]-F9*fh[idx_fh_F_ord4(iF,jF+2,kF,ex)]+fh[idx_fh_F_ord4(iF,jF+3,kF,ex)]);
-                else if (j0<=ex2-3&&(j0-1)>=jminF) f_rhs[p]+=sfy*d12dy*(fh[idx_fh_F_ord4(iF,jF-2,kF,ex)]-EIT*fh[idx_fh_F_ord4(iF,jF-1,kF,ex)]+EIT*fh[idx_fh_F_ord4(iF,jF+1,kF,ex)]-fh[idx_fh_F_ord4(iF,jF+2,kF,ex)]);
-                else if (j0<=ex2-2&&j0>=jminF) f_rhs[p]+=sfy*d2dy*(-fh[idx_fh_F_ord4(iF,jF-1,kF,ex)]+fh[idx_fh_F_ord4(iF,jF+1,kF,ex)]);
-            } else if (sfy<ZEO) {
-                if ((j0-3)>=jminF&&j0<=ex2-3) f_rhs[p]-=sfy*d60dy*(F2*fh[idx_fh_F_ord4(iF,jF+2,kF,ex)]-F24*fh[idx_fh_F_ord4(iF,jF+1,kF,ex)]-F35*fh[idx_fh_F_ord4(iF,jF,kF,ex)]+F80*fh[idx_fh_F_ord4(iF,jF-1,kF,ex)]-F30*fh[idx_fh_F_ord4(iF,jF-2,kF,ex)]+EIT*fh[idx_fh_F_ord4(iF,jF-3,kF,ex)]-fh[idx_fh_F_ord4(iF,jF-4,kF,ex)]);
-                else if ((j0-4)>=jminF&&j0<=ex2-2) f_rhs[p]-=sfy*d60dy*(-F10*fh[idx_fh_F_ord4(iF,jF+1,kF,ex)]-F77*fh[idx_fh_F_ord4(iF,jF,kF,ex)]+F150*fh[idx_fh_F_ord4(iF,jF-1,kF,ex)]-F100*fh[idx_fh_F_ord4(iF,jF-2,kF,ex)]+F50*fh[idx_fh_F_ord4(iF,jF-3,kF,ex)]-F15*fh[idx_fh_F_ord4(iF,jF-4,kF,ex)]+F2*fh[idx_fh_F_ord4(iF,jF-5,kF,ex)]);
-                else if ((j0-2)>=jminF&&j0<=ex2-4) f_rhs[p]+=sfy*d60dy*(-fh[idx_fh_F_ord4(iF,jF-3,kF,ex)]+F9*fh[idx_fh_F_ord4(iF,jF-2,kF,ex)]-F45*fh[idx_fh_F_ord4(iF,jF-1,kF,ex)]+F45*fh[idx_fh_F_ord4(iF,jF+1,kF,ex)]-F9*fh[idx_fh_F_ord4(iF,jF+2,kF,ex)]+fh[idx_fh_F_ord4(iF,jF+3,kF,ex)]);
-                else if ((j0-1)>=jminF&&j0<=ex2-3) f_rhs[p]+=sfy*d12dy*(fh[idx_fh_F_ord4(iF,jF-2,kF,ex)]-EIT*fh[idx_fh_F_ord4(iF,jF-1,kF,ex)]+EIT*fh[idx_fh_F_ord4(iF,jF+1,kF,ex)]-fh[idx_fh_F_ord4(iF,jF+2,kF,ex)]);
-                else if (j0>=jminF&&j0<=ex2-2) f_rhs[p]+=sfy*d2dy*(-fh[idx_fh_F_ord4(iF,jF-1,kF,ex)]+fh[idx_fh_F_ord4(iF,jF+1,kF,ex)]);
-            }
-            /* z */
-            const double sfz=Sfz[p];
-            if (sfz>ZEO) {
-                if (k0<=ex3-5&&(k0-1)>=kminF) f_rhs[p]+=sfz*d60dz*(F2*fh[idx_fh_F_ord4(iF,jF,kF-2,ex)]-F24*fh[idx_fh_F_ord4(iF,jF,kF-1,ex)]-F35*fh[idx_fh_F_ord4(iF,jF,kF,ex)]+F80*fh[idx_fh_F_ord4(iF,jF,kF+1,ex)]-F30*fh[idx_fh_F_ord4(iF,jF,kF+2,ex)]+EIT*fh[idx_fh_F_ord4(iF,jF,kF+3,ex)]-fh[idx_fh_F_ord4(iF,jF,kF+4,ex)]);
-                else if (k0<=ex3-6&&k0>=kminF) f_rhs[p]+=sfz*d60dz*(-F10*fh[idx_fh_F_ord4(iF,jF,kF-1,ex)]-F77*fh[idx_fh_F_ord4(iF,jF,kF,ex)]+F150*fh[idx_fh_F_ord4(iF,jF,kF+1,ex)]-F100*fh[idx_fh_F_ord4(iF,jF,kF+2,ex)]+F50*fh[idx_fh_F_ord4(iF,jF,kF+3,ex)]-F15*fh[idx_fh_F_ord4(iF,jF,kF+4,ex)]+F2*fh[idx_fh_F_ord4(iF,jF,kF+5,ex)]);
-                else if (k0<=ex3-4&&(k0-2)>=kminF) f_rhs[p]+=sfz*d60dz*(-fh[idx_fh_F_ord4(iF,jF,kF-3,ex)]+F9*fh[idx_fh_F_ord4(iF,jF,kF-2,ex)]-F45*fh[idx_fh_F_ord4(iF,jF,kF-1,ex)]+F45*fh[idx_fh_F_ord4(iF,jF,kF+1,ex)]-F9*fh[idx_fh_F_ord4(iF,jF,kF+2,ex)]+fh[idx_fh_F_ord4(iF,jF,kF+3,ex)]);
-                else if (k0<=ex3-3&&(k0-1)>=kminF) f_rhs[p]+=sfz*d12dz*(fh[idx_fh_F_ord4(iF,jF,kF-2,ex)]-EIT*fh[idx_fh_F_ord4(iF,jF,kF-1,ex)]+EIT*fh[idx_fh_F_ord4(iF,jF,kF+1,ex)]-fh[idx_fh_F_ord4(iF,jF,kF+2,ex)]);
-                else if (k0<=ex3-2&&k0>=kminF) f_rhs[p]+=sfz*d2dz*(-fh[idx_fh_F_ord4(iF,jF,kF-1,ex)]+fh[idx_fh_F_ord4(iF,jF,kF+1,ex)]);
-            } else if (sfz<ZEO) {
-                if ((k0-3)>=kminF&&k0<=ex3-3) f_rhs[p]-=sfz*d60dz*(F2*fh[idx_fh_F_ord4(iF,jF,kF+2,ex)]-F24*fh[idx_fh_F_ord4(iF,jF,kF+1,ex)]-F35*fh[idx_fh_F_ord4(iF,jF,kF,ex)]+F80*fh[idx_fh_F_ord4(iF,jF,kF-1,ex)]-F30*fh[idx_fh_F_ord4(iF,jF,kF-2,ex)]+EIT*fh[idx_fh_F_ord4(iF,jF,kF-3,ex)]-fh[idx_fh_F_ord4(iF,jF,kF-4,ex)]);
-                else if ((k0-4)>=kminF&&k0<=ex3-2) f_rhs[p]-=sfz*d60dz*(-F10*fh[idx_fh_F_ord4(iF,jF,kF+1,ex)]-F77*fh[idx_fh_F_ord4(iF,jF,kF,ex)]+F150*fh[idx_fh_F_ord4(iF,jF,kF-1,ex)]-F100*fh[idx_fh_F_ord4(iF,jF,kF-2,ex)]+F50*fh[idx_fh_F_ord4(iF,jF,kF-3,ex)]-F15*fh[idx_fh_F_ord4(iF,jF,kF-4,ex)]+F2*fh[idx_fh_F_ord4(iF,jF,kF-5,ex)]);
-                else if ((k0-2)>=kminF&&k0<=ex3-4) f_rhs[p]+=sfz*d60dz*(-fh[idx_fh_F_ord4(iF,jF,kF-3,ex)]+F9*fh[idx_fh_F_ord4(iF,jF,kF-2,ex)]-F45*fh[idx_fh_F_ord4(iF,jF,kF-1,ex)]+F45*fh[idx_fh_F_ord4(iF,jF,kF+1,ex)]-F9*fh[idx_fh_F_ord4(iF,jF,kF+2,ex)]+fh[idx_fh_F_ord4(iF,jF,kF+3,ex)]);
-                else if ((k0-1)>=kminF&&k0<=ex3-3) f_rhs[p]+=sfz*d12dz*(fh[idx_fh_F_ord4(iF,jF,kF-2,ex)]-EIT*fh[idx_fh_F_ord4(iF,jF,kF-1,ex)]+EIT*fh[idx_fh_F_ord4(iF,jF,kF+1,ex)]-fh[idx_fh_F_ord4(iF,jF,kF+2,ex)]);
-                else if (k0>=kminF&&k0<=ex3-2) f_rhs[p]+=sfz*d2dz*(-fh[idx_fh_F_ord4(iF,jF,kF-1,ex)]+fh[idx_fh_F_ord4(iF,jF,kF+1,ex)]);
-            }
-        }}}
-
-        /* ---- KO dissipation (r=4, cof=256, sign=-) ---- */
-        if (eps > ZEO) {
-            const double cof = 256.0;
-            const double F8k = 8.0, F28 = 28.0, F56 = 56.0, F70 = 70.0;
-            const int i0_lo=(iminF+3>0)?iminF+3:0, j0_lo=(jminF+3>0)?jminF+3:0, k0_lo=(kminF+3>0)?kminF+3:0;
-            const int i0_hi=imaxF-5, j0_hi=jmaxF-5, k0_hi=kmaxF-5;
-            if (!(i0_lo>i0_hi||j0_lo>j0_hi||k0_lo>k0_hi)) {
-                for (int k0=k0_lo;k0<=k0_hi;++k0) { const int kF=k0+1;
-                for (int j0=j0_lo;j0<=j0_hi;++j0) { const int jF=j0+1;
-                for (int i0=i0_lo;i0<=i0_hi;++i0) { const int iF=i0+1;
-                    const size_t p=idx_ex(i0,j0,k0,ex);
-                    const double Dx=((fh[idx_fh_F_ord4(iF-4,jF,kF,ex)]+fh[idx_fh_F_ord4(iF+4,jF,kF,ex)])-F8k*(fh[idx_fh_F_ord4(iF-3,jF,kF,ex)]+fh[idx_fh_F_ord4(iF+3,jF,kF,ex)])+F28*(fh[idx_fh_F_ord4(iF-2,jF,kF,ex)]+fh[idx_fh_F_ord4(iF+2,jF,kF,ex)])-F56*(fh[idx_fh_F_ord4(iF-1,jF,kF,ex)]+fh[idx_fh_F_ord4(iF+1,jF,kF,ex)])+F70*fh[idx_fh_F_ord4(iF,jF,kF,ex)])/dX;
-                    const double Dy=((fh[idx_fh_F_ord4(iF,jF-4,kF,ex)]+fh[idx_fh_F_ord4(iF,jF+4,kF,ex)])-F8k*(fh[idx_fh_F_ord4(iF,jF-3,kF,ex)]+fh[idx_fh_F_ord4(iF,jF+3,kF,ex)])+F28*(fh[idx_fh_F_ord4(iF,jF-2,kF,ex)]+fh[idx_fh_F_ord4(iF,jF+2,kF,ex)])-F56*(fh[idx_fh_F_ord4(iF,jF-1,kF,ex)]+fh[idx_fh_F_ord4(iF,jF+1,kF,ex)])+F70*fh[idx_fh_F_ord4(iF,jF,kF,ex)])/dY;
-                    const double Dz=((fh[idx_fh_F_ord4(iF,jF,kF-4,ex)]+fh[idx_fh_F_ord4(iF,jF,kF+4,ex)])-F8k*(fh[idx_fh_F_ord4(iF,jF,kF-3,ex)]+fh[idx_fh_F_ord4(iF,jF,kF+3,ex)])+F28*(fh[idx_fh_F_ord4(iF,jF,kF-2,ex)]+fh[idx_fh_F_ord4(iF,jF,kF+2,ex)])-F56*(fh[idx_fh_F_ord4(iF,jF,kF-1,ex)]+fh[idx_fh_F_ord4(iF,jF,kF+1,ex)])+F70*fh[idx_fh_F_ord4(iF,jF,kF,ex)])/dZ;
-                    f_rhs[p] -= (eps/cof)*(Dx+Dy+Dz);
-                }}}
-            }
-        }
-        free(fh);
-        return;
-    }
-#elif (ghost_width == 5)
-    {
-        lopsided(ex, X, Y, Z, f, f_rhs, Sfx, Sfy, Sfz, Symmetry, SoA);
-        if (eps > ZEO) kodis(ex, X, Y, Z, f, f_rhs, SoA, Symmetry, eps);
-        return;
-    }
-#else
-#error "lopsided_kodis_c.C: unsupported ghost_width (must be 2, 3, 4, or 5)"
-#endif
+    free(fh);
 }

AMSS_NCKU_source/macrodef.fh

@@ -13,7 +13,7 @@
 
 #define ABV 0
 
-#define EScalar_CC 2
+#define EScalar_CC  2
 
 #if 0
 

AMSS_NCKU_source/macrodef.h

@@ -10,7 +10,7 @@
 
 #define GaussInt
 
-#define ABEtype 0
+#define ABEtype 1
 
 //#define With_AHF
 #define Psi4type 0
@@ -167,3 +167,4 @@
 #define TINY 1e-10
 
 #endif   /* MICRODEF_H */
+

AMSS_NCKU_source/makefile

@@ -1,293 +1,260 @@
-
-
-include makefile.inc
-
--include AMSS_NCKU_build.mk
-
-ABE_TYPE ?= $(shell awk '/^[[:space:]]*\#define[[:space:]]+ABEtype/ {print $$3; exit}' macrodef.h 2>/dev/null)
-
-ifeq ($(USE_TRANSFER_CACHE),auto)
-ifeq ($(ABE_TYPE),0)
-EFFECTIVE_USE_TRANSFER_CACHE = 1
-else
-EFFECTIVE_USE_TRANSFER_CACHE = 0
-endif
-else
-EFFECTIVE_USE_TRANSFER_CACHE = $(USE_TRANSFER_CACHE)
-endif
-
-ifeq ($(USE_CXX_ESCALAR_KERNEL),1)
-ifeq ($(ABE_TYPE),1)
-EFFECTIVE_USE_CXX_ESCALAR_KERNEL = 1
-else
-EFFECTIVE_USE_CXX_ESCALAR_KERNEL = 0
-endif
-else
-EFFECTIVE_USE_CXX_ESCALAR_KERNEL = 0
-endif
-
-ifeq ($(EFFECTIVE_USE_CXX_ESCALAR_KERNEL),1)
-ifeq ($(USE_CXX_KERNELS),0)
-$(error USE_CXX_ESCALAR_KERNEL=1 requires USE_CXX_KERNELS=1 because bssn_escalar_rhs_c.C reuses the C BSSN kernel)
-endif
-endif
-
-ifeq ($(USE_CXX_EM_KERNEL),1)
-ifeq ($(ABE_TYPE),3)
-EFFECTIVE_USE_CXX_EM_KERNEL = 1
-else
-EFFECTIVE_USE_CXX_EM_KERNEL = 0
-endif
-else
-EFFECTIVE_USE_CXX_EM_KERNEL = 0
-endif
-
-ifeq ($(EFFECTIVE_USE_CXX_EM_KERNEL),1)
-ifeq ($(USE_CXX_KERNELS),0)
-$(error USE_CXX_EM_KERNEL=1 requires USE_CXX_KERNELS=1 because bssn_em_rhs_c.C reuses the C BSSN kernel)
-endif
-endif
-
-EM_KERNEL_FLAG = -DBSSN_USE_EM_C_KERNEL=$(EFFECTIVE_USE_CXX_EM_KERNEL)
-
-## polint(ordn=6) kernel selector:
-##   1 (default): barycentric fast path
-##   0          : fallback to Neville path
-POLINT6_USE_BARY ?= 1
-POLINT6_FLAG = -DPOLINT6_USE_BARYCENTRIC=$(POLINT6_USE_BARY)
-TRANSFER_CACHE_FLAG = -DBSSN_USE_TRANSFER_CACHE=$(EFFECTIVE_USE_TRANSFER_CACHE)
-ESCALAR_KERNEL_FLAG = -DBSSN_USE_ESCALAR_C_KERNEL=$(EFFECTIVE_USE_CXX_ESCALAR_KERNEL)
-
-## ABE build flags selected by PGO_MODE (set in makefile.inc, default: opt)
-##   make                        -> opt  (PGO-guided, maximum performance)
-##   make PGO_MODE=instrument    -> instrument (Phase 1: collect fresh profile data)
-PROFDATA = /home/$(shell whoami)/AMSS-NCKU/pgo_profile/default.profdata
-
-ifeq ($(PGO_MODE),instrument)
-## Phase 1: instrumentation — omit -ipo/-fp-model fast=2 for faster build and numerical stability
-CXXAPPFLAGS = -O3 -xHost -fma -fprofile-instr-generate -ipo \
-              -Dfortran3 -Dnewc -I${MKLROOT}/include $(INTERP_LB_FLAGS) \
-              $(TRANSFER_CACHE_FLAG) $(ESCALAR_KERNEL_FLAG) $(EM_KERNEL_FLAG)
-f90appflags = -O3 -xHost -fma -fprofile-instr-generate -ipo \
-              -align array64byte -fpp -I${MKLROOT}/include $(POLINT6_FLAG)
-else
-## opt (default): maximum performance with PGO profile data -fprofile-instr-use=$(PROFDATA) \
-## PGO has been turned off, now tested and found to be negative optimization
-## INTERP_LB_FLAGS has been turned off too, now tested and found to be negative optimization
-
-
-CXXAPPFLAGS = -O3 -xHost -fp-model fast=2 -fma -ipo \
-              -Dfortran3 -Dnewc -I${MKLROOT}/include $(INTERP_LB_FLAGS) \
-              $(TRANSFER_CACHE_FLAG) $(ESCALAR_KERNEL_FLAG) $(EM_KERNEL_FLAG)
-f90appflags = -O3 -xHost -fp-model fast=2 -fma -ipo \
-              -align array64byte -fpp -I${MKLROOT}/include $(POLINT6_FLAG)
-endif
-
-.SUFFIXES: .o .f90 .C .for .cu
-
-.f90.o:
-	$(f90) $(f90appflags) -c $< -o $@
-
-.C.o:
-	${CXX} $(CXXAPPFLAGS) -c $< $(filein) -o $@
-
-# ShellPatch.C uses OpenMP for setupintintstuff search loops
-ShellPatch.o: ShellPatch.C
-	${CXX} $(CXXAPPFLAGS) $(OMP_FLAG) -c $< $(filein) -o $@
-
-.for.o:
-	$(f77) -c $< -o $@
-
+
+
+include makefile.inc
+
+## polint(ordn=6) kernel selector:
+##   1 (default): barycentric fast path
+##   0          : fallback to Neville path
+POLINT6_USE_BARY ?= 1
+POLINT6_FLAG = -DPOLINT6_USE_BARYCENTRIC=$(POLINT6_USE_BARY)
+
+## ABE build flags selected by PGO_MODE (set in makefile.inc, default: opt)
+##   make                        -> opt  (PGO-guided, maximum performance)
+##   make PGO_MODE=instrument    -> instrument (Phase 1: collect fresh profile data)
+PROFDATA = /home/$(shell whoami)/AMSS-NCKU/pgo_profile/default.profdata
+
+ifeq ($(TOOLCHAIN),intel)
+OMP_FLAG = -qopenmp
+
+ifeq ($(PGO_MODE),instrument)
+## Intel Phase 1: instrumentation — omit -ipo/-fp-model fast=2 for faster build and numerical stability
+CXXAPPFLAGS = -O3 -xHost -fma -fprofile-instr-generate -ipo \
+              -Dfortran3 -Dnewc $(MKL_INC) $(INTERP_LB_FLAGS)
+f90appflags = -O3 -xHost -fma -fprofile-instr-generate -ipo \
+              -align array64byte -fpp $(MKL_INC) $(POLINT6_FLAG)
+else
+## opt (default): maximum performance with PGO profile data -fprofile-instr-use=$(PROFDATA) \
+## PGO has been turned off, now tested and found to be negative optimization
+## INTERP_LB_FLAGS has been turned off too, now tested and found to be negative optimization
+
+
+CXXAPPFLAGS = -O3 -xHost -fp-model fast=2 -fma -ipo \
+              -Dfortran3 -Dnewc $(MKL_INC) $(INTERP_LB_FLAGS)
+f90appflags = -O3 -xHost -fp-model fast=2 -fma -ipo \
+              -align array64byte -fpp $(MKL_INC) $(POLINT6_FLAG)
+endif
+
+TP_OPTFLAGS = -O3 -xHost -fp-model fast=2 -fma -ipo \
+              -fprofile-instr-use=$(TP_PROFDATA) \
+              -Dfortran3 -Dnewc $(MKL_INC)
+else
+## NVHPC defaults: mpicc/mpicxx/mpifort wrappers
+## PGO_MODE is ignored in this branch.
+OMP_FLAG = -mp
+CXXAPPFLAGS = -O3 -tp=host -Mcache_align -Mfma \
+              -Dfortran3 -Dnewc $(MKL_INC) $(INTERP_LB_FLAGS)
+f90appflags = -O3 -tp=host -Mcache_align -Mfma -Mpreprocess \
+              $(MKL_INC) $(POLINT6_FLAG)
+TP_OPTFLAGS = -O3 -tp=host -Mcache_align -Mfma \
+              -Dfortran3 -Dnewc $(MKL_INC)
+endif
+
+.SUFFIXES: .o .f90 .C .for .cu
+
+.f90.o:
+	$(f90) $(f90appflags) -c $< -o $@
+
+.C.o:
+	${CXX} $(CXXAPPFLAGS) -c $< $(filein) -o $@
+
+.for.o:
+	$(f77) -c $< -o $@
+
 .cu.o:
 	$(Cu) $(CUDA_APP_FLAGS) -c $< -o $@ $(CUDA_LIB_PATH)
 
-# C rewrite of BSSN RHS kernel and helpers
-bssn_rhs_c.o: bssn_rhs_c.C
-	${CXX} $(CXXAPPFLAGS) -c $< $(filein) -o $@
-
-fderivs_c.o: fderivs_c.C
-	${CXX} $(CXXAPPFLAGS) -c $< $(filein) -o $@
-
-fdderivs_c.o: fdderivs_c.C
-	${CXX} $(CXXAPPFLAGS) -c $< $(filein) -o $@
-
-kodiss_c.o: kodiss_c.C
-	${CXX} $(CXXAPPFLAGS) -c $< $(filein) -o $@
-
-lopsided_c.o: lopsided_c.C
-	${CXX} $(CXXAPPFLAGS) -c $< $(filein) -o $@
-
-lopsided_kodis_c.o: lopsided_kodis_c.C
-	${CXX} $(CXXAPPFLAGS) -c $< $(filein) -o $@
-
-# C rewrite of shell-patch derivative kernels
-fderivs_sh_c.o: fderivs_sh_c.C
-	${CXX} $(CXXAPPFLAGS) -c $< $(filein) -o $@
-
-fdderivs_sh_c.o: fdderivs_sh_c.C
-	${CXX} $(CXXAPPFLAGS) -c $< $(filein) -o $@
-
-fderivs_shc_c.o: fderivs_shc_c.C
-	${CXX} $(CXXAPPFLAGS) -c $< $(filein) -o $@
-
-fdderivs_shc_c.o: fdderivs_shc_c.C
-	${CXX} $(CXXAPPFLAGS) -c $< $(filein) -o $@
-
-kodiss_sh_c.o: kodiss_sh_c.C
-	${CXX} $(CXXAPPFLAGS) -c $< $(filein) -o $@
-
-
-bssn_em_rhs_c.o: bssn_em_rhs_c.C
-	${CXX} $(CXXAPPFLAGS) -c $< $(filein) -o $@
-z4c_rhs_c.o: z4c_rhs_c.C
-	${CXX} $(CXXAPPFLAGS) -c $< $(filein) -o $@
-
-#interp_lb_profile.o: interp_lb_profile.C interp_lb_profile.h
-#	${CXX} $(CXXAPPFLAGS) -c $< $(filein) -o $@
-
-## TwoPunctureABE uses fixed optimal flags with its own PGO profile, independent of CXXAPPFLAGS
-TP_PROFDATA = /home/$(shell whoami)/AMSS-NCKU/pgo_profile/TwoPunctureABE.profdata
-TP_OPTFLAGS = -O3 -xHost -fp-model fast=2 -fma -ipo \
-              -fprofile-instr-use=$(TP_PROFDATA) \
-              -Dfortran3 -Dnewc -I${MKLROOT}/include
-
-TwoPunctures.o: TwoPunctures.C
-	${CXX} $(TP_OPTFLAGS) -qopenmp -c $< -o $@
-
-TwoPunctureABE.o: TwoPunctureABE.C
-	${CXX} $(TP_OPTFLAGS) -qopenmp -c $< -o $@
-
-# Input files
-
-## Kernel implementation switch (set USE_CXX_KERNELS=0 to fall back to Fortran)
-ifeq ($(USE_CXX_KERNELS),0)
-# Fortran mode: no C rewrite files; bssn_rhs.o is included via F90FILES below
-CFILES =
-else
-# C++ mode (default): C rewrite of bssn/bssn-escalar rhs and helper kernels
-CFILES = bssn_rhs_c.o fderivs_c.o fdderivs_c.o kodiss_c.o lopsided_c.o lopsided_kodis_c.o
-ifeq ($(EFFECTIVE_USE_CXX_ESCALAR_KERNEL),1)
-CFILES += bssn_escalar_rhs_c.o
-endif
-ifeq ($(EFFECTIVE_USE_CXX_EM_KERNEL),1)
-CFILES += bssn_em_rhs_c.o
-endif
-endif
-
-ifeq ($(USE_CXX_Z4C_KERNELS),1)
-CFILES += z4c_rhs_c.o
-Z4C_F90_OBJ =
-else
-Z4C_F90_OBJ = Z4c_rhs.o
-endif
-
-## RK4 kernel switch (independent from USE_CXX_KERNELS)
-ifeq ($(USE_CXX_RK4),1)
-CFILES += rungekutta4_rout_c.o
-RK4_F90_OBJ =
-else
-RK4_F90_OBJ = rungekutta4_rout.o
-endif
-
-## Shell-patch derivative kernel switch (independent from USE_CXX_KERNELS)
-##   1 : use C++ rewrite of shell derivative functions (experimental)
-##   0 : use original Fortran diff_new_sh.o and kodiss_sh.o (default)
-USE_CXX_SHELL_KERNELS ?= 0
-ifeq ($(USE_CXX_SHELL_KERNELS),1)
-CFILES += fderivs_sh_c.o fdderivs_sh_c.o fderivs_shc_c.o fdderivs_shc_c.o kodiss_sh_c.o
-SH_F90_OBJ =
-else
-SH_F90_OBJ = diff_new_sh.o kodiss_sh.o point_diff_new_sh.o
-endif
-
-C++FILES = ABE.o Ansorg.o Block.o misc.o monitor.o Parallel.o MPatch.o var.o\
-           cgh.o bssn_class.o surface_integral.o ShellPatch.o\
-	   bssnEScalar_class.o perf.o Z4c_class.o NullShellPatch.o\
-	   bssnEM_class.o cpbc_util.o z4c_rhs_point.o checkpoint.o\
-           Parallel_bam.o scalar_class.o transpbh.o NullShellPatch2.o\
-	   NullShellPatch2_Evo.o writefile_f.o interp_lb_profile.o
-	   
-C++FILES_GPU = ABE.o Ansorg.o Block.o misc.o monitor.o Parallel.o MPatch.o var.o\
-           cgh.o surface_integral.o ShellPatch.o\
-	   bssnEScalar_class.o perf.o Z4c_class.o NullShellPatch.o\
-	   bssnEM_class.o cpbc_util.o z4c_rhs_point.o checkpoint.o\
-           Parallel_bam.o scalar_class.o transpbh.o NullShellPatch2.o\
-	   NullShellPatch2_Evo.o \
-	   bssn_gpu_class.o bssn_step_gpu.o bssn_macro.o writefile_f.o
-
-F90FILES_BASE = enforce_algebra.o fmisc.o initial_puncture.o prolongrestrict.o\
-	   prolongrestrict_cell.o prolongrestrict_vertex.o\
-	   $(RK4_F90_OBJ) diff_new.o kodiss.o\
-	   lopsidediff.o sommerfeld_rout.o getnp4.o $(SH_F90_OBJ)\
-	   shellfunctions.o bssn_rhs_ss.o Set_Rho_ADM.o\
-           getnp4EScalar.o bssnEScalar_rhs.o bssn_constraint.o ricci_gamma.o\
-           fadmquantites_bssn.o $(Z4C_F90_OBJ) Z4c_rhs_ss.o\
-	   cpbc.o getnp4old.o NullEvol.o initial_null.o initial_maxwell.o\
-	   getnpem2.o empart.o NullNews.o fourdcurvature.o\
-	   bssn2adm.o adm_constraint.o adm_ricci_gamma.o\
-	   scalar_rhs.o initial_scalar.o NullEvol2.o initial_null2.o\
-	   NullNews2.o tool_f.o
-
-ifeq ($(USE_CXX_KERNELS),0)
-# Fortran mode: include original bssn_rhs.o
-F90FILES = $(F90FILES_BASE) bssn_rhs.o
-else
-# C++ mode (default): bssn_rhs.o replaced by C++ kernel
-F90FILES = $(F90FILES_BASE)
-endif
-
-F77FILES = zbesh.o
-
-AHFDOBJS = expansion.o expansion_Jacobian.o patch.o coords.o patch_info.o patch_interp.o patch_system.o \
-tgrid.o fd_grid.o ghost_zone.o array.o round.o norm.o fuzzy.o error_exit.o miscfp.o \
-linear_map.o cpm_map.o BH_diagnostics.o setup.o horizon_sequence.o find_horizons.o \
-initial_guess.o Newton.o Jacobian.o ilucg.o IntPnts0.o IntPnts.o
-
-TwoPunctureFILES = TwoPunctureABE.o TwoPunctures.o 
-
-CUDAFILES = bssn_gpu.o bssn_gpu_rhs_ss.o
-
-# file dependences
-$(C++FILES) $(C++FILES_GPU) $(F90FILES) $(CFILES) $(AHFDOBJS) $(CUDAFILES): macrodef.fh
-
-$(C++FILES): Block.h enforce_algebra.h fmisc.h initial_puncture.h macrodef.h\
-	     misc.h monitor.h MyList.h Parallel.h MPatch.h prolongrestrict.h\
-	     rungekutta4_rout.h var.h bssn_class.h bssn_rhs.h sommerfeld_rout.h\
-	     cgh.h surface_integral.h ShellPatch.h shellfunctions.h perf.h\
-             fadmquantites_bssn.h cpbc.h getnp4.h initial_null.h NullEvol.h\
-	     NullShellPatch.h initial_maxwell.h bssnEM_class.h getnpem2.h\
-	     empart.h NullNews.h kodiss.h Parallel_bam.h ricci_gamma.h\
-             initial_null2.h NullShellPatch2.h 
-             
-$(C++FILES_GPU): Block.h enforce_algebra.h fmisc.h initial_puncture.h macrodef.h\
-	     misc.h monitor.h MyList.h Parallel.h MPatch.h prolongrestrict.h\
-	     rungekutta4_rout.h var.h bssn_rhs.h sommerfeld_rout.h\
-	     cgh.h surface_integral.h ShellPatch.h shellfunctions.h perf.h\
-             fadmquantites_bssn.h cpbc.h getnp4.h initial_null.h NullEvol.h\
-	     NullShellPatch.h initial_maxwell.h bssnEM_class.h getnpem2.h\
-	     empart.h NullNews.h kodiss.h Parallel_bam.h ricci_gamma.h\
-             initial_null2.h NullShellPatch2.h \
-             bssn_gpu_class.h bssn_macro.h
-             
-$(AHFDOBJS): cctk.h cctk_Config.h cctk_Types.h cctk_Constants.h myglobal.h
-
-$(C++FILES) $(C++FILES_GPU) $(CFILES) $(AHFDOBJS) $(CUDAFILES): macrodef.h
-
-TwoPunctureFILES: TwoPunctures.h
-
-$(CUDAFILES): bssn_gpu.h gpu_mem.h gpu_rhsSS_mem.h
-
-misc.o : zbesh.o
-
-# projects
-ABE: $(C++FILES) $(CFILES) $(F90FILES) $(F77FILES) $(AHFDOBJS)
-	$(CLINKER) $(CXXAPPFLAGS) -o $@ $(C++FILES) $(CFILES) $(F90FILES) $(F77FILES) $(AHFDOBJS) $(LDLIBS)
-	
-ABEGPU: $(C++FILES_GPU) $(CFILES) $(F90FILES) $(F77FILES) $(AHFDOBJS) $(CUDAFILES)
-	$(CLINKER) $(CXXAPPFLAGS) -o $@ $(C++FILES_GPU) $(CFILES) $(F90FILES) $(F77FILES) $(AHFDOBJS) $(CUDAFILES) $(LDLIBS)
-
-TwoPunctureABE: $(TwoPunctureFILES)
-	$(CLINKER) $(TP_OPTFLAGS) -qopenmp -o $@ $(TwoPunctureFILES) $(LDLIBS)
-
-clean:
-	rm *.o ABE ABEGPU TwoPunctureABE make.log -f
+# CUDA rewrite of BSSN RHS (drop-in replacement for bssn_rhs_c + stencil helpers)
+bssn_rhs_cuda.o: bssn_rhs_cuda.cu bssn_rhs.h macrodef.h
+	$(Cu) $(CUDA_APP_FLAGS) -c $< -o $@ $(CUDA_LIB_PATH)
+
+# CUDA rewrite of Z4C Cartesian RHS
+z4c_rhs_cuda.o: z4c_rhs_cuda.cu z4c_rhs_cuda.h bssn_rhs.h macrodef.h ricci_gamma.h
+	$(Cu) $(CUDA_APP_FLAGS) -c $< -o $@ $(CUDA_LIB_PATH)
+
+# C rewrite of BSSN RHS kernel and helpers
+bssn_rhs_c.o: bssn_rhs_c.C
+	${CXX} $(CXXAPPFLAGS) -c $< $(filein) -o $@
+
+fderivs_c.o: fderivs_c.C
+	${CXX} $(CXXAPPFLAGS) -c $< $(filein) -o $@
+
+fdderivs_c.o: fdderivs_c.C
+	${CXX} $(CXXAPPFLAGS) -c $< $(filein) -o $@
+
+kodiss_c.o: kodiss_c.C
+	${CXX} $(CXXAPPFLAGS) -c $< $(filein) -o $@
+
+lopsided_c.o: lopsided_c.C
+	${CXX} $(CXXAPPFLAGS) -c $< $(filein) -o $@
+
+lopsided_kodis_c.o: lopsided_kodis_c.C
+	${CXX} $(CXXAPPFLAGS) -c $< $(filein) -o $@
+
+z4c_rhs_c.o: z4c_rhs_c.C
+	${CXX} $(CXXAPPFLAGS) -c $< $(filein) -o $@
+
+#interp_lb_profile.o: interp_lb_profile.C interp_lb_profile.h
+#	${CXX} $(CXXAPPFLAGS) -c $< $(filein) -o $@
+
+TwoPunctures.o: TwoPunctures.C
+	${CXX} $(TP_OPTFLAGS) $(OMP_FLAG) -c $< -o $@
+
+TwoPunctureABE.o: TwoPunctureABE.C
+	${CXX} $(TP_OPTFLAGS) $(OMP_FLAG) -c $< -o $@
+
+# Input files
+
+## CUDA BSSN RHS switch
+##   1 : use the rewritten CUDA bssn_rhs backend
+##   0 : keep the normal CPU/Fortran selection below
+USE_CUDA_BSSN ?= 0
+USE_CUDA_Z4C ?= 0
+
+CXXAPPFLAGS += -DUSE_CUDA_BSSN=$(USE_CUDA_BSSN)
+CUDA_APP_FLAGS += -DUSE_CUDA_BSSN=$(USE_CUDA_BSSN)
+CXXAPPFLAGS += -DUSE_CUDA_Z4C=$(USE_CUDA_Z4C)
+CUDA_APP_FLAGS += -DUSE_CUDA_Z4C=$(USE_CUDA_Z4C)
+
+## Kernel implementation switch (set USE_CXX_KERNELS=0 to fall back to Fortran)
+ifeq ($(USE_CXX_KERNELS),0)
+# Fortran mode: no C rewrite files; bssn_rhs.o is included via F90FILES below
+CFILES_CPU =
+else
+# C++ mode (default): C rewrite of bssn_rhs and helper kernels
+CFILES_CPU = bssn_rhs_c.o fderivs_c.o fdderivs_c.o kodiss_c.o lopsided_c.o lopsided_kodis_c.o
+endif
+
+CFILES_CUDA_BSSN = bssn_rhs_cuda.o
+
+ifeq ($(USE_CUDA_BSSN),1)
+CFILES = $(CFILES_CUDA_BSSN)
+else
+CFILES = $(CFILES_CPU)
+endif
+
+ifeq ($(USE_CUDA_Z4C),1)
+CFILES += z4c_rhs_cuda.o
+Z4C_F90_OBJ =
+else ifeq ($(USE_CXX_Z4C_KERNELS),1)
+CFILES += z4c_rhs_c.o
+Z4C_F90_OBJ =
+else
+Z4C_F90_OBJ = Z4c_rhs.o
+endif
+
+## RK4 kernel switch (independent from USE_CXX_KERNELS)
+ifeq ($(USE_CXX_RK4),1)
+RK4_C_OBJ = rungekutta4_rout_c.o
+RK4_F90_OBJ =
+else
+RK4_C_OBJ =
+RK4_F90_OBJ = rungekutta4_rout.o
+endif
+
+CFILES += $(RK4_C_OBJ)
+ABE_CUDA_CFILES = $(CFILES_CUDA_BSSN) z4c_rhs_cuda.o $(RK4_C_OBJ)
+
+ABE_LDLIBS = $(LDLIBS)
+ifeq ($(USE_CUDA_BSSN),1)
+ABE_LDLIBS += -lcudart $(CUDA_LIB_PATH)
+endif
+ifeq ($(USE_CUDA_Z4C),1)
+ABE_LDLIBS += -lcudart $(CUDA_LIB_PATH)
+endif
+
+C++FILES = ABE.o Ansorg.o Block.o misc.o monitor.o Parallel.o MPatch.o var.o\
+           cgh.o bssn_class.o surface_integral.o ShellPatch.o\
+	   bssnEScalar_class.o perf.o Z4c_class.o NullShellPatch.o\
+	   bssnEM_class.o cpbc_util.o z4c_rhs_point.o checkpoint.o\
+           Parallel_bam.o scalar_class.o transpbh.o NullShellPatch2.o\
+	   NullShellPatch2_Evo.o writefile_f.o interp_lb_profile.o
+	   
+#C++FILES_GPU = ABE.o Ansorg.o Block.o misc.o monitor.o Parallel.o MPatch.o var.o\
+           cgh.o surface_integral.o ShellPatch.o\
+	   bssnEScalar_class.o perf.o Z4c_class.o NullShellPatch.o\
+	   bssnEM_class.o cpbc_util.o z4c_rhs_point.o checkpoint.o\
+           Parallel_bam.o scalar_class.o transpbh.o NullShellPatch2.o\
+	   NullShellPatch2_Evo.o \
+	   bssn_gpu_class.o bssn_step_gpu.o bssn_macro.o writefile_f.o
+
+F90FILES_BASE = enforce_algebra.o fmisc.o initial_puncture.o prolongrestrict.o\
+	   prolongrestrict_cell.o prolongrestrict_vertex.o\
+	   $(RK4_F90_OBJ) diff_new.o kodiss.o kodiss_sh.o\
+	   lopsidediff.o sommerfeld_rout.o getnp4.o diff_new_sh.o\
+	   shellfunctions.o bssn_rhs_ss.o Set_Rho_ADM.o\
+           getnp4EScalar.o bssnEScalar_rhs.o bssn_constraint.o ricci_gamma.o\
+           fadmquantites_bssn.o $(Z4C_F90_OBJ) Z4c_rhs_ss.o point_diff_new_sh.o\
+	   cpbc.o getnp4old.o NullEvol.o initial_null.o initial_maxwell.o\
+	   getnpem2.o empart.o NullNews.o fourdcurvature.o\
+	   bssn2adm.o adm_constraint.o adm_ricci_gamma.o\
+	   scalar_rhs.o initial_scalar.o NullEvol2.o initial_null2.o\
+	   NullNews2.o tool_f.o
+
+ifeq ($(USE_CXX_KERNELS),0)
+# Fortran mode: include original bssn_rhs.o
+F90FILES = $(F90FILES_BASE) bssn_rhs.o
+else
+# C++ mode (default): bssn_rhs.o replaced by C++ kernel
+F90FILES = $(F90FILES_BASE)
+endif
+
+F77FILES = zbesh.o
+
+AHFDOBJS = expansion.o expansion_Jacobian.o patch.o coords.o patch_info.o patch_interp.o patch_system.o \
+tgrid.o fd_grid.o ghost_zone.o array.o round.o norm.o fuzzy.o error_exit.o miscfp.o \
+linear_map.o cpm_map.o BH_diagnostics.o setup.o horizon_sequence.o find_horizons.o \
+initial_guess.o Newton.o Jacobian.o ilucg.o IntPnts0.o IntPnts.o
+
+TwoPunctureFILES = TwoPunctureABE.o TwoPunctures.o 
+
+#CUDAFILES = bssn_gpu.o bssn_gpu_rhs_ss.o
+
+# file dependences
+$(C++FILES) $(C++FILES_GPU) $(F90FILES) $(CFILES) $(ABE_CUDA_CFILES) $(AHFDOBJS) $(CUDAFILES): macrodef.fh
+
+$(C++FILES): Block.h enforce_algebra.h fmisc.h initial_puncture.h macrodef.h\
+	     misc.h monitor.h MyList.h Parallel.h MPatch.h prolongrestrict.h\
+	     rungekutta4_rout.h var.h bssn_class.h bssn_rhs.h sommerfeld_rout.h\
+	     cgh.h surface_integral.h ShellPatch.h shellfunctions.h perf.h\
+             fadmquantites_bssn.h cpbc.h getnp4.h initial_null.h NullEvol.h\
+	     NullShellPatch.h initial_maxwell.h bssnEM_class.h getnpem2.h\
+	     empart.h NullNews.h kodiss.h Parallel_bam.h ricci_gamma.h\
+             initial_null2.h NullShellPatch2.h 
+             
+#$(C++FILES_GPU): Block.h enforce_algebra.h fmisc.h initial_puncture.h macrodef.h\
+	     misc.h monitor.h MyList.h Parallel.h MPatch.h prolongrestrict.h\
+	     rungekutta4_rout.h var.h bssn_rhs.h sommerfeld_rout.h\
+	     cgh.h surface_integral.h ShellPatch.h shellfunctions.h perf.h\
+             fadmquantites_bssn.h cpbc.h getnp4.h initial_null.h NullEvol.h\
+	     NullShellPatch.h initial_maxwell.h bssnEM_class.h getnpem2.h\
+	     empart.h NullNews.h kodiss.h Parallel_bam.h ricci_gamma.h\
+             initial_null2.h NullShellPatch2.h \
+             bssn_gpu_class.h bssn_macro.h
+             
+$(AHFDOBJS): cctk.h cctk_Config.h cctk_Types.h cctk_Constants.h myglobal.h
+
+$(C++FILES) $(C++FILES_GPU) $(CFILES) $(ABE_CUDA_CFILES) $(AHFDOBJS) $(CUDAFILES): macrodef.h
+
+TwoPunctureFILES: TwoPunctures.h
+
+$(CUDAFILES): bssn_gpu.h gpu_mem.h gpu_rhsSS_mem.h
+
+misc.o : zbesh.o
+
+# projects
+ABE: $(C++FILES) $(CFILES) $(F90FILES) $(F77FILES) $(AHFDOBJS)
+	$(CLINKER) $(CXXAPPFLAGS) -o $@ $(C++FILES) $(CFILES) $(F90FILES) $(F77FILES) $(AHFDOBJS) $(ABE_LDLIBS)
+
+ABE_CUDA: USE_CUDA_BSSN=1
+ABE_CUDA: USE_CUDA_Z4C=1
+ABE_CUDA: $(C++FILES) $(ABE_CUDA_CFILES) $(F90FILES) $(F77FILES) $(AHFDOBJS)
+	$(CLINKER) $(CXXAPPFLAGS) -o $@ $(C++FILES) $(ABE_CUDA_CFILES) $(F90FILES) $(F77FILES) $(AHFDOBJS) $(LDLIBS) -lcudart $(CUDA_LIB_PATH)
+	
+#ABEGPU: $(C++FILES_GPU) $(CFILES) $(F90FILES) $(F77FILES) $(AHFDOBJS) $(CUDAFILES)
+#	$(CLINKER) $(CXXAPPFLAGS) -o $@ $(C++FILES_GPU) $(CFILES) $(F90FILES) $(F77FILES) $(AHFDOBJS) $(CUDAFILES) $(LDLIBS)
+
+TwoPunctureABE: $(TwoPunctureFILES)
+	$(CLINKER) $(TP_OPTFLAGS) $(OMP_FLAG) -o $@ $(TwoPunctureFILES) $(LDLIBS)
+
+clean:
+	rm *.o ABE ABE_CUDA ABEGPU TwoPunctureABE make.log -f

AMSS_NCKU_source/makefile.inc

@@ -1,28 +1,7 @@
-## GCC version (commented out)
-## filein  = -I/usr/include -I/usr/lib/x86_64-linux-gnu/mpich/include -I/usr/lib/x86_64-linux-gnu/openmpi/lib/ -I/usr/lib/gcc/x86_64-linux-gnu/11/ -I/usr/include/c++/11/
-## filein  = -I/usr/include/ -I/usr/include/openmpi-x86_64/ -I/usr/lib/x86_64-linux-gnu/openmpi/include/ -I/usr/lib/x86_64-linux-gnu/openmpi/lib/ -I/usr/lib/gcc/x86_64-linux-gnu/11/ -I/usr/include/c++/11/
-## LDLIBS  = -L/usr/lib/x86_64-linux-gnu -L/usr/lib64 -L/usr/lib/gcc/x86_64-linux-gnu/11 -lgfortran -lmpi -lgfortran
-
-## Intel oneAPI version with oneMKL (Optimized for performance)
-filein  = -I/usr/include/ -I${MKLROOT}/include
-
-## Using sequential MKL (OpenMP disabled for better single-threaded performance)
-## Added -lifcore for Intel Fortran runtime and -limf for Intel math library
-LDLIBS  = -L${MKLROOT}/lib -lmkl_intel_lp64 -lmkl_sequential -lmkl_core -lifcore -limf -lpthread -lm -ldl -liomp5
-
-## Memory allocator switch
-##   1 (default) : link Intel oneTBB allocator (libtbbmalloc)
-##   0           : use system default allocator (ptmalloc)
-USE_TBBMALLOC ?= 1
-TBBMALLOC_SO ?= /home/intel/oneapi/2025.3/lib/libtbbmalloc.so
-ifneq ($(wildcard $(TBBMALLOC_SO)),)
-TBBMALLOC_LIBS = -Wl,--no-as-needed $(TBBMALLOC_SO) -Wl,--as-needed
-else
-TBBMALLOC_LIBS = -Wl,--no-as-needed -ltbbmalloc -Wl,--as-needed
-endif
-ifeq ($(USE_TBBMALLOC),1)
-LDLIBS := $(TBBMALLOC_LIBS) $(LDLIBS)
-endif
+## Toolchain selection
+##   nvhpc : NVIDIA HPC SDK + CUDA-aware MPI (default)
+##   intel : Intel oneAPI toolchain (legacy path)
+TOOLCHAIN ?= nvhpc
 
 ## PGO build mode switch (ABE only; TwoPunctureABE always uses opt flags)
 ##   opt        : (default) maximum performance with PGO profile-guided optimization
@@ -43,6 +22,14 @@ else
 INTERP_LB_FLAGS =
 endif
 
+MKLROOT ?= /home/intel/oneapi/mkl/latest
+MKL_LIBDIR ?= $(MKLROOT)/lib/intel64
+MKL_INC ?= -I$(MKLROOT)/include
+
+NVHPC_ROOT ?= /home/nvidia/hpc_sdk/Linux_x86_64/25.11
+CUDA_HOME  ?= $(NVHPC_ROOT)/cuda
+CUDA_ARCH  ?= sm_80
+
 ## Kernel implementation switch
 ##   1 (default) : use C++ rewrite of bssn_rhs and helper kernels (faster)
 ##   0           : fall back to original Fortran kernels
@@ -53,36 +40,52 @@ USE_CXX_KERNELS ?= 1
 ##   0           : use original Fortran Z4c_rhs.o
 USE_CXX_Z4C_KERNELS ?= 1
 
-## BSSN-EScalar RHS switch
-##   1 (default) : use BSSN-EScalar C wrapper on the normal patch path
-##   0           : keep the original Fortran BSSN-EScalar RHS for precision-safe runs
-## Note: this requires USE_CXX_KERNELS=1 because the wrapper reuses the C BSSN kernel.
-USE_CXX_ESCALAR_KERNEL ?= 1
-
-## BSSN-EM RHS switch
-##   1 : use BSSN-EM C kernel (bssn_em_rhs_c.C) on the normal patch path
-##   0 : keep the original Fortran empart.f90 RHS for the EM fields (default)
-## Note: experimental, requires USE_CXX_KERNELS=1
-USE_CXX_EM_KERNEL ?= 0
-
-## Cached transfer switch
-##   auto (default): enable for BSSN vacuum, keep other paths on the safe uncached path
-##   1             : force cached Sync/Restrict/OutBd transfer on evolution hot paths
-##   0             : force the original uncached transfer path
-USE_TRANSFER_CACHE ?= auto
-
 ## RK4 kernel implementation switch
 ##   1 (default) : use C/C++ rewrite of rungekutta4_rout (for optimization experiments)
 ##   0           : use original Fortran rungekutta4_rout.o
 USE_CXX_RK4 ?= 1
 
+## Memory allocator switch
+##   1 (default) : link Intel oneTBB allocator (libtbbmalloc)
+##   0           : use system default allocator (ptmalloc)
+USE_TBBMALLOC ?= 1
+TBBMALLOC_SO ?= /home/intel/oneapi/2025.3/lib/libtbbmalloc.so
+ifneq ($(wildcard $(TBBMALLOC_SO)),)
+TBBMALLOC_LIBS = -Wl,--no-as-needed $(TBBMALLOC_SO) -Wl,--as-needed
+else
+TBBMALLOC_LIBS = -Wl,--no-as-needed -ltbbmalloc -Wl,--as-needed
+endif
+
+ifeq ($(TOOLCHAIN),intel)
 f90          = ifx
 f77          = ifx
 CXX          = icpx
 CC           = icx
 CLINKER      = mpiicpx
+filein  = -I/usr/include/ $(MKL_INC) -I$(CUDA_HOME)/include
+LDLIBS       = -L$(MKL_LIBDIR) -Wl,-rpath,$(MKL_LIBDIR) \
+               -lmkl_intel_lp64 -lmkl_sequential -lmkl_core \
+               -lifcore -limf -liomp5 -lpthread -lm -ldl \
+               -L$(CUDA_HOME)/lib64 -Wl,-rpath,$(CUDA_HOME)/lib64 -lcuda -lcudart
+else ifeq ($(TOOLCHAIN),nvhpc)
+f90          = mpifort
+f77          = mpifort
+CXX          = mpicxx
+CC           = mpicc
+CLINKER      = mpicxx
 
-Cu = nvcc
-CUDA_LIB_PATH = -L/usr/lib/cuda/lib64 -I/usr/include -I/usr/lib/cuda/include
-#CUDA_APP_FLAGS = -c -g -O3 --ptxas-options=-v -arch compute_13 -code compute_13,sm_13 -Dfortran3 -Dnewc
-CUDA_APP_FLAGS = -c -g -O3 --ptxas-options=-v -Dfortran3 -Dnewc
+filein       = -I/usr/include/ $(MKL_INC) -I$(CUDA_HOME)/include
+LDLIBS       = -L$(MKL_LIBDIR) -Wl,-rpath,$(MKL_LIBDIR) \
+               -lmkl_intel_lp64 -lmkl_sequential -lmkl_core \
+               -lpthread -lm -ldl \
+               -L$(CUDA_HOME)/lib64 -Wl,-rpath,$(CUDA_HOME)/lib64 -lcuda -lcudart \
+               -fortranlibs
+endif
+
+ifeq ($(USE_TBBMALLOC),1)
+LDLIBS := $(TBBMALLOC_LIBS) $(LDLIBS)
+endif
+
+Cu = $(NVHPC_ROOT)/compilers/bin/nvcc
+CUDA_LIB_PATH = -L$(CUDA_HOME)/lib64 -I$(CUDA_HOME)/include
+CUDA_APP_FLAGS = -c -g -O3 --ptxas-options=-v -Dfortran3 -Dnewc -arch=$(CUDA_ARCH)

AMSS_NCKU_source/share_func.h

@@ -46,45 +46,6 @@ static inline size_t idx_fh_F(int iF, int jF, int kF, const int ex[3]) {
     return (size_t)ii + (size_t)jj * (size_t)nx + (size_t)kk * (size_t)nx * (size_t)ny;
 }
 
-/*
- * fh 对应 Fortran: fh(0:ex1, 0:ex2, 0:ex3)
- * ord=1 => shift=0
- * iF/jF/kF 为 Fortran 索引 (0..ex)
- */
-static inline size_t idx_fh_F_ord1(int iF, int jF, int kF, const int ex[3]) {
-    const int nx = ex[0] + 1;            // ex1 + ord
-    const int ny = ex[1] + 1;
-    return (size_t)iF + (size_t)jF * (size_t)nx + (size_t)kF * (size_t)nx * (size_t)ny;
-}
-
-/*
- * fh 对应 Fortran: fh(-3:ex1, -3:ex2, -3:ex3)
- * ord=4 => shift=3
- */
-static inline size_t idx_fh_F_ord4(int iF, int jF, int kF, const int ex[3]) {
-    const int shift = 3;
-    const int nx = ex[0] + 4;            // ex1 + ord
-    const int ny = ex[1] + 4;
-    const int ii = iF + shift;           // 0..ex1+3
-    const int jj = jF + shift;           // 0..ex2+3
-    const int kk = kF + shift;           // 0..ex3+3
-    return (size_t)ii + (size_t)jj * (size_t)nx + (size_t)kk * (size_t)nx * (size_t)ny;
-}
-
-/*
- * fh 对应 Fortran: fh(-4:ex1, -4:ex2, -4:ex3)
- * ord=5 => shift=4
- */
-static inline size_t idx_fh_F_ord5(int iF, int jF, int kF, const int ex[3]) {
-    const int shift = 4;
-    const int nx = ex[0] + 5;            // ex1 + ord
-    const int ny = ex[1] + 5;
-    const int ii = iF + shift;           // 0..ex1+4
-    const int jj = jF + shift;           // 0..ex2+4
-    const int kk = kF + shift;           // 0..ex3+4
-    return (size_t)ii + (size_t)jj * (size_t)nx + (size_t)kk * (size_t)nx * (size_t)ny;
-}
-
 /*
  * func:  (1..extc1, 1..extc2, 1..extc3)   1-based in Fortran
  * funcc: (-ord+1..extc1, -ord+1..extc2, -ord+1..extc3) in Fortran
@@ -270,10 +231,7 @@ static inline void symmetry_bd(int ord,
 {
     if (ord <= 0) return;
 
-    if (ord == 1) {
-        symmetry_bd_impl(1, 0, extc, func, funcc, SoA);
-        return;
-    }
+    /* Fast paths used by current C kernels: ord=2 (derivs), ord=3 (lopsided/KO). */
     if (ord == 2) {
         symmetry_bd_impl(2, 1, extc, func, funcc, SoA);
         return;
@@ -282,91 +240,7 @@ static inline void symmetry_bd(int ord,
         symmetry_bd_impl(3, 2, extc, func, funcc, SoA);
         return;
     }
-    if (ord == 4) {
-        symmetry_bd_impl(4, 3, extc, func, funcc, SoA);
-        return;
-    }
 
     symmetry_bd_impl(ord, ord - 1, extc, func, funcc, SoA);
 }
-
-/*
- * symmetry_stbd — shell-patch (staggered boundary) ghost fill.
- *
- * Fortran: funcc(-ord+1:extc1+ord, -ord+1:extc2+ord, extc3)
- * Only 2 SoA values (x/y). No z symmetry fill.
- * Ghost on BOTH positive and negative sides of x and y.
- * Reflection uses i+2 (skips boundary) instead of i+1.
- * nx = extc1 + 2*ord, ny = extc2 + 2*ord
- */
-static inline void symmetry_stbd(int ord,
-                                 const int extc[3],
-                                 const double *func,
-                                 double *funcc,
-                                 const double SoA[2])
-{
-    const int extc1 = extc[0], extc2 = extc[1], extc3 = extc[2];
-    const int nx = extc1 + 2 * ord;
-    const int ny = extc2 + 2 * ord;
-    const int sh = ord - 1;
-    const size_t snx = (size_t)nx;
-    const size_t splane = snx * (size_t)ny;
-
-    /* 1) Copy interior: funcc(1:extc1, 1:extc2, 1:extc3) = func */
-    for (int k0 = 0; k0 < extc3; ++k0) {
-        const double *src = func + (size_t)k0 * (size_t)extc2 * (size_t)extc1;
-        const size_t kbase = (size_t)k0 * splane;
-        for (int j0 = 0; j0 < extc2; ++j0) {
-            double *dst = funcc + kbase + (size_t)(sh + j0 + 1) * snx + (size_t)(sh + 1);
-            const double *s = src + (size_t)j0 * (size_t)extc1;
-            for (int i0 = 0; i0 < extc1; ++i0) dst[i0] = s[i0];
-        }
-    }
-
-    /* 2) x-direction ghost fill */
-    const double s1 = SoA[0];
-    for (int k0 = 0; k0 < extc3; ++k0) {
-        const size_t kbase = (size_t)k0 * splane;
-        for (int j0 = 0; j0 < extc2; ++j0) {
-            const size_t off = kbase + (size_t)(sh + j0 + 1) * snx;
-            /* left side: funcc(-i) = funcc(i+2) * s1 */
-            for (int i = 0; i < ord; ++i) {
-                funcc[off + (size_t)(sh - i)]       = funcc[off + (size_t)(sh + i + 2)] * s1;
-                /* right side: funcc(extc1+1+i) = funcc(extc1-1-i) * s1 */
-                funcc[off + (size_t)(sh + extc1 + 1 + i)] = funcc[off + (size_t)(sh + extc1 - 1 - i)] * s1;
-            }
-        }
-    }
-
-    /* 3) y-direction ghost fill */
-    const double s2 = SoA[1];
-    for (int i = 0; i < nx; ++i) {
-        for (int k0 = 0; k0 < extc3; ++k0) {
-            const size_t kbase = (size_t)k0 * splane;
-            /* bottom: funcc(:,-i,:) = funcc(:,i+2,:) * s2 */
-            for (int jj = 0; jj < ord; ++jj) {
-                funcc[kbase + (size_t)(sh - jj) * snx + (size_t)i] =
-                    funcc[kbase + (size_t)(sh + jj + 2) * snx + (size_t)i] * s2;
-                /* top: funcc(:,extc2+1+jj,:) = funcc(:,extc2-1-jj,:) * s2 */
-                funcc[kbase + (size_t)(sh + extc2 + 1 + jj) * snx + (size_t)i] =
-                    funcc[kbase + (size_t)(sh + extc2 - 1 - jj) * snx + (size_t)i] * s2;
-            }
-        }
-    }
-}
-
-/*
- * Indexing for shell fh buffer: Fortran fh(-ord+1:extc1+ord, -ord+1:extc2+ord, extc3)
- * C 0-based: ii = iF + ord - 1
- * nx = extc1 + 2*ord, ny = extc2 + 2*ord
- */
-static inline size_t idx_fh_stbd(int iF, int jF, int kF, int ord, const int extc[3]) {
-    const int sh = ord - 1;
-    const int nx = extc[0] + 2 * ord;
-    const int ny = extc[1] + 2 * ord;
-    const int ii = iF + sh;
-    const int jj = jF + sh;
-    const int kk = kF - 1;  // Fortran 1-based kF → C 0-based
-    return (size_t)ii + (size_t)jj * (size_t)nx + (size_t)kk * (size_t)nx * (size_t)ny;
-}
 #endif

AMSS_NCKU_source/surface_integral.C

@@ -8,10 +8,11 @@
 #include <iostream>
 #include <iomanip>
 #include <fstream>
-#include <strstream>
-#include <cmath>
-#include <map>
-using namespace std;
+#include <strstream>
+#include <cmath>
+#include <map>
+#include <cstdlib>
+using namespace std;
 #else
 #include <iostream.h>
 #include <iomanip.h>
@@ -29,12 +30,26 @@ using namespace std;
 #include "fadmquantites_bssn.h"
 #include "getnpem2.h"
 #include "getnp4.h"
-#include "parameters.h"
-
-#define PI M_PI
-//|============================================================================
-//| Constructor
-//|============================================================================
+#include "parameters.h"
+
+#define PI M_PI
+
+namespace
+{
+bool amss_surface_timing_enabled()
+{
+  static int enabled = -1;
+  if (enabled < 0)
+  {
+    const char *env = getenv("AMSS_SURFACE_TIMING");
+    enabled = (env && atoi(env) != 0) ? 1 : 0;
+  }
+  return enabled != 0;
+}
+}
+//|============================================================================
+//| Constructor
+//|============================================================================
 
 surface_integral::surface_integral(int iSymmetry) : Symmetry(iSymmetry),
                                                     wave_cache_spinw(-1),
@@ -484,9 +499,9 @@ void surface_integral::surf_Wave(double rex, int lev, cgh *GH, var *Rpsi4, var *
   delete[] IP_out;
   DG_List->clearList();
 }
-void surface_integral::surf_Wave(double rex, int lev, cgh *GH, var *Rpsi4, var *Ipsi4,
-                                 int spinw, int maxl, int NN, double *RP, double *IP,
-                                 monitor *Monitor, MPI_Comm Comm_here) // NN is the length of RP and IP
+void surface_integral::surf_Wave(double rex, int lev, cgh *GH, var *Rpsi4, var *Ipsi4,
+                                 int spinw, int maxl, int NN, double *RP, double *IP,
+                                 monitor *Monitor, MPI_Comm Comm_here) // NN is the length of RP and IP
 {
   //   misc::tillherecheck(GH->Commlev[lev],GH->start_rank[lev],"start surface_integral::surf_Wave");
 
@@ -720,10 +735,10 @@ void surface_integral::surf_Wave(double rex, int lev, cgh *GH, var *Rpsi4, var *
   delete[] IP_out;
   DG_List->clearList();
 }
-//|----------------------------------------------------------------
-//  for shell patch
-//|----------------------------------------------------------------
-void surface_integral::surf_Wave(double rex, int lev, ShellPatch *GH, var *Rpsi4, var *Ipsi4,
+//|----------------------------------------------------------------
+//  for shell patch
+//|----------------------------------------------------------------
+void surface_integral::surf_Wave(double rex, int lev, ShellPatch *GH, var *Rpsi4, var *Ipsi4,
                                  int spinw, int maxl, int NN, double *RP, double *IP,
                                  monitor *Monitor) // NN is the length of RP and IP
 {
@@ -3281,6 +3296,8 @@ void surface_integral::surf_WaveMassPAng(double rex, int lev, cgh *GH,
                                          var *Sfx_rhs, var *Sfy_rhs, var *Sfz_rhs,
                                          double *Rout, monitor *Monitor, bool refresh_mass_fields)
 {
+  const bool timing = amss_surface_timing_enabled();
+  const double t_start = timing ? MPI_Wtime() : 0.0;
   if (Symmetry != 0 && Symmetry != 1)
   {
     surf_Wave(rex, lev, GH, Rpsi4, Ipsi4, spinw, maxl, NN, RP, IP, Monitor);
@@ -3325,6 +3342,7 @@ void surface_integral::surf_WaveMassPAng(double rex, int lev, cgh *GH,
       Pp = Pp->next;
     }
   }
+  const double t_refresh_done = timing ? MPI_Wtime() : 0.0;
 
   const int InList = 19;
   const int idx_rpsi4 = 0, idx_ipsi4 = 1;
@@ -3380,6 +3398,7 @@ void surface_integral::surf_WaveMassPAng(double rex, int lev, cgh *GH,
 
   double *shellf = new double[n_tot * InList];
   GH->PatL[lev]->data->Interp_Points(DG_List, n_tot, pox, shellf, Symmetry, Nmin, Nmax);
+  const double t_interp_done = timing ? MPI_Wtime() : 0.0;
 
   double *RP_out = new double[NN];
   double *IP_out = new double[NN];
@@ -3496,6 +3515,7 @@ void surface_integral::surf_WaveMassPAng(double rex, int lev, cgh *GH,
     if (Symmetry == 0)
       p_outz += f1o8 * Psi * (nx_g[n] * axz + ny_g[n] * ayz + nz_g[n] * azz) * theta_weight;
   }
+  const double t_integral_done = timing ? MPI_Wtime() : 0.0;
 
   for (int ii = 0; ii < NN; ii++)
   {
@@ -3534,6 +3554,7 @@ void surface_integral::surf_WaveMassPAng(double rex, int lev, cgh *GH,
     delete[] reduce_out;
     delete[] reduce_in;
   }
+  const double t_reduce_done = timing ? MPI_Wtime() : 0.0;
 
 #ifdef GaussInt
   mass = mass * rex * rex * dphi * factor;
@@ -3565,6 +3586,19 @@ void surface_integral::surf_WaveMassPAng(double rex, int lev, cgh *GH,
   Rout[5] = sy;
   Rout[6] = sz;
 
+  if (timing)
+  {
+    fprintf(stderr,
+            "[AMSS-SURFACE][rank %d] rex=%.6g lev=%d refresh=%.6f interp=%.6f integral=%.6f reduce=%.6f total=%.6f nlocal=%d ntotal=%d modes=%d\n",
+            myrank, rex, lev,
+            t_refresh_done - t_start,
+            t_interp_done - t_refresh_done,
+            t_integral_done - t_interp_done,
+            t_reduce_done - t_integral_done,
+            t_reduce_done - t_start,
+            Nmax - Nmin + 1, n_tot, NN);
+  }
+
   delete[] pox[0];
   delete[] pox[1];
   delete[] pox[2];

AMSS_NCKU_source/surface_integral.h

@@ -46,10 +46,10 @@ public:
 	surface_integral(int iSymmetry);
 	~surface_integral();
 
-	void surf_Wave(double rex, int lev, cgh *GH, var *Rpsi4, var *Ipsi4,
-				   int spinw, int maxl, int NN, double *RP, double *IP,
-				   monitor *Monitor); // NN is the length of RP and IP
-									  // this routine can only deal with the symmetry of Psi4
+	void surf_Wave(double rex, int lev, cgh *GH, var *Rpsi4, var *Ipsi4,
+				   int spinw, int maxl, int NN, double *RP, double *IP,
+				   monitor *Monitor); // NN is the length of RP and IP
+									  // this routine can only deal with the symmetry of Psi4
 	void surf_Wave(double rex, int lev, ShellPatch *GH, var *Rpsi4, var *Ipsi4,
 				   int spinw, int maxl, int NN, double *RP, double *IP,
 				   monitor *Monitor);

AMSS_NCKU_source/z4c_rhs_c.C

@@ -327,9 +327,6 @@ static int compute_rhs_z4c_cartesian(
     double Axxx[all], Axxy[all], Axxz[all], Axyx[all], Axyy[all], Axyz[all];
     double Axzx[all], Axzy[all], Axzz[all], Ayyx[all], Ayyy[all], Ayyz[all];
     double Ayzx[all], Ayzy[all], Ayzz[all], Azzx[all], Azzy[all], Azzz[all];
-#if (GAUGE == 2 || GAUGE == 3 || GAUGE == 4 || GAUGE == 5)
-    double reta[all];
-#endif
 
     const double SSS[3] = {1.0, 1.0, 1.0};
     const double AAS[3] = {-1.0, -1.0, 1.0};
@@ -479,181 +476,8 @@ static int compute_rhs_z4c_cartesian(
         dtSfy_rhs[idx] = ZEO;
         dtSfz_rhs[idx] = ZEO;
     }
-#elif (GAUGE == 2)
-    /* Variable-eta gamma-driver, chi-sqrt denominator */
-    for (int idx = 0; idx < all; ++idx)
-    {
-        const double chin1i = chin1[idx];
-        const double det = gxx[idx] * gyy[idx] * gzz[idx]
-                         + gxy[idx] * gyz[idx] * gxz[idx] * 2.0
-                         - gxz[idx] * gyy[idx] * gxz[idx]
-                         - gxy[idx] * gxy[idx] * gzz[idx]
-                         - gxx[idx] * gyz[idx] * gyz[idx];
-        const double idet = ONE / det;
-        const double upxx = (gyy[idx] * gzz[idx] - gyz[idx] * gyz[idx]) * idet;
-        const double upxy = -(gxy[idx] * gzz[idx] - gyz[idx] * gxz[idx]) * idet;
-        const double upxz = (gxy[idx] * gyz[idx] - gyy[idx] * gxz[idx]) * idet;
-        const double upyy = (gxx[idx] * gzz[idx] - gxz[idx] * gxz[idx]) * idet;
-        const double upyz = -(gxx[idx] * gyz[idx] - gxy[idx] * gxz[idx]) * idet;
-        const double upzz = (gxx[idx] * gyy[idx] - gxy[idx] * gxy[idx]) * idet;
-        const double grdchi2 =
-            upxx * chix[idx] * chix[idx] + upyy * chiy[idx] * chiy[idx] + upzz * chiz[idx] * chiz[idx]
-          + TWO * (upxy * chix[idx] * chiy[idx] + upxz * chix[idx] * chiz[idx] + upyz * chiy[idx] * chiz[idx]);
-        const double sqchi = sqrt(chin1i);
-        reta[idx] = 1.31 / TWO * sqrt(grdchi2 / chin1i) / ((ONE - sqchi) * (ONE - sqchi));
-        betax_rhs[idx] = FF * dtSfx[idx];
-        betay_rhs[idx] = FF * dtSfy[idx];
-        betaz_rhs[idx] = FF * dtSfz[idx];
-        dtSfx_rhs[idx] = Gamx_rhs[idx] - reta[idx] * dtSfx[idx];
-        dtSfy_rhs[idx] = Gamy_rhs[idx] - reta[idx] * dtSfy[idx];
-        dtSfz_rhs[idx] = Gamz_rhs[idx] - reta[idx] * dtSfz[idx];
-    }
-#elif (GAUGE == 3)
-    /* Variable-eta gamma-driver, chi-linear denominator */
-    for (int idx = 0; idx < all; ++idx)
-    {
-        const double chin1i = chin1[idx];
-        const double det = gxx[idx] * gyy[idx] * gzz[idx]
-                         + gxy[idx] * gyz[idx] * gxz[idx] * 2.0
-                         - gxz[idx] * gyy[idx] * gxz[idx]
-                         - gxy[idx] * gxy[idx] * gzz[idx]
-                         - gxx[idx] * gyz[idx] * gyz[idx];
-        const double idet = ONE / det;
-        const double upxx = (gyy[idx] * gzz[idx] - gyz[idx] * gyz[idx]) * idet;
-        const double upxy = -(gxy[idx] * gzz[idx] - gyz[idx] * gxz[idx]) * idet;
-        const double upxz = (gxy[idx] * gyz[idx] - gyy[idx] * gxz[idx]) * idet;
-        const double upyy = (gxx[idx] * gzz[idx] - gxz[idx] * gxz[idx]) * idet;
-        const double upyz = -(gxx[idx] * gyz[idx] - gxy[idx] * gxz[idx]) * idet;
-        const double upzz = (gxx[idx] * gyy[idx] - gxy[idx] * gxy[idx]) * idet;
-        const double grdchi2 =
-            upxx * chix[idx] * chix[idx] + upyy * chiy[idx] * chiy[idx] + upzz * chiz[idx] * chiz[idx]
-          + TWO * (upxy * chix[idx] * chiy[idx] + upxz * chix[idx] * chiz[idx] + upyz * chiy[idx] * chiz[idx]);
-        reta[idx] = 1.31 / TWO * sqrt(grdchi2 / chin1i) / ((ONE - chin1i) * (ONE - chin1i));
-        betax_rhs[idx] = FF * dtSfx[idx];
-        betay_rhs[idx] = FF * dtSfy[idx];
-        betaz_rhs[idx] = FF * dtSfz[idx];
-        dtSfx_rhs[idx] = Gamx_rhs[idx] - reta[idx] * dtSfx[idx];
-        dtSfy_rhs[idx] = Gamy_rhs[idx] - reta[idx] * dtSfy[idx];
-        dtSfz_rhs[idx] = Gamz_rhs[idx] - reta[idx] * dtSfz[idx];
-    }
-#elif (GAUGE == 4)
-    /* Variable-eta gamma-driver, first-order, chi-sqrt denominator */
-    for (int idx = 0; idx < all; ++idx)
-    {
-        const double chin1i = chin1[idx];
-        const double det = gxx[idx] * gyy[idx] * gzz[idx]
-                         + gxy[idx] * gyz[idx] * gxz[idx] * 2.0
-                         - gxz[idx] * gyy[idx] * gxz[idx]
-                         - gxy[idx] * gxy[idx] * gzz[idx]
-                         - gxx[idx] * gyz[idx] * gyz[idx];
-        const double idet = ONE / det;
-        const double upxx = (gyy[idx] * gzz[idx] - gyz[idx] * gyz[idx]) * idet;
-        const double upxy = -(gxy[idx] * gzz[idx] - gyz[idx] * gxz[idx]) * idet;
-        const double upxz = (gxy[idx] * gyz[idx] - gyy[idx] * gxz[idx]) * idet;
-        const double upyy = (gxx[idx] * gzz[idx] - gxz[idx] * gxz[idx]) * idet;
-        const double upyz = -(gxx[idx] * gyz[idx] - gxy[idx] * gxz[idx]) * idet;
-        const double upzz = (gxx[idx] * gyy[idx] - gxy[idx] * gxy[idx]) * idet;
-        const double grdchi2 =
-            upxx * chix[idx] * chix[idx] + upyy * chiy[idx] * chiy[idx] + upzz * chiz[idx] * chiz[idx]
-          + TWO * (upxy * chix[idx] * chiy[idx] + upxz * chix[idx] * chiz[idx] + upyz * chiy[idx] * chiz[idx]);
-        const double sqchi = sqrt(chin1i);
-        reta[idx] = 1.31 / TWO * sqrt(grdchi2 / chin1i) / ((ONE - sqchi) * (ONE - sqchi));
-        betax_rhs[idx] = Gamx_rhs[idx] - reta[idx] * betax[idx];
-        betay_rhs[idx] = Gamy_rhs[idx] - reta[idx] * betay[idx];
-        betaz_rhs[idx] = Gamz_rhs[idx] - reta[idx] * betaz[idx];
-        dtSfx_rhs[idx] = ZEO;
-        dtSfy_rhs[idx] = ZEO;
-        dtSfz_rhs[idx] = ZEO;
-    }
-#elif (GAUGE == 5)
-    /* Variable-eta gamma-driver, first-order, chi-linear denominator */
-    for (int idx = 0; idx < all; ++idx)
-    {
-        const double chin1i = chin1[idx];
-        const double det = gxx[idx] * gyy[idx] * gzz[idx]
-                         + gxy[idx] * gyz[idx] * gxz[idx] * 2.0
-                         - gxz[idx] * gyy[idx] * gxz[idx]
-                         - gxy[idx] * gxy[idx] * gzz[idx]
-                         - gxx[idx] * gyz[idx] * gyz[idx];
-        const double idet = ONE / det;
-        const double upxx = (gyy[idx] * gzz[idx] - gyz[idx] * gyz[idx]) * idet;
-        const double upxy = -(gxy[idx] * gzz[idx] - gyz[idx] * gxz[idx]) * idet;
-        const double upxz = (gxy[idx] * gyz[idx] - gyy[idx] * gxz[idx]) * idet;
-        const double upyy = (gxx[idx] * gzz[idx] - gxz[idx] * gxz[idx]) * idet;
-        const double upyz = -(gxx[idx] * gyz[idx] - gxy[idx] * gxz[idx]) * idet;
-        const double upzz = (gxx[idx] * gyy[idx] - gxy[idx] * gxy[idx]) * idet;
-        const double grdchi2 =
-            upxx * chix[idx] * chix[idx] + upyy * chiy[idx] * chiy[idx] + upzz * chiz[idx] * chiz[idx]
-          + TWO * (upxy * chix[idx] * chiy[idx] + upxz * chix[idx] * chiz[idx] + upyz * chiy[idx] * chiz[idx]);
-        reta[idx] = 1.31 / TWO * sqrt(grdchi2 / chin1i) / ((ONE - chin1i) * (ONE - chin1i));
-        betax_rhs[idx] = Gamx_rhs[idx] - reta[idx] * betax[idx];
-        betay_rhs[idx] = Gamy_rhs[idx] - reta[idx] * betay[idx];
-        betaz_rhs[idx] = Gamz_rhs[idx] - reta[idx] * betaz[idx];
-        dtSfx_rhs[idx] = ZEO;
-        dtSfy_rhs[idx] = ZEO;
-        dtSfz_rhs[idx] = ZEO;
-    }
-#elif (GAUGE == 6 || GAUGE == 7)
-    {
-        /* Jason's position-dependent damping: rational (6) or exponential (7) */
-        int BHN = 0;
-        double Porg[9] = {0.0};
-        double Mass[3] = {0.0};
-    #ifdef fortran1
-        extern "C" { void getpbh(int &, double *, double *); }
-    #elif defined(fortran2)
-        extern "C" { void GETPBH(int &, double *, double *); }
-    #else
-        extern "C" { void getpbh_(int &, double *, double *); }
-    #endif
-        {
-    #ifdef fortran1
-            getpbh(BHN, Porg, Mass);
-    #elif defined(fortran2)
-            GETPBH(BHN, Porg, Mass);
-    #else
-            getpbh_(BHN, Porg, Mass);
-    #endif
-        }
-        if (BHN == 2)
-        {
-            const double M = Mass[0] + Mass[1];
-            const double A = 2.0 / M;
-            const double w1 = 12.0, w2 = 12.0;
-            const double C1 = 1.0 / Mass[0] - A;
-            const double C2 = 1.0 / Mass[1] - A;
-            const double BH_sep2 = (Porg[3] - Porg[0]) * (Porg[3] - Porg[0])
-                                 + (Porg[4] - Porg[1]) * (Porg[4] - Porg[1])
-                                 + (Porg[5] - Porg[2]) * (Porg[5] - Porg[2]);
-            const double inv_BH_sep2 = 1.0 / BH_sep2;
-            for (int k0 = 0; k0 < nz; ++k0) {
-            for (int j0 = 0; j0 < ny; ++j0) {
-            for (int i0 = 0; i0 < nx; ++i0) {
-                const size_t idx = idx_ex(i0, j0, k0, ex);
-                const double xp = X[i0], yp = Y[j0], zp = Z[k0];
-                const double r1 = ((Porg[0]-xp)*(Porg[0]-xp) + (Porg[1]-yp)*(Porg[1]-yp) + (Porg[2]-zp)*(Porg[2]-zp)) * inv_BH_sep2;
-                const double r2 = ((Porg[3]-xp)*(Porg[3]-xp) + (Porg[4]-yp)*(Porg[4]-yp) + (Porg[5]-zp)*(Porg[5]-zp)) * inv_BH_sep2;
-            #if (GAUGE == 6)
-                const double reta_val = A + C1 / (1.0 + w1 * r1) + C2 / (1.0 + w2 * r2);
-            #else
-                const double reta_val = A + C1 * exp(-w1 * r1) + C2 * exp(-w2 * r2);
-            #endif
-                betax_rhs[idx] = FF * dtSfx[idx];
-                betay_rhs[idx] = FF * dtSfy[idx];
-                betaz_rhs[idx] = FF * dtSfz[idx];
-                dtSfx_rhs[idx] = Gamx_rhs[idx] - reta_val * dtSfx[idx];
-                dtSfy_rhs[idx] = Gamy_rhs[idx] - reta_val * dtSfy[idx];
-                dtSfz_rhs[idx] = Gamz_rhs[idx] - reta_val * dtSfz[idx];
-            }}}
-        }
-        else
-        {
-            fprintf(stderr, "z4c_rhs_c: GAUGE %d requires BHN=2, got BHN=%d\n", (int)GAUGE, BHN);
-            return 1;
-        }
-    }
 #else
-#error "z4c_rhs_c.C: unsupported GAUGE value"
+#error "z4c_rhs_c.C currently supports GAUGE == 0 or GAUGE == 1 for Z4C"
 #endif
 
     lopsided(ex, X, Y, Z, gxx, gxx_rhs, betax, betay, betaz, Symmetry, SSS);
@@ -681,7 +505,7 @@ static int compute_rhs_z4c_cartesian(
     lopsided(ex, X, Y, Z, betax, betax_rhs, betax, betay, betaz, Symmetry, ASS);
     lopsided(ex, X, Y, Z, betay, betay_rhs, betax, betay, betaz, Symmetry, SAS);
     lopsided(ex, X, Y, Z, betaz, betaz_rhs, betax, betay, betaz, Symmetry, SSA);
-#if (GAUGE == 0 || GAUGE == 2 || GAUGE == 3 || GAUGE == 6 || GAUGE == 7)
+#if (GAUGE == 0)
     lopsided(ex, X, Y, Z, dtSfx, dtSfx_rhs, betax, betay, betaz, Symmetry, ASS);
     lopsided(ex, X, Y, Z, dtSfy, dtSfy_rhs, betax, betay, betaz, Symmetry, SAS);
     lopsided(ex, X, Y, Z, dtSfz, dtSfz_rhs, betax, betay, betaz, Symmetry, SSA);
@@ -728,7 +552,7 @@ static int compute_rhs_z4c_cartesian(
         kodis(ex, X, Y, Z, betax, betax_rhs, ASS, Symmetry, eps);
         kodis(ex, X, Y, Z, betay, betay_rhs, SAS, Symmetry, eps);
         kodis(ex, X, Y, Z, betaz, betaz_rhs, SSA, Symmetry, eps);
-#if (GAUGE == 0 || GAUGE == 2 || GAUGE == 3 || GAUGE == 6 || GAUGE == 7)
+#if (GAUGE == 0)
         kodis(ex, X, Y, Z, dtSfx, dtSfx_rhs, ASS, Symmetry, eps);
         kodis(ex, X, Y, Z, dtSfy, dtSfy_rhs, SAS, Symmetry, eps);
         kodis(ex, X, Y, Z, dtSfz, dtSfz_rhs, SSA, Symmetry, eps);

AMSS_NCKU_source/z4c_rhs_cuda.h

@@ -0,0 +1,258 @@
+#ifndef Z4C_RHS_CUDA_H
+#define Z4C_RHS_CUDA_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+enum {
+    Z4C_CUDA_STATE_COUNT = 25
+};
+
+int z4c_cuda_rk4_substep(void *block_tag,
+                         int *ex, double *X, double *Y, double *Z,
+                         double **state_host_in,
+                         double **state_host_out,
+                         const double *propspeed,
+                         const double *soa_flat,
+                         const double *bbox,
+                         double &dT,
+                         double &T,
+                         int &RK4,
+                         int &apply_bam_bc,
+                         int &Symmetry,
+                         int &Lev,
+                         double &eps,
+                         int &co,
+                         int &keep_resident_state,
+                         int &apply_enforce_ga,
+                         double &chitiny);
+
+int z4c_cuda_download_resident_state(void *block_tag,
+                                     int *ex,
+                                     double **state_host_out);
+
+int z4c_cuda_pack_state_region_to_host_buffer(void *block_tag,
+                                              int state_index,
+                                              double *host_buffer,
+                                              int *ex,
+                                              int i0, int j0, int k0,
+                                              int sx, int sy, int sz);
+
+int z4c_cuda_unpack_state_region_from_host_buffer(void *block_tag,
+                                                  int state_index,
+                                                  double *host_buffer,
+                                                  int *ex,
+                                                  int i0, int j0, int k0,
+                                                  int sx, int sy, int sz);
+
+int z4c_cuda_pack_state_batch_to_host_buffer(void *block_tag,
+                                             int state_count,
+                                             double *host_buffer,
+                                             int *ex,
+                                             int i0, int j0, int k0,
+                                             int sx, int sy, int sz);
+
+int z4c_cuda_pack_state_batch_to_host_buffer_for_host_views(void *block_tag,
+                                                            double **state_host_key,
+                                                            int state_count,
+                                                            double *host_buffer,
+                                                            int *ex,
+                                                            int i0, int j0, int k0,
+                                                            int sx, int sy, int sz);
+
+int z4c_cuda_unpack_state_batch_from_host_buffer(void *block_tag,
+                                                 int state_count,
+                                                 double *host_buffer,
+                                                 int *ex,
+                                                 int i0, int j0, int k0,
+                                                 int sx, int sy, int sz);
+
+int z4c_cuda_unpack_state_batch_from_host_buffer_for_host_views(void *block_tag,
+                                                                double **state_host_key,
+                                                                int state_count,
+                                                                double *host_buffer,
+                                                                int *ex,
+                                                                int i0, int j0, int k0,
+                                                                int sx, int sy, int sz);
+
+int z4c_cuda_pack_state_batch_to_device_buffer(void *block_tag,
+                                               int state_count,
+                                               double *device_buffer,
+                                               int *ex,
+                                               int i0, int j0, int k0,
+                                               int sx, int sy, int sz);
+
+int z4c_cuda_pack_state_batch_to_device_buffer_for_host_views(void *block_tag,
+                                                              double **state_host_key,
+                                                              int state_count,
+                                                              double *device_buffer,
+                                                              int *ex,
+                                                              int i0, int j0, int k0,
+                                                              int sx, int sy, int sz);
+
+int z4c_cuda_unpack_state_batch_from_device_buffer(void *block_tag,
+                                                   int state_count,
+                                                   double *device_buffer,
+                                                   int *ex,
+                                                   int i0, int j0, int k0,
+                                                   int sx, int sy, int sz);
+
+int z4c_cuda_unpack_state_batch_from_device_buffer_for_host_views(void *block_tag,
+                                                                  double **state_host_key,
+                                                                  int state_count,
+                                                                  double *device_buffer,
+                                                                  int *ex,
+                                                                  int i0, int j0, int k0,
+                                                                  int sx, int sy, int sz);
+
+int z4c_cuda_pack_state_segments_to_device_buffer(void *block_tag,
+                                                  int state_count,
+                                                  double *device_buffer,
+                                                  int *ex,
+                                                  int segment_count,
+                                                  const int *segment_meta);
+
+int z4c_cuda_pack_state_segments_to_device_buffer_for_host_views(void *block_tag,
+                                                                 double **state_host_key,
+                                                                 int state_count,
+                                                                 double *device_buffer,
+                                                                 int *ex,
+                                                                 int segment_count,
+                                                                 const int *segment_meta);
+
+int z4c_cuda_unpack_state_segments_from_device_buffer(void *block_tag,
+                                                      int state_count,
+                                                      double *device_buffer,
+                                                      int *ex,
+                                                      int segment_count,
+                                                      const int *segment_meta);
+
+int z4c_cuda_unpack_state_segments_from_device_buffer_for_host_views(void *block_tag,
+                                                                     double **state_host_key,
+                                                                     int state_count,
+                                                                     double *device_buffer,
+                                                                     int *ex,
+                                                                     int segment_count,
+                                                                     const int *segment_meta);
+
+int z4c_cuda_restrict_state_segments_to_device_buffer(void *block_tag,
+                                                      int state_count,
+                                                      double *device_buffer,
+                                                      int *ex,
+                                                      int segment_count,
+                                                      const int *segment_meta,
+                                                      const double *state_soa);
+
+int z4c_cuda_restrict_state_segments_to_device_buffer_for_host_views(void *block_tag,
+                                                                     double **state_host_key,
+                                                                     int state_count,
+                                                                     double *device_buffer,
+                                                                     int *ex,
+                                                                     int segment_count,
+                                                                     const int *segment_meta,
+                                                                     const double *state_soa);
+
+int z4c_cuda_prolong_state_segments_to_device_buffer(void *block_tag,
+                                                     int state_count,
+                                                     double *device_buffer,
+                                                     int *ex,
+                                                     int segment_count,
+                                                     const int *segment_meta,
+                                                     const double *state_soa);
+
+int z4c_cuda_prolong_state_segments_to_device_buffer_for_host_views(void *block_tag,
+                                                                    double **state_host_key,
+                                                                    int state_count,
+                                                                    double *device_buffer,
+                                                                    int *ex,
+                                                                    int segment_count,
+                                                                    const int *segment_meta,
+                                                                    const double *state_soa);
+
+int z4c_cuda_restrict_state_batch_to_device_buffer(void *block_tag,
+                                                   int state_count,
+                                                   double *device_buffer,
+                                                   int *ex,
+                                                   int sx, int sy, int sz,
+                                                   int fi0, int fj0, int fk0,
+                                                   const double *state_soa);
+
+int z4c_cuda_restrict_state_batch_to_device_buffer_for_host_views(void *block_tag,
+                                                                  double **state_host_key,
+                                                                  int state_count,
+                                                                  double *device_buffer,
+                                                                  int *ex,
+                                                                  int sx, int sy, int sz,
+                                                                  int fi0, int fj0, int fk0,
+                                                                  const double *state_soa);
+
+int z4c_cuda_prolong_state_batch_to_device_buffer(void *block_tag,
+                                                  int state_count,
+                                                  double *device_buffer,
+                                                  int *ex,
+                                                  int sx, int sy, int sz,
+                                                  int ii0, int jj0, int kk0,
+                                                  int lbc_i, int lbc_j, int lbc_k,
+                                                  const double *state_soa);
+
+int z4c_cuda_prolong_state_batch_to_device_buffer_for_host_views(void *block_tag,
+                                                                 double **state_host_key,
+                                                                 int state_count,
+                                                                 double *device_buffer,
+                                                                 int *ex,
+                                                                 int sx, int sy, int sz,
+                                                                 int ii0, int jj0, int kk0,
+                                                                 int lbc_i, int lbc_j, int lbc_k,
+                                                                 const double *state_soa);
+
+int z4c_cuda_download_state_subset(void *block_tag,
+                                   int *ex,
+                                   int subset_count,
+                                   const int *state_indices,
+                                   double **state_host_out);
+
+int z4c_cuda_upload_state_subset(void *block_tag,
+                                 int *ex,
+                                 int subset_count,
+                                 const int *state_indices,
+                                 double **state_host_in);
+
+int z4c_cuda_compute_constraints_resident(void *block_tag,
+                                          int *ex, double *X, double *Y, double *Z,
+                                          int Symmetry, double eps, int co,
+                                          double **constraint_host_out);
+
+int z4c_cuda_interp_state_point3(void *block_tag,
+                                 int *ex,
+                                 int state0,
+                                 int state1,
+                                 int state2,
+                                 double x0,
+                                 double y0,
+                                 double z0,
+                                 double dx,
+                                 double dy,
+                                 double dz,
+                                 double px,
+                                 double py,
+                                 double pz,
+                                 int ordn,
+                                 int symmetry,
+                                 const double *soa3,
+                                 double *out3);
+
+int z4c_cuda_download_constraint_outputs(int *ex,
+                                         double **constraint_host_out);
+
+int z4c_cuda_has_resident_state(void *block_tag);
+int z4c_cuda_resident_state_matches(void *block_tag,
+                                    double **state_host_key);
+
+void z4c_cuda_release_step_ctx(void *block_tag);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

BSSN_BUILD_CONFIG_MIGRATION.md

@@ -1,211 +0,0 @@
-# BSSN Build Config Migration
-
-This note records the build-configuration fix needed when replacing
-`AMSS_NCKU_Input.py` or `generate_macrodef.py` with a newer upstream version.
-
-## Problem
-
-`AMSS_NCKU_source/macrodef.h` is not the authoritative file used by normal
-runs. `AMSS_NCKU_Program.py` first generates macro files under
-`input_data.File_directory`, copies `AMSS_NCKU_source` to
-`<File_directory>/AMSS_NCKU_source_copy`, then copies the generated macro files
-into that copied source tree and compiles there.
-
-Therefore, makefile logic must not depend only on the stale
-`AMSS_NCKU_source/macrodef.h`. The actual equation path must be passed to the
-copied build tree from the same generation step that creates `macrodef.h`.
-
-The performance regression was caused by compiling/linking the
-`BSSN-EScalar` C wrapper into BSSN vacuum builds. For BSSN vacuum (`ABEtype=0`),
-the build must use:
-
-```make
-BSSN_USE_TRANSFER_CACHE=1
-BSSN_USE_ESCALAR_C_KERNEL=0
-```
-
-and must not link `bssn_escalar_rhs_c.o`.
-
-## Required Migration Steps
-
-### 1. Add an ABE type helper in `generate_macrodef.py`
-
-Add a helper that maps `input_data.Equation_Class` to the numeric `ABEtype`.
-Use the same mapping as `macrodef.h`:
-
-```python
-def get_abe_type():
-    if ( input_data.Equation_Class == "BSSN" ):
-        return 0
-    elif ( input_data.Equation_Class == "BSSN-EScalar" ):
-        return 1
-    elif ( input_data.Equation_Class == "BSSN-EM" ):
-        return 3
-    elif ( input_data.Equation_Class == "Z4C" ):
-        return 2
-    else:
-        raise ValueError("Equation_Class setting error!!!")
-```
-
-Update `generate_macrodef_h()` to print `#define ABEtype {get_abe_type()}`
-instead of duplicating the if/elif mapping.
-
-### 2. Generate a makefile fragment
-
-In `generate_macrodef.py`, add:
-
-```python
-def generate_build_config():
-    file1 = open(os.path.join(input_data.File_directory, "AMSS_NCKU_build.mk"), "w")
-    print("# Generated by generate_macrodef.py; do not edit manually.", file=file1)
-    print(f"ABE_TYPE := {get_abe_type()}", file=file1)
-    file1.close()
-```
-
-This file is the build-time authority for the equation path.
-
-### 3. Call and copy the generated build config
-
-In `AMSS_NCKU_Program.py`, after generating `macrodef.h` and `macrodef.fh`, call:
-
-```python
-generate_macrodef.generate_build_config()
-print(" AMSS-NCKU build config AMSS_NCKU_build.mk has been generated. ")
-```
-
-When copying generated files into `AMSS_NCKU_source_copy`, also copy:
-
-```python
-build_config_path = os.path.join(File_directory, "AMSS_NCKU_build.mk")
-shutil.copy2(build_config_path, AMSS_NCKU_source_copy)
-```
-
-### 4. Make the source makefile consume the generated config
-
-At the top of `AMSS_NCKU_source/makefile`, after `include makefile.inc`, add:
-
-```make
--include AMSS_NCKU_build.mk
-
-ABE_TYPE ?= $(shell awk '/^[[:space:]]*\#define[[:space:]]+ABEtype/ {print $$3; exit}' macrodef.h 2>/dev/null)
-```
-
-The generated `AMSS_NCKU_build.mk` is used during normal Python-driven builds.
-The fallback keeps manual source-tree builds usable.
-
-### 5. Gate path-specific build options by `ABE_TYPE`
-
-Use effective build switches:
-
-```make
-ifeq ($(USE_TRANSFER_CACHE),auto)
-ifeq ($(ABE_TYPE),0)
-EFFECTIVE_USE_TRANSFER_CACHE = 1
-else
-EFFECTIVE_USE_TRANSFER_CACHE = 0
-endif
-else
-EFFECTIVE_USE_TRANSFER_CACHE = $(USE_TRANSFER_CACHE)
-endif
-
-ifeq ($(USE_CXX_ESCALAR_KERNEL),1)
-ifeq ($(ABE_TYPE),1)
-EFFECTIVE_USE_CXX_ESCALAR_KERNEL = 1
-else
-EFFECTIVE_USE_CXX_ESCALAR_KERNEL = 0
-endif
-else
-EFFECTIVE_USE_CXX_ESCALAR_KERNEL = 0
-endif
-
-TRANSFER_CACHE_FLAG = -DBSSN_USE_TRANSFER_CACHE=$(EFFECTIVE_USE_TRANSFER_CACHE)
-ESCALAR_KERNEL_FLAG = -DBSSN_USE_ESCALAR_C_KERNEL=$(EFFECTIVE_USE_CXX_ESCALAR_KERNEL)
-```
-
-Only add `bssn_escalar_rhs_c.o` when the effective EScalar C kernel switch is
-enabled:
-
-```make
-ifeq ($(EFFECTIVE_USE_CXX_ESCALAR_KERNEL),1)
-CFILES += bssn_escalar_rhs_c.o
-endif
-```
-
-### 6. Use safe transfer-cache default
-
-In `AMSS_NCKU_source/makefile.inc`, keep:
-
-```make
-USE_TRANSFER_CACHE ?= auto
-```
-
-With the effective switch logic above, this enables cached transfer for BSSN
-vacuum while keeping non-BSSN paths on the uncached path by default.
-
-## Verification Checklist
-
-Run these checks after migrating:
-
-```bash
-python3 -c "import generate_macrodef; generate_macrodef.generate_build_config()"
-cat GW150914/AMSS_NCKU_build.mk
-```
-
-For BSSN, the generated file should contain:
-
-```make
-ABE_TYPE := 0
-```
-
-Dry-run the copied or source makefile:
-
-```bash
-make -n -B INTERP_LB_MODE=off ABE | grep -E 'BSSN_USE_TRANSFER_CACHE|BSSN_USE_ESCALAR_C_KERNEL|bssn_escalar_rhs_c'
-```
-
-Expected BSSN result:
-
-```text
--DBSSN_USE_TRANSFER_CACHE=1 -DBSSN_USE_ESCALAR_C_KERNEL=0
-```
-
-and no `bssn_escalar_rhs_c.o` in the final link command.
-
-Run the full workflow:
-
-```bash
-python3 AMSS_NCKU_Program.py
-```
-
-For the 10-step BSSN test, compare coordinate output:
-
-```bash
-python3 - <<'PY'
-from pathlib import Path
-old = Path('../GW150914-06457/AMSS_NCKU_output/bssn_BH.dat')
-new = Path('GW150914/AMSS_NCKU_output/bssn_BH.dat')
-
-def rows(path):
-    out = []
-    for line in path.read_text().splitlines():
-        if not line.strip() or line.lstrip().startswith('#'):
-            continue
-        out.append([float(x) for x in line.split()])
-    return out
-
-ro, rn = rows(old), rows(new)
-n = min(len(ro), len(rn))
-max_abs = 0.0
-for i in range(n):
-    for a, b in zip(ro[i], rn[i]):
-        max_abs = max(max_abs, abs(a - b))
-print(f"old_rows={len(ro)} new_rows={len(rn)} compared_rows={n}")
-print(f"max_abs_diff={max_abs:.17g}")
-PY
-```
-
-For the validated migration, the first 10 rows matched exactly:
-
-```text
-max_abs_diff=0
-```

generate_macrodef.py

@@ -12,37 +12,6 @@ import os
 import AMSS_NCKU_Input as input_data          ## import program input file
 
 
-##################################################################
-
-def get_abe_type():
-    if ( input_data.Equation_Class == "BSSN" ):
-        return 0
-    elif ( input_data.Equation_Class == "BSSN-EScalar" ):
-        return 1
-    elif ( input_data.Equation_Class == "BSSN-EM" ):
-        return 3
-    elif ( input_data.Equation_Class == "Z4C" ):
-        return 2
-    else:
-        raise ValueError("Equation_Class setting error!!!")
-
-
-##################################################################
-
-## Generate the makefile fragment used by the copied source tree.
-## The source-tree macrodef.h is not authoritative because macro files
-## are regenerated under File_directory for each run.
-
-def generate_build_config():
-
-    file1 = open( os.path.join(input_data.File_directory, "AMSS_NCKU_build.mk"), "w")
-
-    print( "# Generated by generate_macrodef.py; do not edit manually.", file=file1 )
-    print( f"ABE_TYPE := {get_abe_type()}",                             file=file1 )
-
-    file1.close()
-
-
 ##################################################################
 
 ## Generate the macro file macrodef.h according to user settings
@@ -89,10 +58,19 @@ def generate_macrodef_h():
     # 2: Z4c vacuum
     # 3: coupled to Maxwell field
 
-    try:
-        print( f"#define ABEtype {get_abe_type()}", file=file1 )
-        print(                                      file=file1 )
-    except ValueError:
+    if ( input_data.Equation_Class == "BSSN" ):
+        print( "#define ABEtype 0", file=file1 )
+        print(                      file=file1 )
+    elif ( input_data.Equation_Class == "BSSN-EScalar" ):
+        print( "#define ABEtype 1", file=file1 )
+        print(                      file=file1 )
+    elif ( input_data.Equation_Class == "BSSN-EM" ):
+        print( "#define ABEtype 3", file=file1 )
+        print(                      file=file1 )
+    elif ( input_data.Equation_Class == "Z4C" ):
+        print( "#define ABEtype 2", file=file1 )
+        print(                      file=file1 )
+    else:
         print( "Equation_Class setting error!!!"                )
         print()
         print( "# Equation type #define ABEtype setting error!!!", file=file1 )
@@ -226,7 +204,7 @@ def generate_macrodef_h():
     # use GPU or not
     
     if ( input_data.GPU_Calculation == "yes"):
-        print( "#define USE_GPU",   file=file1 )
+        print( "//#define USE_GPU",   file=file1 )
         print(                      file=file1 )
     elif ( input_data.GPU_Calculation == "no"):
         print( "//#define USE_GPU", file=file1 )

makefile_and_run.py

@@ -9,6 +9,8 @@
 
 
 import AMSS_NCKU_Input as input_data
+import os
+import shutil
 import subprocess
 import time
 
@@ -56,6 +58,124 @@ BUILD_JOBS = 64
 
 ##################################################################
 
+def _truthy(value, default=False):
+    if value is None:
+        return default
+    if isinstance(value, bool):
+        return value
+    text = str(value).strip().lower()
+    if text == "":
+        return default
+    return text in ("1", "yes", "y", "true", "on", "enable", "enabled")
+
+
+def _input_or_env(input_name, env_name, default=None):
+    if env_name in os.environ:
+        return os.environ[env_name]
+    return getattr(input_data, input_name, default)
+
+
+def _start_cuda_mps_if_requested(runtime_env):
+    if input_data.GPU_Calculation != "yes":
+        return False
+
+    default_auto_mps = int(getattr(input_data, "MPI_processes", 1)) > 1
+    auto_mps = _truthy(
+        _input_or_env("CUDA_Auto_MPS", "AMSS_CUDA_AUTO_MPS", default_auto_mps),
+        default=default_auto_mps,
+    )
+    if not auto_mps:
+        return False
+
+    mps_control = shutil.which("nvidia-cuda-mps-control")
+    if not mps_control:
+        print(" CUDA MPS control command was not found; running without MPS.")
+        return False
+
+    uid = os.getuid()
+    pipe_dir = str(_input_or_env("CUDA_MPS_PIPE_DIRECTORY", "CUDA_MPS_PIPE_DIRECTORY",
+                                 f"/tmp/amss-ncku-mps-{uid}"))
+    log_dir = str(_input_or_env("CUDA_MPS_LOG_DIRECTORY", "CUDA_MPS_LOG_DIRECTORY",
+                                f"/tmp/amss-ncku-mps-log-{uid}"))
+    os.makedirs(pipe_dir, exist_ok=True)
+    os.makedirs(log_dir, exist_ok=True)
+
+    mps_env = runtime_env.copy()
+    mps_env["CUDA_MPS_PIPE_DIRECTORY"] = pipe_dir
+    mps_env["CUDA_MPS_LOG_DIRECTORY"] = log_dir
+
+    if os.path.exists(os.path.join(pipe_dir, "control")):
+        runtime_env.update({
+            "CUDA_MPS_PIPE_DIRECTORY": pipe_dir,
+            "CUDA_MPS_LOG_DIRECTORY": log_dir,
+        })
+        print(f" Reusing CUDA MPS daemon: {pipe_dir}")
+        return False
+
+    print(f" Starting CUDA MPS daemon for this run: {pipe_dir}")
+    result = subprocess.run([mps_control, "-d"], env=mps_env, text=True,
+                            stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+    if result.returncode != 0:
+        print(" CUDA MPS daemon did not start; running without MPS.")
+        if result.stdout:
+            print(result.stdout, end="")
+        return False
+
+    runtime_env.update({
+        "CUDA_MPS_PIPE_DIRECTORY": pipe_dir,
+        "CUDA_MPS_LOG_DIRECTORY": log_dir,
+    })
+    return True
+
+
+def _stop_cuda_mps(runtime_env):
+    mps_control = shutil.which("nvidia-cuda-mps-control")
+    if not mps_control:
+        return
+    subprocess.run([mps_control], input="quit\n", env=runtime_env, text=True,
+                   stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+
+
+def _gpu_runtime_env():
+    runtime_env = os.environ.copy()
+
+    defaults = {
+        "AMSS_INTERP_FAST": "1",
+        "AMSS_INTERP_GPU": "1",
+        "AMSS_ANALYSIS_MAP_EVERY": "1000000",
+        "AMSS_CUDA_AWARE_MPI": "1",
+        "AMSS_CUDA_KEEP_RESIDENT_AFTER_STEP": "1",
+        "AMSS_CUDA_Z4C_KEEP_RESIDENT_AFTER_STEP": "1",
+        "AMSS_CUDA_KEEP_ALL_LEVELS": "1",
+        "AMSS_CUDA_Z4C_AMR_DEVICE": "0",
+        "AMSS_CUDA_AMR_RESTRICT_DEVICE": "1",
+        "AMSS_CUDA_AMR_RESTRICT_BATCH": "0",
+        "AMSS_CUDA_DEVICE_SEGMENT_BATCH": "0",
+        "AMSS_CUDA_PIN_ESCALAR_TRANSFERS": "0",
+        "AMSS_ESCALAR_GPU_RK": "0",
+    }
+    if getattr(input_data, "Equation_Class", "") == "Z4C":
+        defaults["AMSS_CUDA_Z4C_KEEP_RESIDENT_AFTER_STEP"] = "0"
+        defaults["AMSS_CUDA_KEEP_ALL_LEVELS"] = "0"
+    for key, value in defaults.items():
+        runtime_env.setdefault(key, value)
+
+    optional_overrides = {
+        "AMSS_INTERP_FAST_COMPARE": "AMSS_Interp_Fast_Compare",
+        "AMSS_INTERP_FAST_COMPARE_LIMIT": "AMSS_Interp_Fast_Compare_Limit",
+        "AMSS_INTERP_FAST_COMPARE_TOL": "AMSS_Interp_Fast_Compare_Tol",
+        "AMSS_GPU_STAGE_TIMING": "AMSS_GPU_Stage_Timing",
+        "AMSS_GPU_STAGE_TIMING_EVERY": "AMSS_GPU_Stage_Timing_Every",
+    }
+    for env_name, input_name in optional_overrides.items():
+        if env_name not in runtime_env and hasattr(input_data, input_name):
+            runtime_env[env_name] = str(getattr(input_data, input_name))
+
+    return runtime_env
+
+
+##################################################################
+
 
 
 ##################################################################
@@ -70,9 +190,9 @@ def makefile_ABE():
 
     ## Build command with CPU binding to nohz_full cores
     if (input_data.GPU_Calculation == "no"):
-        makefile_command  = f"{NUMACTL_CPU_BIND} make -j{BUILD_JOBS} INTERP_LB_MODE=off ABE"
+        makefile_command  = f"{NUMACTL_CPU_BIND} make -j{BUILD_JOBS} INTERP_LB_MODE=off USE_CUDA_BSSN=0 USE_CUDA_Z4C=0 ABE"
     elif (input_data.GPU_Calculation == "yes"):
-        makefile_command  = f"{NUMACTL_CPU_BIND} make -j{BUILD_JOBS} ABEGPU"
+        makefile_command  = f"{NUMACTL_CPU_BIND} make -j{BUILD_JOBS} INTERP_LB_MODE=off USE_CUDA_BSSN=1 USE_CUDA_Z4C=1 ABE_CUDA"
     else:
         print( " CPU/GPU numerical calculation setting is wrong " )
         print(                                                    )
@@ -145,29 +265,55 @@ def run_ABE():
     print(                                                      )
 
     ## Define the command to run; cast other values to strings as needed
+    mpi_env = None
+    started_mps = False
     
     if (input_data.GPU_Calculation == "no"):
         mpi_command         = NUMACTL_CPU_BIND + " mpirun -np " + str(input_data.MPI_processes) + " ./ABE"
         #mpi_command         = " mpirun -np " + str(input_data.MPI_processes) + " ./ABE"
         mpi_command_outfile = "ABE_out.log"
     elif (input_data.GPU_Calculation == "yes"):
-        mpi_command         = NUMACTL_CPU_BIND + " mpirun -np " + str(input_data.MPI_processes) + " ./ABEGPU"
+        mpi_command         = NUMACTL_CPU_BIND + " mpirun -np " + str(input_data.MPI_processes) + " ./ABE_CUDA"
         mpi_command_outfile = "ABEGPU_out.log"
+        mpi_env = _gpu_runtime_env()
+        started_mps = _start_cuda_mps_if_requested(mpi_env)
+        print(" GPU optimized runtime switches:")
+        print(f"   AMSS_INTERP_FAST={mpi_env.get('AMSS_INTERP_FAST', '')}")
+        print(f"   AMSS_INTERP_GPU={mpi_env.get('AMSS_INTERP_GPU', '')}")
+        print(f"   AMSS_ANALYSIS_MAP_EVERY={mpi_env.get('AMSS_ANALYSIS_MAP_EVERY', '')}")
+        print(f"   AMSS_CUDA_AWARE_MPI={mpi_env.get('AMSS_CUDA_AWARE_MPI', '')}")
+        print(f"   AMSS_CUDA_KEEP_RESIDENT_AFTER_STEP={mpi_env.get('AMSS_CUDA_KEEP_RESIDENT_AFTER_STEP', '')}")
+        print(f"   AMSS_CUDA_Z4C_KEEP_RESIDENT_AFTER_STEP={mpi_env.get('AMSS_CUDA_Z4C_KEEP_RESIDENT_AFTER_STEP', '')}")
+        print(f"   AMSS_CUDA_KEEP_ALL_LEVELS={mpi_env.get('AMSS_CUDA_KEEP_ALL_LEVELS', '')}")
+        print(f"   AMSS_CUDA_Z4C_AMR_DEVICE={mpi_env.get('AMSS_CUDA_Z4C_AMR_DEVICE', '')}")
+        print(f"   AMSS_CUDA_AMR_RESTRICT_DEVICE={mpi_env.get('AMSS_CUDA_AMR_RESTRICT_DEVICE', '')}")
+        print(f"   AMSS_CUDA_AMR_RESTRICT_BATCH={mpi_env.get('AMSS_CUDA_AMR_RESTRICT_BATCH', '')}")
+        print(f"   AMSS_CUDA_DEVICE_SEGMENT_BATCH={mpi_env.get('AMSS_CUDA_DEVICE_SEGMENT_BATCH', '')}")
+        print(f"   AMSS_CUDA_PIN_ESCALAR_TRANSFERS={mpi_env.get('AMSS_CUDA_PIN_ESCALAR_TRANSFERS', '')}")
+        print(f"   AMSS_ESCALAR_GPU_RK={mpi_env.get('AMSS_ESCALAR_GPU_RK', '')}")
+        if "CUDA_MPS_PIPE_DIRECTORY" in mpi_env:
+            print(f"   CUDA_MPS_PIPE_DIRECTORY={mpi_env['CUDA_MPS_PIPE_DIRECTORY']}")
  
-    ## Execute the MPI command and stream output
-    mpi_process = subprocess.Popen(mpi_command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)
+    try:
+        ## Execute the MPI command and stream output
+        mpi_process = subprocess.Popen(mpi_command, shell=True, stdout=subprocess.PIPE,
+                                       stderr=subprocess.STDOUT, text=True, env=mpi_env)
 
-    ## Write ABE run output to file while printing to stdout
-    with open(mpi_command_outfile, 'w') as file0:  
-        ## Read and print output lines; also write each line to file
-        for line in mpi_process.stdout:
-            print(line, end='')  # stream output in real time
-            file0.write(line)    # write the line to file
-            file0.flush()        # flush to ensure each line is written immediately (optional)            
-    file0.close()
+        ## Write ABE run output to file while printing to stdout
+        with open(mpi_command_outfile, 'w') as file0:
+            ## Read and print output lines; also write each line to file
+            for line in mpi_process.stdout:
+                print(line, end='')  # stream output in real time
+                file0.write(line)    # write the line to file
+                file0.flush()        # flush to ensure each line is written immediately (optional)
 
-    ## Wait for the process to finish
-    mpi_return_code = mpi_process.wait()
+        ## Wait for the process to finish
+        mpi_return_code = mpi_process.wait()
+        if mpi_return_code != 0:
+            raise subprocess.CalledProcessError(mpi_return_code, mpi_command)
+    finally:
+        if started_mps:
+            _stop_cuda_mps(mpi_env)
     
     print(                                           )
     print( " The ABE/ABEGPU simulation is finished " )