Stabilize EScalar CUDA fallback path

AMSS_NCKU_Input.py

@@ -16,7 +16,7 @@ import numpy
 File_directory   = "GW150914"                    ## output file directory
 Output_directory = "binary_output"               ## binary data file directory
                                                  ## The file directory name should not be too long
-MPI_processes    = 8                             ## number of mpi processes used in the simulation
+MPI_processes    = 2                             ## number of mpi processes used in the simulation
 
 GPU_Calculation  = "yes"                          ## Use GPU or not 
                                                  ## (prefer "no" in the current version, because the GPU part may have bugs when integrated in this Python interface)
@@ -31,7 +31,7 @@ GPU_Part         = 0.0
 ## Setting the physical system and numerical method
 
 Symmetry                 = "equatorial-symmetry"   ## Symmetry of System: choose equatorial-symmetry、no-symmetry、octant-symmetry
-Equation_Class           = "BSSN"                  ## Evolution Equation: choose "BSSN", "BSSN-EScalar", "BSSN-EM", "Z4C" 
+Equation_Class           = "BSSN-EScalar"                  ## Evolution Equation: choose "BSSN", "BSSN-EScalar", "BSSN-EM", "Z4C"
                                                    ## If "BSSN-EScalar" is chosen, it is necessary to set other parameters below
 Initial_Data_Method      = "Ansorg-TwoPuncture"    ## initial data method: choose "Ansorg-TwoPuncture", "Lousto-Analytical", "Cao-Analytical", "KerrSchild-Analytical"
 Time_Evolution_Method    = "runge-kutta-45"        ## time evolution method: choose "runge-kutta-45"

AMSS_NCKU_Program.py

@@ -58,31 +58,36 @@ File_directory = os.path.join(input_data.File_directory)
 
 ## If the specified output directory exists, ask the user whether to continue
 if os.path.exists(File_directory):
-    print( " Output dictionary has been existed !!!  "                                                              )
+    auto_overwrite = str(getattr(input_data, "Auto_Overwrite_Output", "yes")).strip().lower()
-    print( " If you want to overwrite the existing file directory, please input 'continue' in the terminal !! "     ) 
+    if auto_overwrite in ("1", "yes", "y", "true", "on", "continue"):
-    print( " If you want to retain the existing file directory, please input 'stop' in the terminal to stop the "   ) 
+        print( " Output dictionary has been existed; Auto_Overwrite_Output=yes, continue the calculation. " )
-    print( " simulation. Then you can reset the output dictionary in the input script file AMSS_NCKU_Input.py !!! " )
+        print(                                                                                              )
-    print(                                                                                                          )
+    else:
-    ## Prompt whether to overwrite the existing directory
+        print( " Output dictionary has been existed !!!  "                                                              )
-    while True:
+        print( " If you want to overwrite the existing file directory, please input 'continue' in the terminal !! "     ) 
-        try:
+        print( " If you want to retain the existing file directory, please input 'stop' in the terminal to stop the "   ) 
-            inputvalue = input()
+        print( " simulation. Then you can reset the output dictionary in the input script file AMSS_NCKU_Input.py !!! " )
-            ## If the user agrees to overwrite, proceed and remove the existing directory
+        print(                                                                                                          )
-            if ( inputvalue == "continue" ):
+        ## Prompt whether to overwrite the existing directory
-                print( " Continue the calculation !!! " )
+        while True:
-                print(                                  )
+            try:
-                break  
+                inputvalue = input()
-            ## If the user chooses not to overwrite, exit and keep the existing directory
+                ## If the user agrees to overwrite, proceed and remove the existing directory
-            elif ( inputvalue == "stop" ):
+                if ( inputvalue == "continue" ):
-                print( " Stop the calculation !!! "    )
+                    print( " Continue the calculation !!! " )
-                sys.exit() 
+                    print(                                  )
-            ## If the user input is invalid, prompt again
+                    break  
-            else:
+                ## If the user chooses not to overwrite, exit and keep the existing directory
+                elif ( inputvalue == "stop" ):
+                    print( " Stop the calculation !!! "    )
+                    sys.exit() 
+                ## If the user input is invalid, prompt again
+                else:
+                    print( " Please input your choice !!! "                   )
+                    print( " Input 'continue' or 'stop' in the terminal !!! " )
+            except ValueError:
                 print( " Please input your choice !!! "                   )
                 print( " Input 'continue' or 'stop' in the terminal !!! " )
-        except ValueError:
-            print( " Please input your choice !!! "                   )
-            print( " Input 'continue' or 'stop' in the terminal !!! " )
         
 ## Remove the existing output directory if present
 shutil.rmtree(File_directory, ignore_errors=True)

AMSS_NCKU_source/Block.C

@@ -12,7 +12,61 @@ using namespace std;
 #include "Block.h"
 #include "misc.h"
 
-Block::Block(int DIM, int *shapei, double *bboxi, int ranki, int ingfsi, int fngfsi, int levi, const int cgpui) : rank(ranki), ingfs(ingfsi), fngfs(fngfsi), lev(levi), cgpu(cgpui)
+#if USE_CUDA_BSSN || USE_CUDA_Z4C
+#include <cuda_runtime_api.h>
+#endif
+
+namespace {
+
+bool cuda_pin_gridfuncs_enabled()
+{
+  static int enabled = -1;
+  if (enabled < 0)
+  {
+    const char *env = getenv("AMSS_CUDA_PIN_GRIDFUNCS");
+    enabled = (env && atoi(env) != 0) ? 1 : 0;
+  }
+  return enabled != 0;
+}
+
+double *alloc_gridfunc(size_t count, unsigned char &pinned)
+{
+  pinned = 0;
+#if USE_CUDA_BSSN || USE_CUDA_Z4C
+  if (cuda_pin_gridfuncs_enabled())
+  {
+    double *ptr = 0;
+    cudaError_t err = cudaMallocHost((void **)&ptr, count * sizeof(double));
+    if (err == cudaSuccess)
+    {
+      pinned = 1;
+      return ptr;
+    }
+    cudaGetLastError();
+  }
+#endif
+  return (double *)malloc(sizeof(double) * count);
+}
+
+void free_gridfunc(double *ptr, unsigned char pinned)
+{
+  if (!ptr)
+    return;
+#if USE_CUDA_BSSN || USE_CUDA_Z4C
+  if (pinned)
+  {
+    cudaFreeHost(ptr);
+    return;
+  }
+#else
+  (void)pinned;
+#endif
+  free(ptr);
+}
+
+}
+
+Block::Block(int DIM, int *shapei, double *bboxi, int ranki, int ingfsi, int fngfsi, int levi, const int cgpui) : rank(ranki), lev(levi), cgpu(cgpui), ingfs(ingfsi), fngfs(fngfsi), igfs(0), fgfs(0), fgfs_pinned(0)
 {
   for (int i = 0; i < dim; i++)
     X[i] = 0;
@@ -70,9 +124,10 @@ Block::Block(int DIM, int *shapei, double *bboxi, int ranki, int ingfsi, int fng
 
     int nn = shape[0] * shape[1] * shape[2];
     fgfs = new double *[fngfs];
+    fgfs_pinned = new unsigned char[fngfs];
     for (int i = 0; i < fngfs; i++)
     {
-      fgfs[i] = (double *)malloc(sizeof(double) * nn);
+      fgfs[i] = alloc_gridfunc((size_t)nn, fgfs_pinned[i]);
       if (!(fgfs[i]))
       {
         cout << "on node#" << rank << ", out of memory when constructing Block." << endl;
@@ -107,11 +162,13 @@ Block::~Block()
       free(igfs[i]);
     delete[] igfs;
     for (int i = 0; i < fngfs; i++)
-      free(fgfs[i]);
+      free_gridfunc(fgfs[i], fgfs_pinned ? fgfs_pinned[i] : 0);
     delete[] fgfs;
+    delete[] fgfs_pinned;
     X[0] = X[1] = X[2] = 0;
     igfs = 0;
     fgfs = 0;
+    fgfs_pinned = 0;
   }
 }
 void Block::checkBlock()
@@ -187,6 +244,8 @@ void Block::swapList(MyList<var> *VarList1, MyList<var> *VarList2, int myrank)
     while (varl1 && varl2)
     {
       misc::swap<double *>(fgfs[varl1->data->sgfn], fgfs[varl2->data->sgfn]);
+      if (fgfs_pinned)
+        misc::swap<unsigned char>(fgfs_pinned[varl1->data->sgfn], fgfs_pinned[varl2->data->sgfn]);
       varl1 = varl1->next;
       varl2 = varl2->next;
     }

AMSS_NCKU_source/Block.h

@@ -18,9 +18,10 @@ public:
    int ingfs, fngfs;
    int *(*igfs);
    double *(*fgfs);
+   unsigned char *fgfs_pinned;
 
 public:
-   Block() {};
+   Block() : rank(0), lev(0), cgpu(0), ingfs(0), fngfs(0), igfs(0), fgfs(0), fgfs_pinned(0) {};
    Block(int DIM, int *shapei, double *bboxi, int ranki, int ingfsi, int fngfs, int levi, const int cgpui = 0);
 
    ~Block();

AMSS_NCKU_source/MPatch.C

@@ -14,6 +14,9 @@ using namespace std;
 #include "MPatch.h"
 #include "Parallel.h"
 #include "fmisc.h"
+#if USE_CUDA_BSSN
+#include "bssn_rhs_cuda.h"
+#endif
 #ifdef INTERP_LB_PROFILE
 #include "interp_lb_profile.h"
 #endif
@@ -178,6 +181,444 @@ int find_block_index_for_point(const BlockBinIndex &index, const double *pox, co
 
   return -1;
 }
+
+inline int fortran_idint_local(double x)
+{
+  return int(x);
+}
+
+bool interp_fast_enabled()
+{
+  static int enabled = -1;
+  if (enabled < 0)
+  {
+    const char *env = getenv("AMSS_INTERP_FAST");
+    enabled = (!env || atoi(env) != 0) ? 1 : 0;
+  }
+  return enabled != 0;
+}
+
+bool interp_gpu_enabled()
+{
+  static int enabled = -1;
+  if (enabled < 0)
+  {
+    const char *env = getenv("AMSS_INTERP_GPU");
+    enabled = (env && atoi(env) != 0) ? 1 : 0;
+  }
+  return enabled != 0;
+}
+
+bool interp_fast_compare_enabled()
+{
+  static int enabled = -1;
+  if (enabled < 0)
+  {
+    const char *env = getenv("AMSS_INTERP_FAST_COMPARE");
+    enabled = (env && atoi(env) != 0) ? 1 : 0;
+  }
+  return enabled != 0;
+}
+
+double interp_fast_compare_tol()
+{
+  static double tol = -1.0;
+  if (tol < 0.0)
+  {
+    const char *env = getenv("AMSS_INTERP_FAST_COMPARE_TOL");
+    tol = (env && atof(env) > 0.0) ? atof(env) : 1.0e-11;
+  }
+  return tol;
+}
+
+long long interp_fast_compare_limit()
+{
+  static long long limit = -1;
+  if (limit < 0)
+  {
+    const char *env = getenv("AMSS_INTERP_FAST_COMPARE_LIMIT");
+    limit = (env && atoll(env) > 0) ? atoll(env) : 4096;
+  }
+  return limit;
+}
+
+struct FastInterpStencil
+{
+  int cxB[dim];
+  double cx[dim];
+  double wx[8];
+  double wy[8];
+  double wz[8];
+  int nsamples;
+  int loc[512];
+  unsigned char sign_mask[512];
+  double weight[512];
+};
+
+inline void lagrange_unit_weights(double x, int ordn, double *w)
+{
+  for (int i = 0; i < ordn; i++)
+  {
+    double num = 1.0;
+    double den = 1.0;
+    for (int j = 0; j < ordn; j++)
+    {
+      if (j == i)
+        continue;
+      num *= (x - double(j));
+      den *= double(i - j);
+    }
+    w[i] = num / den;
+  }
+}
+
+inline void z_unit_weights(double x, int ordn, double *w)
+{
+  if (ordn == 6)
+  {
+    static const double c_uniform[6] = {-1.0, 5.0, -10.0, 10.0, -5.0, 1.0};
+    for (int i = 0; i < 6; i++)
+    {
+      if (x == double(i))
+      {
+        for (int j = 0; j < 6; j++)
+          w[j] = (j == i) ? 1.0 : 0.0;
+        return;
+      }
+    }
+    double den = 0.0;
+    for (int i = 0; i < 6; i++)
+    {
+      w[i] = c_uniform[i] / (x - double(i));
+      den += w[i];
+    }
+    for (int i = 0; i < 6; i++)
+      w[i] /= den;
+    return;
+  }
+  lagrange_unit_weights(x, ordn, w);
+}
+
+inline bool fast_interp_map_index(int idx, int extent, int d,
+                                  int &mapped, unsigned char &mask)
+{
+  if (idx > 0)
+    mapped = idx;
+  else
+  {
+    mask |= (unsigned char)(1u << d);
+#ifdef Vertex
+#ifdef Cell
+#error Both Cell and Vertex are defined
+#endif
+    mapped = 2 - idx;
+#else
+#ifdef Cell
+    mapped = 1 - idx;
+#else
+#error Not define Vertex nor Cell
+#endif
+#endif
+  }
+  return mapped >= 1 && mapped <= extent;
+}
+
+bool prepare_fast_interp_stencil(Block *BP, const double *pox, int ordn,
+                                 int Symmetry, FastInterpStencil &st)
+{
+  if (!BP || ordn <= 0 || ordn > 8)
+    return false;
+
+  st.nsamples = 0;
+
+  const int NO_SYMM = 0;
+  const int OCTANT = 2;
+  int cmin[dim], cmax[dim], cxT[dim];
+  for (int d = 0; d < dim; d++)
+  {
+    const double *X = BP->X[d];
+    const double dX = X[1] - X[0];
+    const int cxI = fortran_idint_local((pox[d] - X[0]) / dX + 0.4) + 1;
+    st.cxB[d] = cxI - ordn / 2 + 1;
+    cxT[d] = st.cxB[d] + ordn - 1;
+    cmin[d] = 1;
+    cmax[d] = BP->shape[d];
+
+#ifdef Vertex
+#ifdef Cell
+#error Both Cell and Vertex are defined
+#endif
+    if (Symmetry == OCTANT && d < 2 && fabs(X[0]) < dX)
+      cmin[d] = -ordn / 2 + 2;
+    if (Symmetry != NO_SYMM && d == 2 && fabs(X[0]) < dX)
+      cmin[d] = -ordn / 2 + 2;
+#else
+#ifdef Cell
+    if (Symmetry == OCTANT && d < 2 && fabs(X[0]) < dX)
+      cmin[d] = -ordn / 2 + 1;
+    if (Symmetry != NO_SYMM && d == 2 && fabs(X[0]) < dX)
+      cmin[d] = -ordn / 2 + 1;
+#else
+#error Not define Vertex nor Cell
+#endif
+#endif
+
+    if (st.cxB[d] < cmin[d])
+    {
+      st.cxB[d] = cmin[d];
+      cxT[d] = st.cxB[d] + ordn - 1;
+    }
+    if (cxT[d] > cmax[d])
+    {
+      cxT[d] = cmax[d];
+      st.cxB[d] = cxT[d] + 1 - ordn;
+    }
+
+    if (st.cxB[d] > 0)
+      st.cx[d] = (pox[d] - X[st.cxB[d] - 1]) / dX;
+    else
+    {
+#ifdef Vertex
+#ifdef Cell
+#error Both Cell and Vertex are defined
+#endif
+      st.cx[d] = (pox[d] + X[1 - st.cxB[d]]) / dX;
+#else
+#ifdef Cell
+      st.cx[d] = (pox[d] + X[-st.cxB[d]]) / dX;
+#else
+#error Not define Vertex nor Cell
+#endif
+#endif
+    }
+  }
+
+  lagrange_unit_weights(st.cx[0], ordn, st.wx);
+  lagrange_unit_weights(st.cx[1], ordn, st.wy);
+  z_unit_weights(st.cx[2], ordn, st.wz);
+
+  for (int kk = 0; kk < ordn; kk++)
+  {
+    for (int jj = 0; jj < ordn; jj++)
+    {
+      for (int ii = 0; ii < ordn; ii++)
+      {
+        unsigned char mask = 0;
+        int ix, iy, iz;
+        if (!fast_interp_map_index(st.cxB[0] + ii, BP->shape[0], 0, ix, mask) ||
+            !fast_interp_map_index(st.cxB[1] + jj, BP->shape[1], 1, iy, mask) ||
+            !fast_interp_map_index(st.cxB[2] + kk, BP->shape[2], 2, iz, mask))
+          return false;
+        const int s = st.nsamples++;
+        st.loc[s] = (ix - 1) + (iy - 1) * BP->shape[0] +
+                    (iz - 1) * BP->shape[0] * BP->shape[1];
+        st.sign_mask[s] = mask;
+        st.weight[s] = st.wx[ii] * st.wy[jj] * st.wz[kk];
+      }
+    }
+  }
+  return true;
+}
+
+bool interpolate_var_list_with_stencil(Block *BP, MyList<var> *VarList,
+                                       int num_var, const double *pox,
+                                       int ordn, int Symmetry,
+                                       const FastInterpStencil &st,
+                                       double *out)
+{
+  if (num_var <= 0 || num_var > 128)
+    return false;
+
+  double *data_ptrs[128];
+  double *soa_ptrs[128];
+  var *vars[128];
+  MyList<var> *varl = VarList;
+  int k = 0;
+  while (varl)
+  {
+    if (k >= num_var)
+      return false;
+    vars[k] = varl->data;
+    data_ptrs[k] = BP->fgfs[vars[k]->sgfn];
+    soa_ptrs[k] = vars[k]->SoA;
+    out[k] = 0.0;
+    varl = varl->next;
+    k++;
+  }
+
+  if (k != num_var)
+    return false;
+
+  for (int s = 0; s < st.nsamples; s++)
+  {
+    const int loc = st.loc[s];
+    const double w = st.weight[s];
+    const unsigned char mask = st.sign_mask[s];
+    if (mask == 0)
+    {
+      for (int v = 0; v < num_var; v++)
+        out[v] += w * data_ptrs[v][loc];
+    }
+    else
+    {
+      for (int v = 0; v < num_var; v++)
+      {
+        const double *SoA = soa_ptrs[v];
+        double sgn = 1.0;
+        if (mask & 1u)
+          sgn *= SoA[0];
+        if (mask & 2u)
+          sgn *= SoA[1];
+        if (mask & 4u)
+          sgn *= SoA[2];
+        out[v] += w * sgn * data_ptrs[v][loc];
+      }
+    }
+  }
+
+  if (interp_fast_compare_enabled())
+  {
+    static int report_count = 0;
+    static long long compare_calls = 0;
+    if (compare_calls++ >= interp_fast_compare_limit())
+      return true;
+    const double tol = interp_fast_compare_tol();
+    varl = VarList;
+    k = 0;
+    while (varl)
+    {
+      var *vp = vars[k];
+      double ref = 0.0;
+      double x = pox[0], y = pox[1], z = pox[2];
+      f_global_interp(BP->shape, BP->X[0], BP->X[1], BP->X[2],
+                      BP->fgfs[vp->sgfn], ref,
+                      x, y, z, ordn, vp->SoA, Symmetry);
+      const double diff = fabs(ref - out[k]);
+      const double scale = 1.0 + fabs(ref);
+      if (diff > tol * scale && report_count < 32)
+      {
+        int rank = 0;
+        MPI_Comm_rank(MPI_COMM_WORLD, &rank);
+        fprintf(stderr,
+                "[AMSS-INTERP-CMP][rank %d] var=%s diff=%.17e ref=%.17e fast=%.17e p=(%.17e,%.17e,%.17e)\n",
+                rank, vp->name, diff, ref, out[k], pox[0], pox[1], pox[2]);
+        report_count++;
+      }
+      varl = varl->next;
+      k++;
+    }
+  }
+
+  return true;
+}
+
+bool interpolate_var_list_fast(Block *BP, MyList<var> *VarList, int num_var,
+                               const double *pox, int ordn, int Symmetry,
+                               double *out)
+{
+  if (!interp_fast_enabled())
+    return false;
+
+  FastInterpStencil st;
+  if (!prepare_fast_interp_stencil(BP, pox, ordn, Symmetry, st))
+    return false;
+
+  return interpolate_var_list_with_stencil(BP, VarList, num_var, pox,
+                                           ordn, Symmetry, st, out);
+}
+
+struct CachedInterpPoint
+{
+  Block *bp;
+  int owner_rank;
+  FastInterpStencil stencil;
+};
+
+struct SurfaceInterpCache
+{
+  Patch *patch;
+  int NN;
+  int symmetry;
+  double key[9];
+  vector<CachedInterpPoint> points;
+
+  SurfaceInterpCache() : patch(0), NN(0), symmetry(-1) {}
+};
+
+bool surface_cache_key_matches(const SurfaceInterpCache &cache, Patch *patch,
+                               int NN, double **XX, int Symmetry)
+{
+  if (cache.patch != patch || cache.NN != NN || cache.symmetry != Symmetry ||
+      int(cache.points.size()) != NN || NN <= 0)
+    return false;
+  const int mid = NN / 2;
+  const int last = NN - 1;
+  const int ids[3] = {0, mid, last};
+  int p = 0;
+  for (int q = 0; q < 3; q++)
+    for (int d = 0; d < dim; d++)
+      if (cache.key[p++] != XX[d][ids[q]])
+        return false;
+  return true;
+}
+
+SurfaceInterpCache *find_surface_cache(Patch *patch, int NN, double **XX,
+                                       int Symmetry)
+{
+  static vector<SurfaceInterpCache> caches;
+  for (size_t i = 0; i < caches.size(); i++)
+    if (surface_cache_key_matches(caches[i], patch, NN, XX, Symmetry))
+      return &caches[i];
+  if (caches.size() >= 24)
+    caches.erase(caches.begin());
+  caches.push_back(SurfaceInterpCache());
+  return &caches.back();
+}
+
+bool build_surface_cache(SurfaceInterpCache &cache, Patch *patch, int NN,
+                         double **XX, int Symmetry, const double *DH,
+                         const BlockBinIndex &block_index, int ordn)
+{
+  int myrank = 0;
+  MPI_Comm_rank(MPI_COMM_WORLD, &myrank);
+  cache.patch = patch;
+  cache.NN = NN;
+  cache.symmetry = Symmetry;
+  cache.points.clear();
+  cache.points.resize(NN);
+  const int mid = NN / 2;
+  const int last = NN - 1;
+  const int ids[3] = {0, mid, last};
+  int p = 0;
+  for (int q = 0; q < 3; q++)
+    for (int d = 0; d < dim; d++)
+      cache.key[p++] = XX[d][ids[q]];
+
+  for (int j = 0; j < NN; j++)
+  {
+    double pox[dim];
+    for (int d = 0; d < dim; d++)
+      pox[d] = XX[d][j];
+    const int block_i = find_block_index_for_point(block_index, pox, DH);
+    if (block_i < 0)
+    {
+      cache.points[j].bp = 0;
+      cache.points[j].owner_rank = -1;
+      continue;
+    }
+    Block *BP = block_index.views[block_i].bp;
+    cache.points[j].bp = BP;
+    cache.points[j].owner_rank = BP->rank;
+    cache.points[j].stencil.nsamples = 0;
+    if (BP->rank == myrank)
+    {
+      if (!prepare_fast_interp_stencil(BP, pox, ordn, Symmetry,
+                                       cache.points[j].stencil))
+        return false;
+    }
+  }
+  return true;
+}
 } // namespace
 
 Patch::Patch(int DIM, int *shapei, double *bboxi, int levi, bool buflog, int Symmetry) : lev(levi)
@@ -565,14 +1006,18 @@ void Patch::Interp_Points(MyList<var> *VarList,
       if (myrank == BP->rank)
       {
         //---> interpolation
-        varl = VarList;
+        if (!interpolate_var_list_fast(BP, VarList, num_var, pox, ordn,
-        int k = 0;
+                                       Symmetry, Shellf + j * num_var))
-        while (varl) // run along variables
         {
-          f_global_interp(BP->shape, BP->X[0], BP->X[1], BP->X[2], BP->fgfs[varl->data->sgfn], Shellf[j * num_var + k],
+          varl = VarList;
-                          pox[0], pox[1], pox[2], ordn, varl->data->SoA, Symmetry);
+          int k = 0;
-          varl = varl->next;
+          while (varl) // run along variables
-          k++;
+          {
+            f_global_interp(BP->shape, BP->X[0], BP->X[1], BP->X[2], BP->fgfs[varl->data->sgfn], Shellf[j * num_var + k],
+                            pox[0], pox[1], pox[2], ordn, varl->data->SoA, Symmetry);
+            varl = varl->next;
+            k++;
+          }
         }
       }
     }
@@ -659,8 +1104,6 @@ void Patch::Interp_Points(MyList<var> *VarList,
     varl = varl->next;
   }
 
-  memset(Shellf, 0, sizeof(double) * NN * num_var);
-
   // owner_rank[j] records which MPI rank owns point j
   int *owner_rank;
   owner_rank = new int[NN];
@@ -672,8 +1115,113 @@ void Patch::Interp_Points(MyList<var> *VarList,
     DH[i] = getdX(i);
   BlockBinIndex block_index;
   build_block_bin_index(this, DH, block_index);
+  SurfaceInterpCache *surface_cache = 0;
+  bool use_surface_cache = false;
+  if (interp_fast_enabled())
+  {
+    surface_cache = find_surface_cache(this, NN, XX, Symmetry);
+    use_surface_cache = surface_cache_key_matches(*surface_cache, this, NN, XX, Symmetry);
+    if (!use_surface_cache)
+      use_surface_cache = build_surface_cache(*surface_cache, this, NN, XX,
+                                              Symmetry, DH, block_index, ordn);
+  }
 
   // --- Interpolation phase (identical to original) ---
+#if USE_CUDA_BSSN
+  const bool use_gpu_interp = interp_gpu_enabled() && use_surface_cache && num_var == 2 &&
+                              VarList && VarList->next && !VarList->next->next;
+#else
+  const bool use_gpu_interp = false;
+#endif
+  if (use_gpu_interp)
+  {
+#if USE_CUDA_BSSN
+    vector<vector<int> > local_points(block_index.views.size());
+    for (int j = 0; j < NN; j++)
+    {
+      for (int i = 0; i < dim; i++)
+      {
+        if (myrank == 0 && (XX[i][j] < bbox[i] + lli[i] * DH[i] || XX[i][j] > bbox[dim + i] - uui[i] * DH[i]))
+        {
+          cout << "Patch::Interp_Points: point (";
+          for (int k = 0; k < dim; k++)
+          {
+            cout << XX[k][j];
+            if (k < dim - 1)
+              cout << ",";
+            else
+              cout << ") is out of current Patch." << endl;
+          }
+          MPI_Abort(MPI_COMM_WORLD, 1);
+        }
+      }
+
+      CachedInterpPoint &cp = surface_cache->points[j];
+      Block *BP = cp.bp;
+      owner_rank[j] = cp.owner_rank;
+      if (BP && myrank == BP->rank)
+      {
+        for (size_t bi = 0; bi < block_index.views.size(); bi++)
+        {
+          if (block_index.views[bi].bp == BP)
+          {
+            local_points[bi].push_back(j);
+            break;
+          }
+        }
+      }
+    }
+
+    var *v0 = VarList->data;
+    var *v1 = VarList->next->data;
+    double soa6[6] = {
+        v0->SoA[0], v0->SoA[1], v0->SoA[2],
+        v1->SoA[0], v1->SoA[1], v1->SoA[2]};
+
+    for (size_t bi = 0; bi < local_points.size(); bi++)
+    {
+      const int count = int(local_points[bi].size());
+      if (count <= 0)
+        continue;
+
+      Block *BP = block_index.views[bi].bp;
+      vector<double> px(count), py(count), pz(count), out(2 * count);
+      for (int q = 0; q < count; q++)
+      {
+        const int j = local_points[bi][q];
+        px[q] = XX[0][j];
+        py[q] = XX[1][j];
+        pz[q] = XX[2][j];
+      }
+
+      const double dx = BP->X[0][1] - BP->X[0][0];
+      const double dy = BP->X[1][1] - BP->X[1][0];
+      const double dz = BP->X[2][1] - BP->X[2][0];
+      const int ok = bssn_cuda_interp_host_two_fields(
+          BP, BP->shape,
+          BP->fgfs[v0->sgfn], BP->fgfs[v1->sgfn],
+          BP->X[0][0], BP->X[1][0], BP->X[2][0],
+          dx, dy, dz,
+          &px[0], &py[0], &pz[0], count,
+          ordn, Symmetry, soa6, &out[0]);
+      if (ok != 0)
+      {
+        if (myrank == 0)
+          cout << "Patch::Interp_Points: CUDA two-field interpolation failed" << endl;
+        MPI_Abort(MPI_COMM_WORLD, 1);
+      }
+
+      for (int q = 0; q < count; q++)
+      {
+        const int j = local_points[bi][q];
+        Shellf[j * num_var] = out[2 * q];
+        Shellf[j * num_var + 1] = out[2 * q + 1];
+      }
+    }
+#endif
+  }
+  else
+  {
   for (int j = 0; j < NN; j++)
   {
     double pox[dim];
@@ -695,24 +1243,55 @@ void Patch::Interp_Points(MyList<var> *VarList,
       }
     }
 
-    const int block_i = find_block_index_for_point(block_index, pox, DH);
+    if (use_surface_cache)
-    if (block_i >= 0)
     {
-      Block *BP = block_index.views[block_i].bp;
+      CachedInterpPoint &cp = surface_cache->points[j];
-      owner_rank[j] = BP->rank;
+      Block *BP = cp.bp;
-      if (myrank == BP->rank)
+      owner_rank[j] = cp.owner_rank;
+      if (BP && myrank == BP->rank)
       {
-        varl = VarList;
+        if (!interpolate_var_list_with_stencil(BP, VarList, num_var, pox,
-        int k = 0;
+                                               ordn, Symmetry, cp.stencil,
-        while (varl)
+                                               Shellf + j * num_var))
         {
-          f_global_interp(BP->shape, BP->X[0], BP->X[1], BP->X[2], BP->fgfs[varl->data->sgfn], Shellf[j * num_var + k],
+          MyList<var> *varl_fallback = VarList;
-                          pox[0], pox[1], pox[2], ordn, varl->data->SoA, Symmetry);
+          int k = 0;
-          varl = varl->next;
+          while (varl_fallback)
-          k++;
+          {
+            f_global_interp(BP->shape, BP->X[0], BP->X[1], BP->X[2], BP->fgfs[varl_fallback->data->sgfn], Shellf[j * num_var + k],
+                            pox[0], pox[1], pox[2], ordn, varl_fallback->data->SoA, Symmetry);
+            varl_fallback = varl_fallback->next;
+            k++;
+          }
         }
       }
     }
+    else
+    {
+      const int block_i = find_block_index_for_point(block_index, pox, DH);
+      if (block_i >= 0)
+      {
+        Block *BP = block_index.views[block_i].bp;
+        owner_rank[j] = BP->rank;
+      if (myrank == BP->rank)
+      {
+        if (!interpolate_var_list_fast(BP, VarList, num_var, pox, ordn,
+                                       Symmetry, Shellf + j * num_var))
+        {
+          MyList<var> *varl_fallback = VarList;
+          int k = 0;
+          while (varl_fallback)
+          {
+            f_global_interp(BP->shape, BP->X[0], BP->X[1], BP->X[2], BP->fgfs[varl_fallback->data->sgfn], Shellf[j * num_var + k],
+                            pox[0], pox[1], pox[2], ordn, varl_fallback->data->SoA, Symmetry);
+            varl_fallback = varl_fallback->next;
+            k++;
+          }
+        }
+      }
+      }
+    }
+  }
   }
 
 #ifdef INTERP_LB_PROFILE
@@ -969,14 +1548,18 @@ void Patch::Interp_Points(MyList<var> *VarList,
       if (myrank == BP->rank)
       {
         //---> interpolation
-        varl = VarList;
+        if (!interpolate_var_list_fast(BP, VarList, num_var, pox, ordn,
-        int k = 0;
+                                       Symmetry, Shellf + j * num_var))
-        while (varl) // run along variables
         {
-          f_global_interp(BP->shape, BP->X[0], BP->X[1], BP->X[2], BP->fgfs[varl->data->sgfn], Shellf[j * num_var + k],
+          varl = VarList;
-                          pox[0], pox[1], pox[2], ordn, varl->data->SoA, Symmetry);
+          int k = 0;
-          varl = varl->next;
+          while (varl) // run along variables
-          k++;
+          {
+            f_global_interp(BP->shape, BP->X[0], BP->X[1], BP->X[2], BP->fgfs[varl->data->sgfn], Shellf[j * num_var + k],
+                            pox[0], pox[1], pox[2], ordn, varl->data->SoA, Symmetry);
+            varl = varl->next;
+            k++;
+          }
         }
       }
     }

AMSS_NCKU_source/Parallel.h

@@ -106,6 +106,12 @@ namespace Parallel
     int *recv_buf_caps;
     unsigned char *send_buf_pinned;
     unsigned char *recv_buf_pinned;
+    unsigned char *send_buf_is_dev;
+    unsigned char *recv_buf_is_dev;
+    int *send_buf_caps_dev;
+    int *recv_buf_caps_dev;
+    double **send_bufs_dev;
+    double **recv_bufs_dev;
     MPI_Request *reqs;
     MPI_Status *stats;
     int max_reqs;
@@ -113,6 +119,7 @@ namespace Parallel
     int *tc_req_node;
     int *tc_req_is_recv;
     int *tc_completed;
+    bool cuda_aware_mode;
     SyncCache();
     void invalidate();
     void destroy();

AMSS_NCKU_source/Z4c_class.C

@@ -2,6 +2,7 @@
 #ifdef newc
 #include <sstream>
 #include <cstdio>
+#include <cstdlib>
 #include <map>
 using namespace std;
 #else
@@ -126,6 +127,8 @@ void Z4c_class::Initialize()
     CheckPoint->readcheck_sh(SH, myrank);
 #endif
 
+  Initialize_Level_Runtime();
+
   double h = GH->PatL[0]->data->blb->data->getdX(0);
   for (int i = 1; i < dim; i++)
     h = Mymin(h, GH->PatL[0]->data->blb->data->getdX(i));
@@ -213,6 +216,35 @@ bool fill_z4c_cuda_views(Block *cg, MyList<var> *vars,
   return idx == Z4C_CUDA_STATE_COUNT && vars == 0;
 }
 
+bool z4c_cuda_keep_resident_after_step(int lev, int trfls_in, int analysis_lev)
+{
+  static int keep_all_levels = -1;
+  if (keep_all_levels < 0)
+  {
+    const char *env = getenv("AMSS_CUDA_KEEP_ALL_LEVELS");
+    keep_all_levels = (env && atoi(env) != 0) ? 1 : 0;
+  }
+  static int enabled = -1;
+  if (enabled < 0)
+  {
+    const char *env = getenv("AMSS_CUDA_Z4C_KEEP_RESIDENT_AFTER_STEP");
+    if (env)
+      enabled = (atoi(env) != 0) ? 1 : 0;
+    else
+    {
+      env = getenv("AMSS_CUDA_KEEP_RESIDENT_AFTER_STEP");
+      enabled = (env && atoi(env) != 0) ? 1 : 0;
+    }
+  }
+  if (!enabled)
+    return false;
+  if (lev == analysis_lev)
+    return false;
+  if (keep_all_levels)
+    return true;
+  return lev < trfls_in;
+}
+
 void z4c_cuda_download_level_state(MyList<Patch> *PatL, MyList<var> *vars, int myrank, bool release_ctx)
 {
   MyList<Patch> *Pp = PatL;
@@ -356,41 +388,57 @@ bool z4c_cuda_interp_bh_point_resident(MyList<Patch> *PatL,
           if (z4c_cuda_has_resident_state(block) &&
               block->shape[0] >= ordn && block->shape[1] >= ordn && block->shape[2] >= ordn)
           {
-            const int sx = ordn;
-            const int sy = ordn;
-            const int sz = ordn;
-            const int region_all = sx * sy * sz;
-            const int i0 = z4c_cuda_interp_tile_start(block->X[0], block->shape[0], x, DH[0], ordn);
-            const int j0 = z4c_cuda_interp_tile_start(block->X[1], block->shape[1], y, DH[1], ordn);
-            const int k0 = z4c_cuda_interp_tile_start(block->X[2], block->shape[2], z, DH[2], ordn);
-            double *packed_fields = new double[3 * region_all];
             var *vars[3] = {forx, fory, forz};
-            for (int f = 0; f < 3; f++)
+            static int use_device_bh_interp = -1;
+            if (use_device_bh_interp < 0)
             {
-              if (z4c_cuda_pack_state_region_to_host_buffer(block,
+              const char *env = getenv("AMSS_CUDA_Z4C_BH_INTERP_DEVICE");
-                                                            k_z4c_cuda_bh_state_indices[f],
+              use_device_bh_interp = (env && atoi(env) != 0) ? 1 : 0;
-                                                            packed_fields + f * region_all,
+            }
-                                                            block->shape,
+            bool used_device_interp = false;
-                                                            i0, j0, k0,
+            if (use_device_bh_interp)
-                                                            sx, sy, sz) != 0)
+            {
+              double soa3[9];
+              for (int f = 0; f < 3; f++)
               {
-                delete[] packed_fields;
+                soa3[3 * f + 0] = vars[f]->SoA[0];
-                cout << "CUDA Z4C BH tile download failed" << endl;
+                soa3[3 * f + 1] = vars[f]->SoA[1];
+                soa3[3 * f + 2] = vars[f]->SoA[2];
+              }
+              used_device_interp =
+                  (z4c_cuda_interp_state_point3(block, block->shape,
+                                                k_z4c_cuda_bh_state_indices[0],
+                                                k_z4c_cuda_bh_state_indices[1],
+                                                k_z4c_cuda_bh_state_indices[2],
+                                                block->X[0][0], block->X[1][0], block->X[2][0],
+                                                DH[0], DH[1], DH[2],
+                                                x, y, z,
+                                                interp_ordn, interp_sym,
+                                                soa3, shellf) == 0);
+            }
+            if (!used_device_interp)
+            {
+              double *shift_views[3] = {
+                  block->fgfs[forx->sgfn],
+                  block->fgfs[fory->sgfn],
+                  block->fgfs[forz->sgfn]};
+              if (z4c_cuda_download_state_subset(block, block->shape, 3,
+                                                 k_z4c_cuda_bh_state_indices,
+                                                 shift_views) != 0)
+              {
+                cout << "CUDA Z4C BH shift download failed" << endl;
                 MPI_Abort(MPI_COMM_WORLD, 1);
               }
-              int tile_shape[3] = {sx, sy, sz};
+              f_global_interp(block->shape, block->X[0], block->X[1], block->X[2],
-              f_global_interp(tile_shape,
+                              block->fgfs[forx->sgfn], shellf[0],
-                              block->X[0] + i0,
+                              x, y, z, interp_ordn, forx->SoA, interp_sym);
-                              block->X[1] + j0,
+              f_global_interp(block->shape, block->X[0], block->X[1], block->X[2],
-                              block->X[2] + k0,
+                              block->fgfs[fory->sgfn], shellf[1],
-                              packed_fields + f * region_all,
+                              x, y, z, interp_ordn, fory->SoA, interp_sym);
-                              shellf[f],
+              f_global_interp(block->shape, block->X[0], block->X[1], block->X[2],
-                              x, y, z,
+                              block->fgfs[forz->sgfn], shellf[2],
-                              interp_ordn,
+                              x, y, z, interp_ordn, forz->SoA, interp_sym);
-                              vars[f]->SoA,
-                              interp_sym);
             }
-            delete[] packed_fields;
           }
           else
           {
@@ -452,6 +500,117 @@ bool z4c_cuda_compute_porg_rhs_resident(cgh *GH,
   return true;
 }
 
+bool z4c_cuda_download_bh_shift_level(MyList<Patch> *PatL,
+                                      int myrank,
+                                      var *forx, var *fory, var *forz)
+{
+  MyList<Patch> *Pp = PatL;
+  while (Pp)
+  {
+    MyList<Block> *BP = Pp->data->blb;
+    while (BP)
+    {
+      Block *cg = BP->data;
+      if (myrank == cg->rank && z4c_cuda_has_resident_state(cg))
+      {
+        double *fields[3] = {
+            cg->fgfs[forx->sgfn],
+            cg->fgfs[fory->sgfn],
+            cg->fgfs[forz->sgfn]};
+        if (z4c_cuda_download_state_subset(cg, cg->shape, 3,
+                                           k_z4c_cuda_bh_state_indices,
+                                           fields))
+          return false;
+      }
+      if (BP == Pp->data->ble)
+        break;
+      BP = BP->next;
+    }
+    Pp = Pp->next;
+  }
+  return true;
+}
+
+bool z4c_cuda_refresh_constraint_level(MyList<Patch> *PatL,
+                                       int myrank,
+                                       var *Cons_Ham, var *Cons_Px,
+                                       var *Cons_Py, var *Cons_Pz,
+                                       var *Cons_Gx, var *Cons_Gy,
+                                       var *Cons_Gz, var *TZ0,
+                                       int Symmetry, int lev, double eps)
+{
+  bool all_resident = true;
+  const int tz_index = 24;
+  MyList<Patch> *Pp = PatL;
+  while (Pp)
+  {
+    MyList<Block> *BP = Pp->data->blb;
+    while (BP)
+    {
+      Block *cg = BP->data;
+      if (myrank == cg->rank)
+      {
+        if (!z4c_cuda_has_resident_state(cg))
+        {
+          all_resident = false;
+        }
+        else
+        {
+          double *constraints[7] = {
+              cg->fgfs[Cons_Ham->sgfn], cg->fgfs[Cons_Px->sgfn],
+              cg->fgfs[Cons_Py->sgfn], cg->fgfs[Cons_Pz->sgfn],
+              cg->fgfs[Cons_Gx->sgfn], cg->fgfs[Cons_Gy->sgfn],
+              cg->fgfs[Cons_Gz->sgfn]};
+          double *tz_out[1] = {cg->fgfs[TZ0->sgfn]};
+          int co = 0;
+          if (z4c_cuda_compute_constraints_resident(cg, cg->shape,
+                                                   cg->X[0], cg->X[1], cg->X[2],
+                                                   Symmetry, eps, co,
+                                                   constraints) ||
+              z4c_cuda_download_state_subset(cg, cg->shape, 1, &tz_index, tz_out))
+          {
+            cout << "CUDA Z4C resident constraint refresh failed" << endl;
+            MPI_Abort(MPI_COMM_WORLD, 1);
+          }
+        }
+      }
+      if (BP == Pp->data->ble)
+        break;
+      BP = BP->next;
+    }
+    Pp = Pp->next;
+  }
+  return all_resident;
+}
+
+long long &z4c_constraint_output_counter()
+{
+  static long long counter = 0;
+  return counter;
+}
+
+int z4c_constraint_output_every()
+{
+  static int every = -1;
+  if (every < 0)
+  {
+    const char *env = getenv("AMSS_CUDA_Z4C_CONSTRAINT_EVERY");
+    every = (env && atoi(env) > 0) ? atoi(env) : 1;
+  }
+  return every;
+}
+
+bool z4c_constraint_output_due_now()
+{
+  const int every = z4c_constraint_output_every();
+  return every <= 1 || (z4c_constraint_output_counter() % every) == 0;
+}
+
+void z4c_constraint_output_advance()
+{
+  z4c_constraint_output_counter()++;
+}
+
 } // namespace
 #endif
 
@@ -470,6 +629,34 @@ void Z4c_class::Step(int lev, int YN)
   int iter_count = 0;
   int pre = 0, cor = 1;
   int ERROR = 0;
+  const double dT_mon = dT * pow(0.5, Mymax(0, trfls));
+  const bool need_constraint_after_step =
+      (LastConsOut + dT_mon >= AnasTime) && z4c_constraint_output_due_now();
+
+  if (BH_num > 0 && lev == GH->levels - 1)
+  {
+    if (!z4c_cuda_download_bh_shift_level(GH->PatL[lev], myrank, Sfx0, Sfy0, Sfz0))
+    {
+      if (myrank == 0 && ErrorMonitor->outfile)
+        ErrorMonitor->outfile << "CUDA Z4C failed to download predictor black-hole shift at t = "
+                              << PhysTime << endl;
+      MPI_Abort(MPI_COMM_WORLD, 1);
+    }
+    compute_Porg_rhs(Porg0, Porg_rhs, Sfx0, Sfy0, Sfz0, lev);
+    for (int ithBH = 0; ithBH < BH_num; ithBH++)
+    {
+      f_rungekutta4_scalar(dT_lev, Porg0[ithBH][0], Porg[ithBH][0], Porg_rhs[ithBH][0], iter_count);
+      f_rungekutta4_scalar(dT_lev, Porg0[ithBH][1], Porg[ithBH][1], Porg_rhs[ithBH][1], iter_count);
+      f_rungekutta4_scalar(dT_lev, Porg0[ithBH][2], Porg[ithBH][2], Porg_rhs[ithBH][2], iter_count);
+      if (Symmetry > 0)
+        Porg[ithBH][2] = fabs(Porg[ithBH][2]);
+      if (Symmetry == 2)
+      {
+        Porg[ithBH][0] = fabs(Porg[ithBH][0]);
+        Porg[ithBH][1] = fabs(Porg[ithBH][1]);
+      }
+    }
+  }
 
   MyList<Patch> *Pp = GH->PatL[lev];
   while (Pp)
@@ -537,24 +724,10 @@ void Z4c_class::Step(int lev, int YN)
     MPI_Abort(MPI_COMM_WORLD, 1);
   }
 
-  Parallel::Sync(GH->PatL[lev], SynchList_pre, Symmetry);
-
-  if (BH_num > 0 && lev == GH->levels - 1)
   {
-    compute_Porg_rhs(Porg0, Porg_rhs, Sfx0, Sfy0, Sfz0, lev);
+    Parallel::AsyncSyncState async_pre;
-    for (int ithBH = 0; ithBH < BH_num; ithBH++)
+    Parallel::Sync_start(GH->PatL[lev], SynchList_pre, Symmetry, sync_cache_pre[lev], async_pre);
-    {
+    Parallel::Sync_finish(sync_cache_pre[lev], async_pre, SynchList_pre, Symmetry);
-      f_rungekutta4_scalar(dT_lev, Porg0[ithBH][0], Porg[ithBH][0], Porg_rhs[ithBH][0], iter_count);
-      f_rungekutta4_scalar(dT_lev, Porg0[ithBH][1], Porg[ithBH][1], Porg_rhs[ithBH][1], iter_count);
-      f_rungekutta4_scalar(dT_lev, Porg0[ithBH][2], Porg[ithBH][2], Porg_rhs[ithBH][2], iter_count);
-      if (Symmetry > 0)
-        Porg[ithBH][2] = fabs(Porg[ithBH][2]);
-      if (Symmetry == 2)
-      {
-        Porg[ithBH][0] = fabs(Porg[ithBH][0]);
-        Porg[ithBH][1] = fabs(Porg[ithBH][1]);
-      }
-    }
   }
 
   if ((lev == a_lev) && (LastAnas + dT_lev >= AnasTime))
@@ -614,6 +787,25 @@ void Z4c_class::Step(int lev, int YN)
                  << cg->bbox[2] << ":" << cg->bbox[5] << ")" << endl;
             ERROR = 1;
           }
+          if (!ERROR && iter_count == 3 && need_constraint_after_step)
+          {
+            double *constraints[7] = {
+                cg->fgfs[Cons_Ham->sgfn], cg->fgfs[Cons_Px->sgfn],
+                cg->fgfs[Cons_Py->sgfn], cg->fgfs[Cons_Pz->sgfn],
+                cg->fgfs[Cons_Gx->sgfn], cg->fgfs[Cons_Gy->sgfn],
+                cg->fgfs[Cons_Gz->sgfn]};
+            double *tz_out[1] = {cg->fgfs[TZ0->sgfn]};
+            const int tz_index = 24;
+            if (z4c_cuda_download_constraint_outputs(cg->shape, constraints) ||
+                z4c_cuda_download_state_subset(cg, cg->shape, 1, &tz_index, tz_out))
+            {
+              cout << "CUDA Z4C constraint download failed in domain: ("
+                   << cg->bbox[0] << ":" << cg->bbox[3] << ","
+                   << cg->bbox[1] << ":" << cg->bbox[4] << ","
+                   << cg->bbox[2] << ":" << cg->bbox[5] << ")" << endl;
+              ERROR = 1;
+            }
+          }
         }
         if (BP == Pp->data->ble)
           break;
@@ -635,7 +827,11 @@ void Z4c_class::Step(int lev, int YN)
       MPI_Abort(MPI_COMM_WORLD, 1);
     }
 
-    Parallel::Sync(GH->PatL[lev], SynchList_cor, Symmetry);
+    {
+      Parallel::AsyncSyncState async_cor;
+      Parallel::Sync_start(GH->PatL[lev], SynchList_cor, Symmetry, sync_cache_cor[lev], async_cor);
+      Parallel::Sync_finish(sync_cache_cor[lev], async_cor, SynchList_cor, Symmetry);
+    }
 
     if (BH_num > 0 && lev == GH->levels - 1)
     {
@@ -691,7 +887,13 @@ void Z4c_class::Step(int lev, int YN)
     }
   }
 
-  z4c_cuda_download_level_state(GH->PatL[lev], SynchList_cor, myrank, true);
+  {
+    const bool keep_resident = z4c_cuda_keep_resident_after_step(lev, trfls, a_lev);
+    const bool need_host_after_step =
+        ((lev == a_lev) && (LastAnas + dT_lev >= AnasTime));
+    if (!keep_resident || need_host_after_step)
+      z4c_cuda_download_level_state(GH->PatL[lev], SynchList_cor, myrank, !keep_resident);
+  }
 
 #if (RPS == 0)
   RestrictProlong(lev, YN, BB);
@@ -2970,8 +3172,14 @@ void Z4c_class::Constraint_Out()
   if (LastConsOut >= AnasTime)
   // Constraint violation
   {
+#if USE_CUDA_Z4C && (ABEtype == 2)
+    bool cuda_constraints_ready = true;
+#else
+    const bool cuda_constraints_ready = false;
+#endif
     // recompute least the constraint data lost for moved new grid
-    for (int lev = 0; lev < GH->levels; lev++)
+    if (!cuda_constraints_ready)
+      for (int lev = 0; lev < GH->levels; lev++)
     {
       // make sure the data consistent for higher levels
       if (lev > 0)

AMSS_NCKU_source/bssnEM_class.C

@@ -18,6 +18,9 @@ using namespace std;
 #include "Parallel.h"
 #include "bssnEM_class.h"
 #include "bssn_rhs.h"
+#if USE_CUDA_BSSN
+#include "bssn_rhs_cuda.h"
+#endif
 #include "empart.h"
 #include "initial_puncture.h"
 #include "initial_maxwell.h"
@@ -36,6 +39,106 @@ using namespace std;
 
 //================================================================================================
 
+#if USE_CUDA_BSSN
+namespace {
+
+bool fill_bssn_cuda_views_prefix(Block *cg, MyList<var> *vars,
+                                 double **host_views,
+                                 double *propspeeds = nullptr,
+                                 double *soa_flat = nullptr)
+{
+  int idx = 0;
+  while (vars && idx < BSSN_CUDA_STATE_COUNT)
+  {
+    host_views[idx] = cg->fgfs[vars->data->sgfn];
+    if (propspeeds)
+      propspeeds[idx] = vars->data->propspeed;
+    if (soa_flat)
+    {
+      soa_flat[3 * idx + 0] = vars->data->SoA[0];
+      soa_flat[3 * idx + 1] = vars->data->SoA[1];
+      soa_flat[3 * idx + 2] = vars->data->SoA[2];
+    }
+    vars = vars->next;
+    ++idx;
+  }
+  return idx == BSSN_CUDA_STATE_COUNT;
+}
+
+void skip_bssn_cuda_prefix(MyList<var> *&a, MyList<var> *&b, MyList<var> *&c)
+{
+  for (int i = 0; i < BSSN_CUDA_STATE_COUNT && a && b && c; ++i)
+  {
+    a = a->next;
+    b = b->next;
+    c = c->next;
+  }
+}
+
+void skip_bssn_cuda_prefix(MyList<var> *&a, MyList<var> *&b,
+                           MyList<var> *&c, MyList<var> *&d)
+{
+  for (int i = 0; i < BSSN_CUDA_STATE_COUNT && a && b && c && d; ++i)
+  {
+    a = a->next;
+    b = b->next;
+    c = c->next;
+    d = d->next;
+  }
+}
+
+int run_bssn_em_cuda_substep(Block *cg,
+                             MyList<var> *state_in_list,
+                             MyList<var> *state_out_list,
+                             Patch *patch,
+                             double &dT_lev,
+                             double &TRK4,
+                             int &iter_count,
+                             int &Symmetry,
+                             int lev,
+                             double &ndeps,
+                             int &co,
+                             double &chitiny,
+                             var *rho, var *Sx, var *Sy, var *Sz,
+                             var *Sxx, var *Sxy, var *Sxz,
+                             var *Syy, var *Syz, var *Szz)
+{
+  double *state_in[BSSN_CUDA_STATE_COUNT];
+  double *state_out[BSSN_CUDA_STATE_COUNT];
+  double *matter[BSSN_CUDA_MATTER_COUNT] = {
+      cg->fgfs[rho->sgfn], cg->fgfs[Sx->sgfn], cg->fgfs[Sy->sgfn], cg->fgfs[Sz->sgfn],
+      cg->fgfs[Sxx->sgfn], cg->fgfs[Sxy->sgfn], cg->fgfs[Sxz->sgfn],
+      cg->fgfs[Syy->sgfn], cg->fgfs[Syz->sgfn], cg->fgfs[Szz->sgfn]};
+  double propspeed[BSSN_CUDA_STATE_COUNT];
+  double soa_flat[3 * BSSN_CUDA_STATE_COUNT];
+  if (!fill_bssn_cuda_views_prefix(cg, state_in_list, state_in, propspeed, soa_flat) ||
+      !fill_bssn_cuda_views_prefix(cg, state_out_list, state_out))
+    return 1;
+
+  int apply_bam_bc = 0;
+#if (SommerType == 0)
+#ifndef WithShell
+  apply_bam_bc = (lev == 0) ? 1 : 0;
+#endif
+#endif
+  int use_zero_matter = 0;
+  int keep_resident_state = 0;
+  int apply_enforce_ga = 0;
+  return bssn_cuda_rk4_substep(cg,
+                               cg->shape, cg->X[0], cg->X[1], cg->X[2],
+                               state_in, state_out, matter,
+                               propspeed, soa_flat, patch->bbox,
+                               dT_lev, TRK4, iter_count, apply_bam_bc,
+                               Symmetry, lev, ndeps, co,
+                               use_zero_matter,
+                               keep_resident_state, apply_enforce_ga, chitiny);
+}
+
+}
+#endif
+
+//================================================================================================
+
 // Define bssnEM_class
 
 // It inherits some members and methods from the parent class bssn_class and modifies others.
@@ -244,6 +347,8 @@ void bssnEM_class::Initialize()
     CheckPoint->readcheck_sh(SH, myrank);
 #endif
 
+  Initialize_Level_Runtime();
+
   double h = GH->PatL[0]->data->blb->data->getdX(0);
   for (int i = 1; i < dim; i++)
     h = Mymin(h, GH->PatL[0]->data->blb->data->getdX(i));
@@ -853,6 +958,7 @@ void bssnEM_class::Step(int lev, int YN)
                      cg->fgfs[Ayy0->sgfn], cg->fgfs[Ayz0->sgfn], cg->fgfs[Azz0->sgfn]);
 #endif
 
+        bool used_gpu_substep = false;
         if (
             f_compute_rhs_empart(cg->shape, cg->X[0], cg->X[1], cg->X[2],
                                  cg->fgfs[phi0->sgfn],
@@ -874,7 +980,16 @@ void bssnEM_class::Step(int lev, int YN)
                                  cg->fgfs[Sxx->sgfn], cg->fgfs[Sxy->sgfn], cg->fgfs[Sxz->sgfn],
                                  cg->fgfs[Syy->sgfn], cg->fgfs[Syz->sgfn], cg->fgfs[Szz->sgfn],
                                  Symmetry, lev, ndeps) ||
-            f_compute_rhs_bssn(cg->shape, TRK4, cg->X[0], cg->X[1], cg->X[2],
+#if USE_CUDA_BSSN
+            ((used_gpu_substep =
+                  (run_bssn_em_cuda_substep(cg, StateList, SynchList_pre, Pp->data,
+                                            dT_lev, TRK4, iter_count, Symmetry, lev,
+                                            ndeps, pre, chitiny,
+                                            rho, Sx, Sy, Sz, Sxx, Sxy, Sxz, Syy, Syz, Szz) == 0))
+                 ? 0
+                 : 1) ||
+#endif
+            (!used_gpu_substep && f_compute_rhs_bssn(cg->shape, TRK4, cg->X[0], cg->X[1], cg->X[2],
                                cg->fgfs[phi0->sgfn], cg->fgfs[trK0->sgfn],
                                cg->fgfs[gxx0->sgfn], cg->fgfs[gxy0->sgfn], cg->fgfs[gxz0->sgfn], 
                                cg->fgfs[gyy0->sgfn], cg->fgfs[gyz0->sgfn], cg->fgfs[gzz0->sgfn],
@@ -907,7 +1022,7 @@ void bssnEM_class::Step(int lev, int YN)
                                cg->fgfs[Cons_Ham->sgfn],
                                cg->fgfs[Cons_Px->sgfn], cg->fgfs[Cons_Py->sgfn], cg->fgfs[Cons_Pz->sgfn],
                                cg->fgfs[Cons_Gx->sgfn], cg->fgfs[Cons_Gy->sgfn], cg->fgfs[Cons_Gz->sgfn],
-                               Symmetry, lev, ndeps, pre))
+                               Symmetry, lev, ndeps, pre)))
         {
           cout << "find NaN in domain: (" 
                << cg->bbox[0] << ":" << cg->bbox[3] << "," 
@@ -920,6 +1035,10 @@ void bssnEM_class::Step(int lev, int YN)
         {
           MyList<var> *varl0 = StateList, *varl = SynchList_pre, *varlrhs = RHSList;
           // we do not check the correspondence here
+#if USE_CUDA_BSSN
+          if (used_gpu_substep)
+            skip_bssn_cuda_prefix(varl0, varl, varlrhs);
+#endif
 
           while (varl0)
           {
@@ -1221,7 +1340,7 @@ void bssnEM_class::Step(int lev, int YN)
   }
 #endif
 
-  Parallel::Sync(GH->PatL[lev], SynchList_pre, Symmetry);
+  Parallel::Sync_cached(GH->PatL[lev], SynchList_pre, Symmetry, sync_cache_pre[lev]);
 
 #ifdef WithShell
   if (lev == 0)
@@ -1309,6 +1428,7 @@ void bssnEM_class::Step(int lev, int YN)
                          cg->fgfs[Ayy->sgfn], cg->fgfs[Ayz->sgfn], cg->fgfs[Azz->sgfn]);
 #endif
 
+          bool used_gpu_substep = false;
           if (
               f_compute_rhs_empart(cg->shape, cg->X[0], cg->X[1], cg->X[2],
                                    cg->fgfs[phi->sgfn],
@@ -1330,7 +1450,16 @@ void bssnEM_class::Step(int lev, int YN)
                                    cg->fgfs[Sxx->sgfn], cg->fgfs[Sxy->sgfn], cg->fgfs[Sxz->sgfn],
                                    cg->fgfs[Syy->sgfn], cg->fgfs[Syz->sgfn], cg->fgfs[Szz->sgfn],
                                    Symmetry, lev, ndeps) ||
-              f_compute_rhs_bssn(cg->shape, TRK4, cg->X[0], cg->X[1], cg->X[2],
+#if USE_CUDA_BSSN
+              ((used_gpu_substep =
+                    (run_bssn_em_cuda_substep(cg, SynchList_pre, SynchList_cor, Pp->data,
+                                              dT_lev, TRK4, iter_count, Symmetry, lev,
+                                              ndeps, cor, chitiny,
+                                              rho, Sx, Sy, Sz, Sxx, Sxy, Sxz, Syy, Syz, Szz) == 0))
+                   ? 0
+                   : 1) ||
+#endif
+              (!used_gpu_substep && f_compute_rhs_bssn(cg->shape, TRK4, cg->X[0], cg->X[1], cg->X[2],
                                  cg->fgfs[phi->sgfn], cg->fgfs[trK->sgfn],
                                  cg->fgfs[gxx->sgfn], cg->fgfs[gxy->sgfn], cg->fgfs[gxz->sgfn], 
                                  cg->fgfs[gyy->sgfn], cg->fgfs[gyz->sgfn], cg->fgfs[gzz->sgfn],
@@ -1362,7 +1491,7 @@ void bssnEM_class::Step(int lev, int YN)
                                  cg->fgfs[Cons_Ham->sgfn],
                                  cg->fgfs[Cons_Px->sgfn], cg->fgfs[Cons_Py->sgfn], cg->fgfs[Cons_Pz->sgfn],
                                  cg->fgfs[Cons_Gx->sgfn], cg->fgfs[Cons_Gy->sgfn], cg->fgfs[Cons_Gz->sgfn],
-                                 Symmetry, lev, ndeps, cor))
+                                 Symmetry, lev, ndeps, cor)))
           {
             cout << "find NaN in domain: (" 
                  << cg->bbox[0] << ":" << cg->bbox[3] << "," 
@@ -1374,6 +1503,10 @@ void bssnEM_class::Step(int lev, int YN)
           {
             MyList<var> *varl0 = StateList, *varl = SynchList_pre, *varl1 = SynchList_cor, *varlrhs = RHSList;
             // we do not check the correspondence here
+#if USE_CUDA_BSSN
+            if (used_gpu_substep)
+              skip_bssn_cuda_prefix(varl0, varl, varl1, varlrhs);
+#endif
 
             while (varl0)
             {
@@ -1683,7 +1816,7 @@ void bssnEM_class::Step(int lev, int YN)
     }
 #endif
 
-    Parallel::Sync(GH->PatL[lev], SynchList_cor, Symmetry);
+    Parallel::Sync_cached(GH->PatL[lev], SynchList_cor, Symmetry, sync_cache_cor[lev]);
 
 #ifdef WithShell
     if (lev == 0)

AMSS_NCKU_source/bssnEScalar_class.C

@@ -18,6 +18,9 @@ using namespace std;
 #include "Parallel.h"
 #include "bssnEScalar_class.h"
 #include "bssn_rhs.h"
+#if USE_CUDA_BSSN
+#include "bssn_rhs_cuda.h"
+#endif
 #include "initial_puncture.h"
 #include "enforce_algebra.h"
 #include "rungekutta4_rout.h"
@@ -33,6 +36,350 @@ using namespace std;
 
 //================================================================================================
 
+namespace {
+
+int amss_escalar_analysis_map_every()
+{
+  static int every = -1;
+  if (every < 0)
+  {
+    const char *env = getenv("AMSS_ANALYSIS_MAP_EVERY");
+    every = (env && atoi(env) > 0) ? atoi(env) : 1;
+  }
+  return every;
+}
+
+}
+
+//================================================================================================
+
+#if USE_CUDA_BSSN
+extern "C" {
+#ifdef fortran1
+void set_escalar_parameter(double &, double &, double &, double &, double &);
+#endif
+#ifdef fortran2
+void SET_ESCALAR_PARAMETER(double &, double &, double &, double &, double &);
+#endif
+#ifdef fortran3
+void set_escalar_parameter_(double &, double &, double &, double &, double &);
+#endif
+}
+
+namespace {
+
+bool fill_bssn_cuda_views_prefix(Block *cg, MyList<var> *vars,
+                                 double **host_views,
+                                 double *propspeeds = nullptr,
+                                 double *soa_flat = nullptr)
+{
+  int idx = 0;
+  while (vars && idx < BSSN_CUDA_STATE_COUNT)
+  {
+    host_views[idx] = cg->fgfs[vars->data->sgfn];
+    if (propspeeds)
+      propspeeds[idx] = vars->data->propspeed;
+    if (soa_flat)
+    {
+      soa_flat[3 * idx + 0] = vars->data->SoA[0];
+      soa_flat[3 * idx + 1] = vars->data->SoA[1];
+      soa_flat[3 * idx + 2] = vars->data->SoA[2];
+    }
+    vars = vars->next;
+    ++idx;
+  }
+  return idx == BSSN_CUDA_STATE_COUNT;
+}
+
+void skip_bssn_cuda_prefix(MyList<var> *&a, MyList<var> *&b, MyList<var> *&c)
+{
+  for (int i = 0; i < BSSN_CUDA_STATE_COUNT && a && b && c; ++i)
+  {
+    a = a->next;
+    b = b->next;
+    c = c->next;
+  }
+}
+
+void skip_bssn_cuda_prefix(MyList<var> *&a, MyList<var> *&b,
+                           MyList<var> *&c, MyList<var> *&d)
+{
+  for (int i = 0; i < BSSN_CUDA_STATE_COUNT && a && b && c && d; ++i)
+  {
+    a = a->next;
+    b = b->next;
+    c = c->next;
+    d = d->next;
+  }
+}
+
+MyList<var> *clone_var_list_prefix(MyList<var> *src, int count)
+{
+  MyList<var> *dst = nullptr;
+  MyList<var> *tail = nullptr;
+  for (int i = 0; i < count && src; ++i, src = src->next)
+  {
+    MyList<var> *node = new MyList<var>(src->data);
+    if (!dst)
+      dst = node;
+    else
+      tail->next = node;
+    tail = node;
+  }
+  return dst;
+}
+
+bool escalar_gpu_rk_enabled()
+{
+  static int enabled = -1;
+  if (enabled < 0)
+  {
+    const char *env = getenv("AMSS_ESCALAR_GPU_RK");
+    enabled = (env && atoi(env) != 0) ? 1 : 0;
+  }
+  return enabled != 0;
+}
+
+bool escalar_resident_enabled()
+{
+  static int enabled = -1;
+  if (enabled < 0)
+  {
+    const char *env = getenv("AMSS_ESCALAR_RESIDENT");
+    const char *experimental = getenv("AMSS_ESCALAR_RESIDENT_EXPERIMENTAL");
+    enabled = (env && atoi(env) != 0 &&
+               experimental && atoi(experimental) != 0) ? 1 : 0;
+  }
+  return enabled != 0;
+}
+
+bool escalar_step_profile_enabled()
+{
+  static int enabled = -1;
+  if (enabled < 0)
+  {
+    const char *env = getenv("AMSS_ESCALAR_STEP_PROFILE");
+    enabled = (env && atoi(env) != 0) ? 1 : 0;
+  }
+  return enabled != 0;
+}
+
+int escalar_step_profile_every()
+{
+  static int every = -1;
+  if (every < 0)
+  {
+    const char *env = getenv("AMSS_ESCALAR_STEP_PROFILE_EVERY");
+    every = (env && atoi(env) > 0) ? atoi(env) : 1;
+  }
+  return every;
+}
+
+struct EScalarStepProfile
+{
+  double start;
+  double predictor_rhs;
+  double predictor_sync;
+  double analysis;
+  double corrector_rhs;
+  double corrector_sync;
+  double restrict_prolong;
+  double other_sync;
+};
+
+void escalar_profile_init(EScalarStepProfile &p)
+{
+  p.start = MPI_Wtime();
+  p.predictor_rhs = 0.0;
+  p.predictor_sync = 0.0;
+  p.analysis = 0.0;
+  p.corrector_rhs = 0.0;
+  p.corrector_sync = 0.0;
+  p.restrict_prolong = 0.0;
+  p.other_sync = 0.0;
+}
+
+void escalar_profile_add(double &bucket, double t0)
+{
+  bucket += MPI_Wtime() - t0;
+}
+
+void escalar_profile_report(const EScalarStepProfile &p, int lev, int myrank)
+{
+  if (myrank != 0 || !escalar_step_profile_enabled())
+    return;
+  static long long call_count = 0;
+  ++call_count;
+  const int every = escalar_step_profile_every();
+  if (every > 1 && (call_count % every) != 0)
+    return;
+  const double total = MPI_Wtime() - p.start;
+  fprintf(stderr,
+          "[AMSS-ESCALAR-PROFILE] call=%lld lev=%d total=%.6f pred_rhs=%.6f pred_sync=%.6f analysis=%.6f corr_rhs=%.6f corr_sync=%.6f rp=%.6f other_sync=%.6f\n",
+          call_count, lev, total, p.predictor_rhs, p.predictor_sync,
+          p.analysis, p.corrector_rhs, p.corrector_sync,
+          p.restrict_prolong, p.other_sync);
+  fflush(stderr);
+}
+
+void clear_var_list(MyList<var> *&list)
+{
+  if (list)
+  {
+    list->clearList();
+    list = nullptr;
+  }
+}
+
+void download_bssn_cuda_prefix_if_present(MyList<Patch> *PatL,
+                                          MyList<var> *vars,
+                                          int myrank)
+{
+  while (PatL)
+  {
+    MyList<Block> *BP = PatL->data->blb;
+    while (BP)
+    {
+      Block *cg = BP->data;
+      if (myrank == cg->rank)
+      {
+        double *views[BSSN_CUDA_STATE_COUNT];
+        if (fill_bssn_cuda_views_prefix(cg, vars, views))
+          bssn_cuda_download_resident_state_if_present(cg, cg->shape, views);
+      }
+      if (BP == PatL->data->ble)
+        break;
+      BP = BP->next;
+    }
+    PatL = PatL->next;
+  }
+}
+
+void download_escalar_cuda_pair_if_present(MyList<Patch> *PatL,
+                                           var *Sphi_var,
+                                           var *Spi_var,
+                                           int myrank)
+{
+  if (!Sphi_var || !Spi_var)
+    return;
+  while (PatL)
+  {
+    MyList<Block> *BP = PatL->data->blb;
+    while (BP)
+    {
+      Block *cg = BP->data;
+      if (myrank == cg->rank)
+      {
+        bssn_cuda_escalar_download_fields_if_present(
+            cg, cg->shape,
+            cg->fgfs[Sphi_var->sgfn],
+            cg->fgfs[Spi_var->sgfn]);
+      }
+      if (BP == PatL->data->ble)
+        break;
+      BP = BP->next;
+    }
+    PatL = PatL->next;
+  }
+}
+
+int run_bssn_escalar_cuda_substep(Block *cg,
+                                  MyList<var> *state_in_list,
+                                  MyList<var> *state_out_list,
+                                  Patch *patch,
+                                  double &dT_lev,
+                                  double &TRK4,
+                                  int &iter_count,
+                                  int &Symmetry,
+                                  int lev,
+                                  double &ndeps,
+                                  int &co,
+                                  double &chitiny,
+                                  var *Sphi_in, var *Spi_in,
+                                  var *Sphi_out, var *Spi_out,
+                                  var *Sphi_rhs, var *Spi_rhs,
+                                  var *rho, var *Sx, var *Sy, var *Sz,
+                                  var *Sxx, var *Sxy, var *Sxz,
+                                  var *Syy, var *Syz, var *Szz)
+{
+  double *state_in[BSSN_CUDA_STATE_COUNT];
+  double *state_out[BSSN_CUDA_STATE_COUNT];
+  double propspeed[BSSN_CUDA_STATE_COUNT];
+  double soa_flat[3 * BSSN_CUDA_STATE_COUNT];
+  if (!fill_bssn_cuda_views_prefix(cg, state_in_list, state_in, propspeed, soa_flat) ||
+      !fill_bssn_cuda_views_prefix(cg, state_out_list, state_out))
+    return 1;
+
+  double a2 = 0.0, phi0 = 0.0, r0 = 0.0, sigma0 = 0.0, l2 = 0.0;
+#ifdef fortran1
+  set_escalar_parameter(a2, phi0, r0, sigma0, l2);
+#endif
+#ifdef fortran2
+  SET_ESCALAR_PARAMETER(a2, phi0, r0, sigma0, l2);
+#endif
+#ifdef fortran3
+  set_escalar_parameter_(a2, phi0, r0, sigma0, l2);
+#endif
+  int apply_enforce_ga = 0;
+#if (AGM == 0)
+  apply_enforce_ga = 1;
+#elif (AGM == 1)
+  apply_enforce_ga = (iter_count == 3) ? 1 : 0;
+#endif
+  if (bssn_cuda_compute_escalar_matter(cg,
+                                       cg->shape, cg->X[0], cg->X[1], cg->X[2],
+                                       state_in,
+                                       cg->fgfs[Sphi_in->sgfn],
+                                       cg->fgfs[Spi_in->sgfn],
+                                       cg->fgfs[Sphi_rhs->sgfn],
+                                       cg->fgfs[Spi_rhs->sgfn],
+                                       a2, Symmetry, lev, ndeps, co, apply_enforce_ga))
+    return 1;
+
+  int apply_bam_bc = 0;
+#if (SommerType == 0)
+#ifndef WithShell
+  apply_bam_bc = (lev == 0) ? 1 : 0;
+#endif
+#endif
+  if (escalar_gpu_rk_enabled())
+  {
+    double scalar_propspeed[2] = {
+      Sphi_in->propspeed, Spi_in->propspeed
+    };
+    double scalar_soa[6] = {
+      Sphi_in->SoA[0], Sphi_in->SoA[1], Sphi_in->SoA[2],
+      Spi_in->SoA[0], Spi_in->SoA[1], Spi_in->SoA[2]
+    };
+    if (bssn_cuda_escalar_finalize_scalar_fields(cg,
+                                                 cg->shape, cg->X[0], cg->X[1], cg->X[2],
+                                                 cg->fgfs[Sphi_out->sgfn],
+                                                 cg->fgfs[Spi_out->sgfn],
+                                                 scalar_propspeed,
+                                                 scalar_soa,
+                                                 patch->bbox,
+                                                 dT_lev, iter_count, apply_bam_bc,
+                                                 Symmetry, lev, ndeps, co))
+      return 1;
+  }
+  int use_zero_matter = 0;
+  int keep_resident_state = 1;
+  double **matter_precomputed = nullptr;
+  return bssn_cuda_rk4_substep(cg,
+                               cg->shape, cg->X[0], cg->X[1], cg->X[2],
+                               state_in, state_out, matter_precomputed,
+                               propspeed, soa_flat, patch->bbox,
+                               dT_lev, TRK4, iter_count, apply_bam_bc,
+                               Symmetry, lev, ndeps, co,
+                               use_zero_matter,
+                               keep_resident_state, apply_enforce_ga, chitiny);
+}
+
+}
+#endif
+
+//================================================================================================
+
 // Define bssnEScalar_class
 
 // It inherits some members and methods from the parent class bssn_class and modifies others.
@@ -52,6 +399,14 @@ bssnEScalar_class::bssnEScalar_class(double Couranti, double StartTimei, double
                                                   Symmetryi, checkruni, checkfilenamei, numepssi, numepsbi, numepshi,
                                                   a_levi, maxli, decni, maxrexi, drexi)
 {
+  BSSNStateList = nullptr;
+  BSSNSynchList_pre = nullptr;
+  BSSNSynchList_cor = nullptr;
+  ScalarSynchList_pre = nullptr;
+  ScalarSynchList_cor = nullptr;
+  sync_cache_scalar_pre = nullptr;
+  sync_cache_scalar_cor = nullptr;
+
   // setup Monitors
   {
     char str[50];
@@ -110,6 +465,16 @@ void bssnEScalar_class::Initialize()
   DumpList->insert(Spi0);
   DumpList->insert(Cons_fR);
 
+#if USE_CUDA_BSSN
+  BSSNStateList = clone_var_list_prefix(StateList, BSSN_CUDA_STATE_COUNT);
+  BSSNSynchList_pre = clone_var_list_prefix(SynchList_pre, BSSN_CUDA_STATE_COUNT);
+  BSSNSynchList_cor = clone_var_list_prefix(SynchList_cor, BSSN_CUDA_STATE_COUNT);
+  ScalarSynchList_pre = new MyList<var>(Sphi);
+  ScalarSynchList_pre->insert(Spi);
+  ScalarSynchList_cor = new MyList<var>(Sphi1);
+  ScalarSynchList_cor->insert(Spi1);
+#endif
+
   CheckPoint->addvariablelist(StateList);
   CheckPoint->addvariablelist(OldStateList);
   
@@ -151,6 +516,14 @@ void bssnEScalar_class::Initialize()
     CheckPoint->readcheck_sh(SH, myrank);
 #endif
 
+  Initialize_Level_Runtime();
+#if USE_CUDA_BSSN
+  if (!sync_cache_scalar_pre)
+    sync_cache_scalar_pre = new Parallel::SyncCache[GH->levels];
+  if (!sync_cache_scalar_cor)
+    sync_cache_scalar_cor = new Parallel::SyncCache[GH->levels];
+#endif
+
   double h = GH->PatL[0]->data->blb->data->getdX(0);
   for (int i = 1; i < dim; i++)
     h = Mymin(h, GH->PatL[0]->data->blb->data->getdX(i));
@@ -179,6 +552,30 @@ void bssnEScalar_class::Initialize()
 
 bssnEScalar_class::~bssnEScalar_class()
 {
+#if USE_CUDA_BSSN
+  clear_var_list(BSSNStateList);
+  clear_var_list(BSSNSynchList_pre);
+  clear_var_list(BSSNSynchList_cor);
+  clear_var_list(ScalarSynchList_pre);
+  clear_var_list(ScalarSynchList_cor);
+  if (sync_cache_scalar_pre)
+  {
+    const int levels = GH ? GH->levels : 0;
+    for (int i = 0; i < levels; ++i)
+      sync_cache_scalar_pre[i].destroy();
+    delete[] sync_cache_scalar_pre;
+    sync_cache_scalar_pre = nullptr;
+  }
+  if (sync_cache_scalar_cor)
+  {
+    const int levels = GH ? GH->levels : 0;
+    for (int i = 0; i < levels; ++i)
+      sync_cache_scalar_cor[i].destroy();
+    delete[] sync_cache_scalar_cor;
+    sync_cache_scalar_cor = nullptr;
+  }
+#endif
+
   delete Sphio;
   delete Spio;
   delete Sphi0;
@@ -719,27 +1116,44 @@ void bssnEScalar_class::Step(int lev, int YN)
   int iter_count = 0; // count RK4 substeps
   int pre = 0, cor = 1;
   int ERROR = 0;
+  EScalarStepProfile escalar_profile;
+  escalar_profile_init(escalar_profile);
 
   MyList<ss_patch> *sPp;
   // Predictor
+  const double escalar_profile_predictor_rhs_start = MPI_Wtime();
   MyList<Patch> *Pp = GH->PatL[lev];
   while (Pp)
   {
     MyList<Block> *BP = Pp->data->blb;
     while (BP)
-    {
-      Block *cg = BP->data;
-      if (myrank == cg->rank)
       {
+        Block *cg = BP->data;
+        if (myrank == cg->rank)
+        {
+#if !USE_CUDA_BSSN
 #if (AGM == 0)
         f_enforce_ga(cg->shape,
                      cg->fgfs[gxx0->sgfn], cg->fgfs[gxy0->sgfn], cg->fgfs[gxz0->sgfn],
                      cg->fgfs[gyy0->sgfn], cg->fgfs[gyz0->sgfn], cg->fgfs[gzz0->sgfn],
                      cg->fgfs[Axx0->sgfn], cg->fgfs[Axy0->sgfn], cg->fgfs[Axz0->sgfn],
                      cg->fgfs[Ayy0->sgfn], cg->fgfs[Ayz0->sgfn], cg->fgfs[Azz0->sgfn]);
+#endif
 #endif
 
-        if (f_compute_rhs_bssn_escalar(cg->shape, TRK4, cg->X[0], cg->X[1], cg->X[2],
+        bool used_gpu_substep = false;
+        if (
+#if USE_CUDA_BSSN
+            ((used_gpu_substep =
+                  (run_bssn_escalar_cuda_substep(cg, StateList, SynchList_pre, Pp->data,
+                                                 dT_lev, TRK4, iter_count, Symmetry, lev,
+                                                 ndeps, pre, chitiny,
+                                                 Sphi0, Spi0, Sphi, Spi, Sphi_rhs, Spi_rhs,
+                                                 rho, Sx, Sy, Sz, Sxx, Sxy, Sxz, Syy, Syz, Szz) == 0))
+                 ? 0
+                 : 1) ||
+#endif
+            (!used_gpu_substep && f_compute_rhs_bssn_escalar(cg->shape, TRK4, cg->X[0], cg->X[1], cg->X[2],
                                        cg->fgfs[phi0->sgfn], cg->fgfs[trK0->sgfn],
                                        cg->fgfs[gxx0->sgfn], cg->fgfs[gxy0->sgfn], cg->fgfs[gxz0->sgfn], 
                                        cg->fgfs[gyy0->sgfn], cg->fgfs[gyz0->sgfn], cg->fgfs[gzz0->sgfn],
@@ -774,7 +1188,7 @@ void bssnEScalar_class::Step(int lev, int YN)
                                        cg->fgfs[Cons_Ham->sgfn],
                                        cg->fgfs[Cons_Px->sgfn], cg->fgfs[Cons_Py->sgfn], cg->fgfs[Cons_Pz->sgfn],
                                        cg->fgfs[Cons_Gx->sgfn], cg->fgfs[Cons_Gy->sgfn], cg->fgfs[Cons_Gz->sgfn],
-                                       Symmetry, lev, ndeps, pre))
+                                       Symmetry, lev, ndeps, pre)))
         {
           cout << "find NaN in domain: (" 
                << cg->bbox[0] << ":" << cg->bbox[3] << "," 
@@ -786,8 +1200,40 @@ void bssnEScalar_class::Step(int lev, int YN)
         // rk4 substep and boundary
         {
           MyList<var> *varl0 = StateList, *varl = SynchList_pre, *varlrhs = RHSList; // we do not check the correspondence here
+#if USE_CUDA_BSSN
+          if (used_gpu_substep)
+            skip_bssn_cuda_prefix(varl0, varl, varlrhs);
+#endif
+          const bool scalar_gpu_rk_done =
+#if USE_CUDA_BSSN
+            used_gpu_substep && escalar_gpu_rk_enabled();
+#else
+            false;
+#endif
           while (varl0)
           {
+            if (scalar_gpu_rk_done)
+            {
+              if (!escalar_resident_enabled())
+              {
+#ifndef WithShell
+              if (lev > 0) // fix BD point
+#endif
+                f_sommerfeld_rout(cg->shape, cg->X[0], cg->X[1], cg->X[2],
+                                  Pp->data->bbox[0], Pp->data->bbox[1], Pp->data->bbox[2],
+                                  Pp->data->bbox[3], Pp->data->bbox[4], Pp->data->bbox[5],
+                                  dT_lev, cg->fgfs[phi0->sgfn],
+                                  cg->fgfs[Lap0->sgfn],
+                                  cg->fgfs[varl0->data->sgfn], cg->fgfs[varl->data->sgfn],
+                                  varl0->data->SoA,
+                                  Symmetry, cor);
+              }
+
+              varl0 = varl0->next;
+              varl = varl->next;
+              varlrhs = varlrhs->next;
+              continue;
+            }
 #ifndef WithShell
             if (lev == 0) // sommerfeld indeed
               f_sommerfeld_routbam(cg->shape, cg->X[0], cg->X[1], cg->X[2],
@@ -821,7 +1267,8 @@ void bssnEScalar_class::Step(int lev, int YN)
             varlrhs = varlrhs->next;
           }
         }
-        f_lowerboundset(cg->shape, cg->fgfs[phi->sgfn], chitiny);
+        if (!used_gpu_substep)
+          f_lowerboundset(cg->shape, cg->fgfs[phi->sgfn], chitiny);
       }
       if (BP == Pp->data->ble)
         break;
@@ -829,6 +1276,7 @@ void bssnEScalar_class::Step(int lev, int YN)
     }
     Pp = Pp->next;
   }
+  escalar_profile_add(escalar_profile.predictor_rhs, escalar_profile_predictor_rhs_start);
   // check error information
   {
     int erh = ERROR;
@@ -993,7 +1441,16 @@ void bssnEScalar_class::Step(int lev, int YN)
   }
 #endif
 
-  Parallel::Sync(GH->PatL[lev], SynchList_pre, Symmetry);
+#if USE_CUDA_BSSN
+  const double escalar_profile_predictor_sync_start = MPI_Wtime();
+  Parallel::Sync_cached(GH->PatL[lev], BSSNSynchList_pre, Symmetry, sync_cache_pre[lev]);
+  Parallel::Sync_cached(GH->PatL[lev], ScalarSynchList_pre, Symmetry, sync_cache_scalar_pre[lev]);
+  escalar_profile_add(escalar_profile.predictor_sync, escalar_profile_predictor_sync_start);
+#else
+  const double escalar_profile_predictor_sync_start = MPI_Wtime();
+  Parallel::Sync_cached(GH->PatL[lev], SynchList_pre, Symmetry, sync_cache_pre[lev]);
+  escalar_profile_add(escalar_profile.predictor_sync, escalar_profile_predictor_sync_start);
+#endif
 
 #ifdef WithShell
   if (lev == 0)
@@ -1049,7 +1506,13 @@ void bssnEScalar_class::Step(int lev, int YN)
   // Warning NOTE: the variables1 are used as temp storege room
   if (lev == a_lev)
   {
+    const double escalar_profile_analysis_start = MPI_Wtime();
+#if USE_CUDA_BSSN
+    if (escalar_resident_enabled())
+      download_escalar_cuda_pair_if_present(GH->PatL[lev], Sphi, Spi, myrank);
+#endif
     AnalysisStuff_EScalar(lev, dT_lev);
+    escalar_profile_add(escalar_profile.analysis, escalar_profile_analysis_start);
   }
   // corrector
   for (iter_count = 1; iter_count < 4; iter_count++)
@@ -1057,6 +1520,7 @@ void bssnEScalar_class::Step(int lev, int YN)
     // for RK4: t0, t0+dt/2, t0+dt/2, t0+dt;
     if (iter_count == 1 || iter_count == 3)
       TRK4 += dT_lev / 2;
+    const double escalar_profile_corrector_rhs_start = MPI_Wtime();
     Pp = GH->PatL[lev];
     while (Pp)
     {
@@ -1066,6 +1530,7 @@ void bssnEScalar_class::Step(int lev, int YN)
         Block *cg = BP->data;
         if (myrank == cg->rank)
         {
+#if !USE_CUDA_BSSN
 #if (AGM == 0)
           f_enforce_ga(cg->shape,
                        cg->fgfs[gxx->sgfn], cg->fgfs[gxy->sgfn], cg->fgfs[gxz->sgfn],
@@ -1079,9 +1544,22 @@ void bssnEScalar_class::Step(int lev, int YN)
                          cg->fgfs[gyy->sgfn], cg->fgfs[gyz->sgfn], cg->fgfs[gzz->sgfn],
                          cg->fgfs[Axx->sgfn], cg->fgfs[Axy->sgfn], cg->fgfs[Axz->sgfn],
                          cg->fgfs[Ayy->sgfn], cg->fgfs[Ayz->sgfn], cg->fgfs[Azz->sgfn]);
+#endif
 #endif
 
-          if (f_compute_rhs_bssn_escalar(cg->shape, TRK4, cg->X[0], cg->X[1], cg->X[2],
+          bool used_gpu_substep = false;
+          if (
+#if USE_CUDA_BSSN
+              ((used_gpu_substep =
+                    (run_bssn_escalar_cuda_substep(cg, SynchList_pre, SynchList_cor, Pp->data,
+                                                   dT_lev, TRK4, iter_count, Symmetry, lev,
+                                                   ndeps, cor, chitiny,
+                                                   Sphi, Spi, Sphi1, Spi1, Sphi_rhs, Spi_rhs,
+                                                   rho, Sx, Sy, Sz, Sxx, Sxy, Sxz, Syy, Syz, Szz) == 0))
+                   ? 0
+                   : 1) ||
+#endif
+              (!used_gpu_substep && f_compute_rhs_bssn_escalar(cg->shape, TRK4, cg->X[0], cg->X[1], cg->X[2],
                                          cg->fgfs[phi->sgfn], cg->fgfs[trK->sgfn],
                                          cg->fgfs[gxx->sgfn], cg->fgfs[gxy->sgfn], cg->fgfs[gxz->sgfn], 
                                          cg->fgfs[gyy->sgfn], cg->fgfs[gyz->sgfn], cg->fgfs[gzz->sgfn],
@@ -1117,7 +1595,7 @@ void bssnEScalar_class::Step(int lev, int YN)
                                          cg->fgfs[Cons_Ham->sgfn],
                                          cg->fgfs[Cons_Px->sgfn], cg->fgfs[Cons_Py->sgfn], cg->fgfs[Cons_Pz->sgfn],
                                          cg->fgfs[Cons_Gx->sgfn], cg->fgfs[Cons_Gy->sgfn], cg->fgfs[Cons_Gz->sgfn],
-                                         Symmetry, lev, ndeps, cor))
+                                         Symmetry, lev, ndeps, cor)))
           {
             cout << "find NaN in domain: (" 
                  << cg->bbox[0] << ":" << cg->bbox[3] << "," 
@@ -1129,9 +1607,42 @@ void bssnEScalar_class::Step(int lev, int YN)
           {
             MyList<var> *varl0 = StateList, *varl = SynchList_pre, *varl1 = SynchList_cor, *varlrhs = RHSList;
             // we do not check the correspondence here
+#if USE_CUDA_BSSN
+            if (used_gpu_substep)
+              skip_bssn_cuda_prefix(varl0, varl, varl1, varlrhs);
+#endif
+            const bool scalar_gpu_rk_done =
+#if USE_CUDA_BSSN
+              used_gpu_substep && escalar_gpu_rk_enabled();
+#else
+              false;
+#endif
 
             while (varl0)
             {
+              if (scalar_gpu_rk_done)
+              {
+                if (!escalar_resident_enabled())
+                {
+#ifndef WithShell
+                if (lev > 0) // fix BD point
+#endif
+                  f_sommerfeld_rout(cg->shape, cg->X[0], cg->X[1], cg->X[2],
+                                    Pp->data->bbox[0], Pp->data->bbox[1], Pp->data->bbox[2],
+                                    Pp->data->bbox[3], Pp->data->bbox[4], Pp->data->bbox[5],
+                                    dT_lev, cg->fgfs[phi0->sgfn],
+                                    cg->fgfs[Lap0->sgfn],
+                                    cg->fgfs[varl0->data->sgfn], cg->fgfs[varl1->data->sgfn],
+                                    varl0->data->SoA,
+                                    Symmetry, cor);
+                }
+
+                varl0 = varl0->next;
+                varl = varl->next;
+                varl1 = varl1->next;
+                varlrhs = varlrhs->next;
+                continue;
+              }
 #ifndef WithShell
               if (lev == 0) // sommerfeld indeed
                 f_sommerfeld_routbam(cg->shape, cg->X[0], cg->X[1], cg->X[2],
@@ -1166,7 +1677,8 @@ void bssnEScalar_class::Step(int lev, int YN)
               varlrhs = varlrhs->next;
             }
           }
-          f_lowerboundset(cg->shape, cg->fgfs[phi1->sgfn], chitiny);
+          if (!used_gpu_substep)
+            f_lowerboundset(cg->shape, cg->fgfs[phi1->sgfn], chitiny);
         }
         if (BP == Pp->data->ble)
           break;
@@ -1174,6 +1686,7 @@ void bssnEScalar_class::Step(int lev, int YN)
       }
       Pp = Pp->next;
     }
+    escalar_profile_add(escalar_profile.corrector_rhs, escalar_profile_corrector_rhs_start);
 
     // check error information
     {
@@ -1349,7 +1862,16 @@ void bssnEScalar_class::Step(int lev, int YN)
     }
 #endif
 
-    Parallel::Sync(GH->PatL[lev], SynchList_cor, Symmetry);
+#if USE_CUDA_BSSN
+    const double escalar_profile_corrector_sync_start = MPI_Wtime();
+    Parallel::Sync_cached(GH->PatL[lev], BSSNSynchList_cor, Symmetry, sync_cache_cor[lev]);
+    Parallel::Sync_cached(GH->PatL[lev], ScalarSynchList_cor, Symmetry, sync_cache_scalar_cor[lev]);
+    escalar_profile_add(escalar_profile.corrector_sync, escalar_profile_corrector_sync_start);
+#else
+    const double escalar_profile_corrector_sync_start = MPI_Wtime();
+    Parallel::Sync_cached(GH->PatL[lev], SynchList_cor, Symmetry, sync_cache_cor[lev]);
+    escalar_profile_add(escalar_profile.corrector_sync, escalar_profile_corrector_sync_start);
+#endif
 
 #ifdef WithShell
     if (lev == 0)
@@ -1451,7 +1973,21 @@ void bssnEScalar_class::Step(int lev, int YN)
 
 #if (RPS == 0)
   // mesh refinement boundary part
+  const double escalar_profile_rp_start = MPI_Wtime();
+#if USE_CUDA_BSSN
+  {
+    const char *mixed_env = getenv("AMSS_ESCALAR_MIXED_GPU_RP");
+    const bool mixed_gpu_rp = (mixed_env && atoi(mixed_env) != 0);
+    const char *split_env = getenv("AMSS_ESCALAR_SPLIT_RP");
+    const bool split_rp = (split_env && atoi(split_env) != 0);
+    if (escalar_resident_enabled() && !split_rp)
+      download_escalar_cuda_pair_if_present(GH->PatL[lev], Sphi1, Spi1, myrank);
+    if (!mixed_gpu_rp && !split_rp)
+      download_bssn_cuda_prefix_if_present(GH->PatL[lev], SynchList_cor, myrank);
+  }
+#endif
   RestrictProlong(lev, YN, BB);
+  escalar_profile_add(escalar_profile.restrict_prolong, escalar_profile_rp_start);
 
 #ifdef WithShell
   if (lev == 0)
@@ -1523,6 +2059,7 @@ void bssnEScalar_class::Step(int lev, int YN)
       Porg0[ithBH][2] = Porg1[ithBH][2];
     }
   }
+  escalar_profile_report(escalar_profile, lev, myrank);
 }
 
 //================================================================================================
@@ -2060,6 +2597,23 @@ void bssnEScalar_class::Constraint_Out()
   if (LastConsOut >= AnasTime)
   // Constraint violation
   {
+    const int constraint_map_every = amss_escalar_analysis_map_every();
+    static long long constraint_map_counter = 0;
+    const bool refresh_constraints =
+        constraint_map_every <= 1 ||
+        (constraint_map_counter % constraint_map_every) == 0;
+    constraint_map_counter++;
+    if (!refresh_constraints)
+    {
+      LastConsOut = 0;
+      return;
+    }
+
+#if USE_CUDA_BSSN
+    for (int lev = 0; lev < GH->levels; lev++)
+      download_bssn_cuda_prefix_if_present(GH->PatL[lev], StateList, myrank);
+#endif
+
     // recompute least the constraint data lost for moved new grid
     for (int lev = 0; lev < GH->levels; lev++)
     {

AMSS_NCKU_source/bssnEScalar_class.h

@@ -63,6 +63,10 @@ protected:
 
      var *Cons_fR;
 
+     MyList<var> *BSSNStateList, *BSSNSynchList_pre, *BSSNSynchList_cor;
+     MyList<var> *ScalarSynchList_pre, *ScalarSynchList_cor;
+     Parallel::SyncCache *sync_cache_scalar_pre, *sync_cache_scalar_cor;
+
      monitor *MaxScalar_Monitor;
 };
 

AMSS_NCKU_source/bssnEScalar_rhs.f90

@@ -5,6 +5,138 @@
 
 #include "macrodef.fh"
 
+! scalar RHS and stress-energy only; BSSN RHS can be supplied by CUDA.
+  function compute_rhs_bssn_escalar_matter(ex, T, X, Y, Z,                    &
+               chi    ,   trK    ,                                            &
+               dxx    ,   gxy    ,   gxz    ,   dyy    ,   gyz    ,   dzz,    &
+               Axx    ,   Axy    ,   Axz    ,   Ayy    ,   Ayz    ,   Azz,    &
+               Gamx   ,  Gamy    ,  Gamz    ,                                 &
+               Lap    ,  betax   ,  betay   ,  betaz   ,                      &
+               dtSfx  ,  dtSfy   ,  dtSfz   ,                                 &
+               Sphi   ,   Spi    ,                                            &
+               Sphi_rhs ,   Spi_rhs ,                                         &
+               rho,Sx,Sy,Sz,Sxx,Sxy,Sxz,Syy,Syz,Szz,                          &
+               Symmetry,Lev,eps) result(gont)
+  implicit none
+
+  integer,intent(in ):: ex(1:3), Symmetry,Lev
+  real*8, intent(in ):: T
+  real*8, intent(in ):: X(1:ex(1)),Y(1:ex(2)),Z(1:ex(3))
+  real*8, dimension(ex(1),ex(2),ex(3)),intent(inout) :: chi,dxx,dyy,dzz
+  real*8, dimension(ex(1),ex(2),ex(3)),intent(in ) :: trK
+  real*8, dimension(ex(1),ex(2),ex(3)),intent(in ) :: gxy,gxz,gyz
+  real*8, dimension(ex(1),ex(2),ex(3)),intent(in ) :: Axx,Axy,Axz,Ayy,Ayz,Azz
+  real*8, dimension(ex(1),ex(2),ex(3)),intent(in ) :: Gamx,Gamy,Gamz
+  real*8, dimension(ex(1),ex(2),ex(3)),intent(inout) :: Lap, betax, betay, betaz
+  real*8, dimension(ex(1),ex(2),ex(3)),intent(in ) :: dtSfx,  dtSfy,  dtSfz
+  real*8, dimension(ex(1),ex(2),ex(3)),intent(in ) :: Sphi,Spi
+  real*8, dimension(ex(1),ex(2),ex(3)),intent(out) :: Sphi_rhs,Spi_rhs
+  real*8, dimension(ex(1),ex(2),ex(3)),intent(inout) :: rho,Sx,Sy,Sz
+  real*8, dimension(ex(1),ex(2),ex(3)),intent(inout) :: Sxx,Sxy,Sxz,Syy,Syz,Szz
+  real*8,intent(in) :: eps
+  integer::gont
+
+  real*8, dimension(ex(1),ex(2),ex(3)) :: gxx,gyy,gzz
+  real*8, dimension(ex(1),ex(2),ex(3)) :: chix,chiy,chiz
+  real*8, dimension(ex(1),ex(2),ex(3)) :: Lapx,Lapy,Lapz
+  real*8, dimension(ex(1),ex(2),ex(3)) :: Kx,Ky,Kz,S
+  real*8, dimension(ex(1),ex(2),ex(3)) :: f,fxx,fxy,fxz,fyy,fyz,fzz
+  real*8, dimension(ex(1),ex(2),ex(3)) :: alpn1,chin1
+  real*8, dimension(ex(1),ex(2),ex(3)) :: gupxx,gupxy,gupxz
+  real*8, dimension(ex(1),ex(2),ex(3)) :: gupyy,gupyz,gupzz
+
+  real*8            :: dX
+  real*8, parameter :: ZEO=0.d0, ONE = 1.D0, TWO = 2.D0, HALF = 0.5D0
+  real*8, parameter :: SYM = 1.D0
+
+  dX = sum(chi)+sum(trK)+sum(dxx)+sum(gxy)+sum(gxz)+sum(dyy)+sum(gyz)+sum(dzz) &
+      +sum(Gamx)+sum(Gamy)+sum(Gamz)                                           &
+      +sum(Lap)+sum(Sphi)+sum(Spi)
+  if(dX.ne.dX) then
+     if(sum(chi).ne.sum(chi))write(*,*)"bssn_escalar_matter: find NaN in chi"
+     if(sum(trK).ne.sum(trK))write(*,*)"bssn_escalar_matter: find NaN in trk"
+     if(sum(dxx).ne.sum(dxx))write(*,*)"bssn_escalar_matter: find NaN in dxx"
+     if(sum(gxy).ne.sum(gxy))write(*,*)"bssn_escalar_matter: find NaN in gxy"
+     if(sum(gxz).ne.sum(gxz))write(*,*)"bssn_escalar_matter: find NaN in gxz"
+     if(sum(dyy).ne.sum(dyy))write(*,*)"bssn_escalar_matter: find NaN in dyy"
+     if(sum(gyz).ne.sum(gyz))write(*,*)"bssn_escalar_matter: find NaN in gyz"
+     if(sum(dzz).ne.sum(dzz))write(*,*)"bssn_escalar_matter: find NaN in dzz"
+     if(sum(Gamx).ne.sum(Gamx))write(*,*)"bssn_escalar_matter: find NaN in Gamx"
+     if(sum(Gamy).ne.sum(Gamy))write(*,*)"bssn_escalar_matter: find NaN in Gamy"
+     if(sum(Gamz).ne.sum(Gamz))write(*,*)"bssn_escalar_matter: find NaN in Gamz"
+     if(sum(Lap).ne.sum(Lap))write(*,*)"bssn_escalar_matter: find NaN in Lap"
+     if(sum(Sphi).ne.sum(Sphi))write(*,*)"bssn_escalar_matter: find NaN in Sphi"
+     if(sum(Spi).ne.sum(Spi))write(*,*)"bssn_escalar_matter: find NaN in Spi"
+     gont = 1
+     return
+  endif
+
+  alpn1 = Lap + ONE
+  chin1 = chi + ONE
+  gxx = dxx + ONE
+  gyy = dyy + ONE
+  gzz = dzz + ONE
+
+  call fderivs(ex,chi,chix,chiy,chiz,X,Y,Z,SYM,SYM,SYM,Symmetry,Lev)
+  call fderivs(ex,Lap,Lapx,Lapy,Lapz,X,Y,Z,SYM,SYM,SYM,Symmetry,Lev)
+
+  gupzz =  gxx * gyy * gzz + gxy * gyz * gxz + gxz * gxy * gyz - &
+           gxz * gyy * gxz - gxy * gxy * gzz - gxx * gyz * gyz
+  gupxx =   ( gyy * gzz - gyz * gyz ) / gupzz
+  gupxy = - ( gxy * gzz - gyz * gxz ) / gupzz
+  gupxz =   ( gxy * gyz - gyy * gxz ) / gupzz
+  gupyy =   ( gxx * gzz - gxz * gxz ) / gupzz
+  gupyz = - ( gxx * gyz - gxy * gxz ) / gupzz
+  gupzz =   ( gxx * gyy - gxy * gxy ) / gupzz
+
+#if 1
+  Sphi_rhs = alpn1 * Spi
+  call fderivs(ex,Sphi,Kx,Ky,Kz,X,Y,Z,SYM,SYM,SYM,Symmetry,Lev)
+  call fdderivs(ex,Sphi,fxx,fxy,fxz,fyy,fyz,fzz,X,Y,Z,SYM,SYM,SYM,Symmetry,Lev)
+  Spi_rhs =   gupxx * fxx + gupyy * fyy + gupzz * fzz +           &
+             ( gupxy * fxy + gupxz * fxz + gupyz * fyz ) * TWO -  &
+             ((Gamx+(gupxx*chix+gupxy*chiy+gupxz*chiz)/TWO/chin1)*Kx  &
+            + (Gamy+(gupxy*chix+gupyy*chiy+gupyz*chiz)/TWO/chin1)*Ky  &
+            + (Gamz+(gupxz*chix+gupyz*chiy+gupzz*chiz)/TWO/chin1)*Kz)
+  Spi_rhs = Spi_rhs*alpn1 +                                      &
+            (gupxx*Lapx*Kx + gupxy*Lapx*Ky + gupxz*Lapx*Kz       &
+            +gupxy*Lapy*Kx + gupyy*Lapy*Ky + gupyz*Lapy*Kz       &
+            +gupxz*Lapz*Kx + gupyz*Lapz*Ky + gupzz*Lapz*Kz)
+
+  call frpotential(ex,Sphi,f,S)
+  Spi_rhs = Spi_rhs*chin1 + alpn1*(trK*Spi - S)
+  rho = chin1*((gupxx * Kx * Kx + gupyy * Ky * Ky + gupzz * Kz * Kz)/TWO + &
+                gupxy * Kx * Ky + gupxz * Kx * Kz + gupyz * Ky * Kz )      &
+        + Spi*Spi/TWO+f
+  Sx = -Spi*Kx
+  Sy = -Spi*Ky
+  Sz = -Spi*Kz
+  f = (rho - Spi*Spi)/chin1
+  Sxx = Kx*Kx-f*gxx
+  Sxy = Kx*Ky-f*gxy
+  Sxz = Kx*Kz-f*gxz
+  Syy = Ky*Ky-f*gyy
+  Syz = Ky*Kz-f*gyz
+  Szz = Kz*Kz-f*gzz
+#else
+  Sphi_rhs = ZEO
+  Spi_rhs = ZEO
+  rho = ZEO
+  Sx = ZEO
+  Sy = ZEO
+  Sz = ZEO
+  Sxx = ZEO
+  Sxy = ZEO
+  Sxz = ZEO
+  Syy = ZEO
+  Syz = ZEO
+  Szz = ZEO
+#endif
+
+  gont = 0
+  return
+end function compute_rhs_bssn_escalar_matter
+
 ! rhs for scalar and GR variables
 ! here we consider vacuum spacetime only
   function compute_rhs_bssn_escalar(ex, T,X, Y, Z,                             &

AMSS_NCKU_source/bssn_class.h

@@ -184,6 +184,9 @@ public:
        virtual void Constraint_Out();
        virtual void Compute_Constraint();
 
+protected:
+       void Initialize_Level_Runtime();
+
 #ifdef With_AHF
 protected:
        MyList<var> *AHList, *AHDList, *GaugeList;

AMSS_NCKU_source/bssn_rhs.h

@@ -6,6 +6,7 @@
 #define f_compute_rhs_bssn compute_rhs_bssn
 #define f_compute_rhs_bssn_ss compute_rhs_bssn_ss
 #define f_compute_rhs_bssn_escalar compute_rhs_bssn_escalar
+#define f_compute_rhs_bssn_escalar_matter compute_rhs_bssn_escalar_matter
 #define f_compute_rhs_bssn_escalar_ss compute_rhs_bssn_escalar_ss
 #define f_compute_rhs_Z4c compute_rhs_z4c
 #define f_compute_rhs_Z4cnot compute_rhs_z4cnot
@@ -16,6 +17,7 @@
 #define f_compute_rhs_bssn COMPUTE_RHS_BSSN
 #define f_compute_rhs_bssn_ss COMPUTE_RHS_BSSN_SS
 #define f_compute_rhs_bssn_escalar COMPUTE_RHS_BSSN_ESCALAR
+#define f_compute_rhs_bssn_escalar_matter COMPUTE_RHS_BSSN_ESCALAR_MATTER
 #define f_compute_rhs_bssn_escalar_ss COMPUTE_RHS_BSSN_ESCALAR_SS
 #define f_compute_rhs_Z4c COMPUTE_RHS_Z4C
 #define f_compute_rhs_Z4cnot COMPUTE_RHS_Z4CNOT
@@ -26,6 +28,7 @@
 #define f_compute_rhs_bssn compute_rhs_bssn_
 #define f_compute_rhs_bssn_ss compute_rhs_bssn_ss_
 #define f_compute_rhs_bssn_escalar compute_rhs_bssn_escalar_
+#define f_compute_rhs_bssn_escalar_matter compute_rhs_bssn_escalar_matter_
 #define f_compute_rhs_bssn_escalar_ss compute_rhs_bssn_escalar_ss_
 #define f_compute_rhs_Z4c compute_rhs_z4c_
 #define f_compute_rhs_Z4cnot compute_rhs_z4cnot_
@@ -96,6 +99,20 @@ extern "C"
                                   int &, int &, double &, int &, int &);
 }
 
+extern "C"
+{
+        int f_compute_rhs_bssn_escalar_matter(int *, double &, double *, double *, double *,                                             // ex,T,X,Y,Z
+                                              double *, double *,                                                                        // chi, trK
+                                              double *, double *, double *, double *, double *, double *,                                // gij
+                                              double *, double *, double *, double *, double *, double *,                                // Aij
+                                              double *, double *, double *,                                                              // Gam
+                                              double *, double *, double *, double *, double *, double *, double *,                      // Gauge
+                                              double *, double *,                                                                        // Sphi, Spi
+                                              double *, double *,                                                                        // Sphi, Spi rhs
+                                              double *, double *, double *, double *, double *, double *, double *, double *, double *, double *, // stress-energy
+                                              int &, int &, double &);
+}
+
 extern "C"
 {
         int f_compute_rhs_bssn_escalar(int *, double &, double *, double *, double *,                                                      // ex,T,X,Y,Z

AMSS_NCKU_source/bssn_rhs_cuda.h

@@ -55,6 +55,117 @@ int bssn_cuda_rk4_substep(void *block_tag,
                           int &apply_enforce_ga,
                           double &chitiny);
 
+int bssn_cuda_compute_escalar_matter(void *block_tag,
+                                     int *ex, double *X, double *Y, double *Z,
+                                     double **state_host_in,
+                                     double *Sphi_host,
+                                     double *Spi_host,
+                                     double *Sphi_rhs_host,
+                                     double *Spi_rhs_host,
+                                     double a2,
+                                     int &Symmetry,
+                                     int &Lev,
+                                     double &eps,
+                                     int &co,
+                                     int &apply_enforce_ga);
+
+int bssn_cuda_escalar_finalize_scalar_fields(void *block_tag,
+                                             int *ex, double *X, double *Y, double *Z,
+                                             double *Sphi_out_host,
+                                             double *Spi_out_host,
+                                             const double *propspeed,
+                                             const double *soa_flat,
+                                             const double *bbox,
+                                             double &dT,
+                                             int &RK4,
+                                             int &apply_bam_bc,
+                                             int &Symmetry,
+                                             int &Lev,
+                                             double &eps,
+                                             int &precor);
+
+int bssn_cuda_escalar_has_resident_fields(void *block_tag,
+                                          double *Sphi_host,
+                                          double *Spi_host);
+
+int bssn_cuda_escalar_has_any_resident_fields(void *block_tag);
+
+int bssn_cuda_escalar_download_fields_if_present(void *block_tag,
+                                                 int *ex,
+                                                 double *Sphi_host,
+                                                 double *Spi_host);
+
+int bssn_cuda_pack_escalar_batch_to_host_buffer(void *block_tag,
+                                                double **scalar_host_key,
+                                                double *host_buffer,
+                                                int *ex,
+                                                int i0, int j0, int k0,
+                                                int sx, int sy, int sz);
+
+int bssn_cuda_unpack_escalar_batch_from_host_buffer(void *block_tag,
+                                                    double **scalar_host_key,
+                                                    double *host_buffer,
+                                                    int *ex,
+                                                    int i0, int j0, int k0,
+                                                    int sx, int sy, int sz);
+
+int bssn_cuda_pack_escalar_batch_to_device_buffer(void *block_tag,
+                                                  double **scalar_host_key,
+                                                  double *device_buffer,
+                                                  int *ex,
+                                                  int i0, int j0, int k0,
+                                                  int sx, int sy, int sz);
+
+int bssn_cuda_unpack_escalar_batch_from_device_buffer(void *block_tag,
+                                                      double **scalar_host_key,
+                                                      double *device_buffer,
+                                                      int *ex,
+                                                      int i0, int j0, int k0,
+                                                      int sx, int sy, int sz);
+
+int bssn_cuda_restrict_escalar_batch_to_host_buffer(void *block_tag,
+                                                    double **scalar_host_key,
+                                                    double *host_buffer,
+                                                    int *ex,
+                                                    int sx, int sy, int sz,
+                                                    int fi0, int fj0, int fk0,
+                                                    const double *scalar_soa);
+
+int bssn_cuda_prolong_escalar_batch_to_host_buffer(void *block_tag,
+                                                   double **scalar_host_key,
+                                                   double *host_buffer,
+                                                   int *ex,
+                                                   int sx, int sy, int sz,
+                                                   int ii0, int jj0, int kk0,
+                                                   int lbc_i, int lbc_j, int lbc_k,
+                                                   const double *scalar_soa);
+
+int bssn_cuda_restrict_escalar_batch_to_device_buffer(void *block_tag,
+                                                      double **scalar_host_key,
+                                                      double *device_buffer,
+                                                      int *ex,
+                                                      int sx, int sy, int sz,
+                                                      int fi0, int fj0, int fk0,
+                                                      const double *scalar_soa);
+
+int bssn_cuda_prolong_escalar_batch_to_device_buffer(void *block_tag,
+                                                     double **scalar_host_key,
+                                                     double *device_buffer,
+                                                     int *ex,
+                                                     int sx, int sy, int sz,
+                                                     int ii0, int jj0, int kk0,
+                                                     int lbc_i, int lbc_j, int lbc_k,
+                                                     const double *scalar_soa);
+
+int bssn_cuda_prepare_escalar_inter_time_level(void *block_tag,
+                                               int *ex,
+                                               double **src1_host_key,
+                                               double **src2_host_key,
+                                               double **src3_host_key,
+                                               double **dst_host_key,
+                                               int source_count,
+                                               int tindex);
+
 int bssn_cuda_copy_state_region_to_host(void *block_tag,
                                         int state_index,
                                         double *host_state,
@@ -73,6 +184,13 @@ int bssn_cuda_download_resident_state(void *block_tag,
                                       int *ex,
                                       double **state_host_out);
 
+int bssn_cuda_download_resident_state_if_present(void *block_tag,
+                                                int *ex,
+                                                double **state_host_out);
+
+int bssn_cuda_resident_state_matches(void *block_tag,
+                                     double **state_host_key);
+
 int bssn_cuda_download_constraint_outputs(int *ex,
                                           double **constraint_host_out);
 
@@ -83,6 +201,44 @@ int bssn_cuda_pack_state_region_to_host_buffer(void *block_tag,
                                                int i0, int j0, int k0,
                                                int sx, int sy, int sz);
 
+int bssn_cuda_interp_state_point3(void *block_tag,
+                                  int *ex,
+                                  int state0,
+                                  int state1,
+                                  int state2,
+                                  double x0,
+                                  double y0,
+                                  double z0,
+                                  double dx,
+                                  double dy,
+                                  double dz,
+                                  double px,
+                                  double py,
+                                  double pz,
+                                  int ordn,
+                                  int symmetry,
+                                  const double *soa3,
+                                  double *out3);
+
+int bssn_cuda_interp_host_two_fields(void *block_tag,
+                                     int *ex,
+                                     double *field0,
+                                     double *field1,
+                                     double x0,
+                                     double y0,
+                                     double z0,
+                                     double dx,
+                                     double dy,
+                                     double dz,
+                                     const double *px,
+                                     const double *py,
+                                     const double *pz,
+                                     int npoints,
+                                     int ordn,
+                                     int symmetry,
+                                     const double *soa6,
+                                     double *out_interleaved);
+
 int bssn_cuda_unpack_state_region_from_host_buffer(void *block_tag,
                                                    int state_index,
                                                    double *host_buffer,
@@ -97,6 +253,14 @@ int bssn_cuda_pack_state_batch_to_host_buffer(void *block_tag,
                                               int i0, int j0, int k0,
                                               int sx, int sy, int sz);
 
+int bssn_cuda_pack_state_batch_to_host_buffer_for_host_views(void *block_tag,
+                                                             double **state_host_key,
+                                                             int state_count,
+                                                             double *host_buffer,
+                                                             int *ex,
+                                                             int i0, int j0, int k0,
+                                                             int sx, int sy, int sz);
+
 int bssn_cuda_unpack_state_batch_from_host_buffer(void *block_tag,
                                                   int state_count,
                                                   double *host_buffer,
@@ -104,6 +268,176 @@ int bssn_cuda_unpack_state_batch_from_host_buffer(void *block_tag,
                                                   int i0, int j0, int k0,
                                                   int sx, int sy, int sz);
 
+int bssn_cuda_unpack_state_batch_from_host_buffer_for_host_views(void *block_tag,
+                                                                 double **state_host_key,
+                                                                 int state_count,
+                                                                 double *host_buffer,
+                                                                 int *ex,
+                                                                 int i0, int j0, int k0,
+                                                                 int sx, int sy, int sz);
+
+int bssn_cuda_restrict_state_batch_to_host_buffer_for_host_views(void *block_tag,
+                                                                 double **state_host_key,
+                                                                 int state_count,
+                                                                 double *host_buffer,
+                                                                 int *ex,
+                                                                 int sx, int sy, int sz,
+                                                                 int fi0, int fj0, int fk0,
+                                                                 const double *state_soa);
+
+int bssn_cuda_restrict_state_batch_to_host_buffer(void *block_tag,
+                                                  int state_count,
+                                                  double *host_buffer,
+                                                  int *ex,
+                                                  int sx, int sy, int sz,
+                                                  int fi0, int fj0, int fk0,
+                                                  const double *state_soa);
+
+int bssn_cuda_prolong_state_batch_to_host_buffer_for_host_views(void *block_tag,
+                                                                double **state_host_key,
+                                                                int state_count,
+                                                                double *host_buffer,
+                                                                int *ex,
+                                                                int sx, int sy, int sz,
+                                                                int ii0, int jj0, int kk0,
+                                                                int lbc_i, int lbc_j, int lbc_k,
+                                                                const double *state_soa);
+
+int bssn_cuda_prolong_state_batch_to_host_buffer(void *block_tag,
+                                                 int state_count,
+                                                 double *host_buffer,
+                                                 int *ex,
+                                                 int sx, int sy, int sz,
+                                                 int ii0, int jj0, int kk0,
+                                                 int lbc_i, int lbc_j, int lbc_k,
+                                                 const double *state_soa);
+
+int bssn_cuda_pack_state_batch_to_device_buffer(void *block_tag,
+                                                 int state_count,
+                                                double *device_buffer,
+                                                int *ex,
+                                                int i0, int j0, int k0,
+                                                int sx, int sy, int sz);
+
+int bssn_cuda_pack_state_batch_to_device_buffer_for_host_views(void *block_tag,
+                                                               double **state_host_key,
+                                                               int state_count,
+                                                               double *device_buffer,
+                                                               int *ex,
+                                                               int i0, int j0, int k0,
+                                                               int sx, int sy, int sz);
+
+int bssn_cuda_unpack_state_batch_from_device_buffer(void *block_tag,
+                                                    int state_count,
+                                                    double *device_buffer,
+                                                    int *ex,
+                                                    int i0, int j0, int k0,
+                                                    int sx, int sy, int sz);
+
+int bssn_cuda_unpack_state_batch_from_device_buffer_for_host_views(void *block_tag,
+                                                                   double **state_host_key,
+                                                                   int state_count,
+                                                                   double *device_buffer,
+                                                                   int *ex,
+                                                                   int i0, int j0, int k0,
+                                                                   int sx, int sy, int sz);
+
+int bssn_cuda_pack_state_segments_to_device_buffer(void *block_tag,
+                                                   int state_count,
+                                                   double *device_buffer,
+                                                   int *ex,
+                                                   int segment_count,
+                                                   const int *segment_meta);
+
+int bssn_cuda_pack_state_segments_to_device_buffer_for_host_views(void *block_tag,
+                                                                  double **state_host_key,
+                                                                  int state_count,
+                                                                  double *device_buffer,
+                                                                  int *ex,
+                                                                  int segment_count,
+                                                                  const int *segment_meta);
+
+int bssn_cuda_unpack_state_segments_from_device_buffer(void *block_tag,
+                                                       int state_count,
+                                                       double *device_buffer,
+                                                       int *ex,
+                                                       int segment_count,
+                                                       const int *segment_meta);
+
+int bssn_cuda_unpack_state_segments_from_device_buffer_for_host_views(void *block_tag,
+                                                                      double **state_host_key,
+                                                                      int state_count,
+                                                                      double *device_buffer,
+                                                                      int *ex,
+                                                                      int segment_count,
+                                                                      const int *segment_meta);
+
+int bssn_cuda_restrict_state_segments_to_device_buffer(void *block_tag,
+                                                       int state_count,
+                                                       double *device_buffer,
+                                                       int *ex,
+                                                       int segment_count,
+                                                       const int *segment_meta);
+
+int bssn_cuda_restrict_state_segments_to_device_buffer_for_host_views(void *block_tag,
+                                                                      double **state_host_key,
+                                                                      int state_count,
+                                                                      double *device_buffer,
+                                                                      int *ex,
+                                                                      int segment_count,
+                                                                      const int *segment_meta,
+                                                                      const double *state_soa);
+
+int bssn_cuda_prolong_state_segments_to_device_buffer(void *block_tag,
+                                                      int state_count,
+                                                      double *device_buffer,
+                                                      int *ex,
+                                                      int segment_count,
+                                                      const int *segment_meta);
+
+int bssn_cuda_prolong_state_segments_to_device_buffer_for_host_views(void *block_tag,
+                                                                     double **state_host_key,
+                                                                     int state_count,
+                                                                     double *device_buffer,
+                                                                     int *ex,
+                                                                     int segment_count,
+                                                                     const int *segment_meta,
+                                                                     const double *state_soa);
+
+int bssn_cuda_restrict_state_batch_to_device_buffer(void *block_tag,
+                                                    int state_count,
+                                                    double *device_buffer,
+                                                    int *ex,
+                                                    int sx, int sy, int sz,
+                                                    int fi0, int fj0, int fk0);
+
+int bssn_cuda_restrict_state_batch_to_device_buffer_for_host_views(void *block_tag,
+                                                                   double **state_host_key,
+                                                                   int state_count,
+                                                                   double *device_buffer,
+                                                                   int *ex,
+                                                                   int sx, int sy, int sz,
+                                                                   int fi0, int fj0, int fk0,
+                                                                   const double *state_soa);
+
+int bssn_cuda_prolong_state_batch_to_device_buffer(void *block_tag,
+                                                   int state_count,
+                                                   double *device_buffer,
+                                                   int *ex,
+                                                   int sx, int sy, int sz,
+                                                   int ii0, int jj0, int kk0,
+                                                   int lbc_i, int lbc_j, int lbc_k);
+
+int bssn_cuda_prolong_state_batch_to_device_buffer_for_host_views(void *block_tag,
+                                                                  double **state_host_key,
+                                                                  int state_count,
+                                                                  double *device_buffer,
+                                                                  int *ex,
+                                                                  int sx, int sy, int sz,
+                                                                  int ii0, int jj0, int kk0,
+                                                                  int lbc_i, int lbc_j, int lbc_k,
+                                                                  const double *state_soa);
+
 int bssn_cuda_download_state_subset(void *block_tag,
                                     int *ex,
                                     int subset_count,
@@ -116,6 +450,15 @@ int bssn_cuda_upload_state_subset(void *block_tag,
                                   const int *state_indices,
                                   double **state_host_in);
 
+int bssn_cuda_prepare_inter_time_level(void *block_tag,
+                                       int *ex,
+                                       double **src1_host_key,
+                                       double **src2_host_key,
+                                       double **src3_host_key,
+                                       double **dst_host_key,
+                                       int source_count,
+                                       int tindex);
+
 int bssn_cuda_has_resident_state(void *block_tag);
 
 void bssn_cuda_release_step_ctx(void *block_tag);

AMSS_NCKU_source/macrodef.fh

@@ -13,7 +13,7 @@
 
 #define ABV 0
 
-#define EScalar_CC 2
+#define EScalar_CC  2
 
 #if 0
 

AMSS_NCKU_source/macrodef.h

@@ -10,7 +10,7 @@
 
 #define GaussInt
 
-#define ABEtype 0
+#define ABEtype 1
 
 //#define With_AHF
 #define Psi4type 0
@@ -167,3 +167,4 @@
 #define TINY 1e-10
 
 #endif   /* MICRODEF_H */
+

AMSS_NCKU_source/surface_integral.C

@@ -11,6 +11,7 @@
 #include <strstream>
 #include <cmath>
 #include <map>
+#include <cstdlib>
 using namespace std;
 #else
 #include <iostream.h>
@@ -32,6 +33,20 @@ using namespace std;
 #include "parameters.h"
 
 #define PI M_PI
+
+namespace
+{
+bool amss_surface_timing_enabled()
+{
+  static int enabled = -1;
+  if (enabled < 0)
+  {
+    const char *env = getenv("AMSS_SURFACE_TIMING");
+    enabled = (env && atoi(env) != 0) ? 1 : 0;
+  }
+  return enabled != 0;
+}
+}
 //|============================================================================
 //| Constructor
 //|============================================================================
@@ -3281,6 +3296,8 @@ void surface_integral::surf_WaveMassPAng(double rex, int lev, cgh *GH,
                                          var *Sfx_rhs, var *Sfy_rhs, var *Sfz_rhs,
                                          double *Rout, monitor *Monitor, bool refresh_mass_fields)
 {
+  const bool timing = amss_surface_timing_enabled();
+  const double t_start = timing ? MPI_Wtime() : 0.0;
   if (Symmetry != 0 && Symmetry != 1)
   {
     surf_Wave(rex, lev, GH, Rpsi4, Ipsi4, spinw, maxl, NN, RP, IP, Monitor);
@@ -3325,6 +3342,7 @@ void surface_integral::surf_WaveMassPAng(double rex, int lev, cgh *GH,
       Pp = Pp->next;
     }
   }
+  const double t_refresh_done = timing ? MPI_Wtime() : 0.0;
 
   const int InList = 19;
   const int idx_rpsi4 = 0, idx_ipsi4 = 1;
@@ -3380,6 +3398,7 @@ void surface_integral::surf_WaveMassPAng(double rex, int lev, cgh *GH,
 
   double *shellf = new double[n_tot * InList];
   GH->PatL[lev]->data->Interp_Points(DG_List, n_tot, pox, shellf, Symmetry, Nmin, Nmax);
+  const double t_interp_done = timing ? MPI_Wtime() : 0.0;
 
   double *RP_out = new double[NN];
   double *IP_out = new double[NN];
@@ -3496,6 +3515,7 @@ void surface_integral::surf_WaveMassPAng(double rex, int lev, cgh *GH,
     if (Symmetry == 0)
       p_outz += f1o8 * Psi * (nx_g[n] * axz + ny_g[n] * ayz + nz_g[n] * azz) * theta_weight;
   }
+  const double t_integral_done = timing ? MPI_Wtime() : 0.0;
 
   for (int ii = 0; ii < NN; ii++)
   {
@@ -3534,6 +3554,7 @@ void surface_integral::surf_WaveMassPAng(double rex, int lev, cgh *GH,
     delete[] reduce_out;
     delete[] reduce_in;
   }
+  const double t_reduce_done = timing ? MPI_Wtime() : 0.0;
 
 #ifdef GaussInt
   mass = mass * rex * rex * dphi * factor;
@@ -3565,6 +3586,19 @@ void surface_integral::surf_WaveMassPAng(double rex, int lev, cgh *GH,
   Rout[5] = sy;
   Rout[6] = sz;
 
+  if (timing)
+  {
+    fprintf(stderr,
+            "[AMSS-SURFACE][rank %d] rex=%.6g lev=%d refresh=%.6f interp=%.6f integral=%.6f reduce=%.6f total=%.6f nlocal=%d ntotal=%d modes=%d\n",
+            myrank, rex, lev,
+            t_refresh_done - t_start,
+            t_interp_done - t_refresh_done,
+            t_integral_done - t_interp_done,
+            t_reduce_done - t_integral_done,
+            t_reduce_done - t_start,
+            Nmax - Nmin + 1, n_tot, NN);
+  }
+
   delete[] pox[0];
   delete[] pox[1];
   delete[] pox[2];

AMSS_NCKU_source/z4c_rhs_cuda.h

@@ -53,6 +53,14 @@ int z4c_cuda_pack_state_batch_to_host_buffer(void *block_tag,
                                              int i0, int j0, int k0,
                                              int sx, int sy, int sz);
 
+int z4c_cuda_pack_state_batch_to_host_buffer_for_host_views(void *block_tag,
+                                                            double **state_host_key,
+                                                            int state_count,
+                                                            double *host_buffer,
+                                                            int *ex,
+                                                            int i0, int j0, int k0,
+                                                            int sx, int sy, int sz);
+
 int z4c_cuda_unpack_state_batch_from_host_buffer(void *block_tag,
                                                  int state_count,
                                                  double *host_buffer,
@@ -60,6 +68,144 @@ int z4c_cuda_unpack_state_batch_from_host_buffer(void *block_tag,
                                                  int i0, int j0, int k0,
                                                  int sx, int sy, int sz);
 
+int z4c_cuda_unpack_state_batch_from_host_buffer_for_host_views(void *block_tag,
+                                                                double **state_host_key,
+                                                                int state_count,
+                                                                double *host_buffer,
+                                                                int *ex,
+                                                                int i0, int j0, int k0,
+                                                                int sx, int sy, int sz);
+
+int z4c_cuda_pack_state_batch_to_device_buffer(void *block_tag,
+                                               int state_count,
+                                               double *device_buffer,
+                                               int *ex,
+                                               int i0, int j0, int k0,
+                                               int sx, int sy, int sz);
+
+int z4c_cuda_pack_state_batch_to_device_buffer_for_host_views(void *block_tag,
+                                                              double **state_host_key,
+                                                              int state_count,
+                                                              double *device_buffer,
+                                                              int *ex,
+                                                              int i0, int j0, int k0,
+                                                              int sx, int sy, int sz);
+
+int z4c_cuda_unpack_state_batch_from_device_buffer(void *block_tag,
+                                                   int state_count,
+                                                   double *device_buffer,
+                                                   int *ex,
+                                                   int i0, int j0, int k0,
+                                                   int sx, int sy, int sz);
+
+int z4c_cuda_unpack_state_batch_from_device_buffer_for_host_views(void *block_tag,
+                                                                  double **state_host_key,
+                                                                  int state_count,
+                                                                  double *device_buffer,
+                                                                  int *ex,
+                                                                  int i0, int j0, int k0,
+                                                                  int sx, int sy, int sz);
+
+int z4c_cuda_pack_state_segments_to_device_buffer(void *block_tag,
+                                                  int state_count,
+                                                  double *device_buffer,
+                                                  int *ex,
+                                                  int segment_count,
+                                                  const int *segment_meta);
+
+int z4c_cuda_pack_state_segments_to_device_buffer_for_host_views(void *block_tag,
+                                                                 double **state_host_key,
+                                                                 int state_count,
+                                                                 double *device_buffer,
+                                                                 int *ex,
+                                                                 int segment_count,
+                                                                 const int *segment_meta);
+
+int z4c_cuda_unpack_state_segments_from_device_buffer(void *block_tag,
+                                                      int state_count,
+                                                      double *device_buffer,
+                                                      int *ex,
+                                                      int segment_count,
+                                                      const int *segment_meta);
+
+int z4c_cuda_unpack_state_segments_from_device_buffer_for_host_views(void *block_tag,
+                                                                     double **state_host_key,
+                                                                     int state_count,
+                                                                     double *device_buffer,
+                                                                     int *ex,
+                                                                     int segment_count,
+                                                                     const int *segment_meta);
+
+int z4c_cuda_restrict_state_segments_to_device_buffer(void *block_tag,
+                                                      int state_count,
+                                                      double *device_buffer,
+                                                      int *ex,
+                                                      int segment_count,
+                                                      const int *segment_meta,
+                                                      const double *state_soa);
+
+int z4c_cuda_restrict_state_segments_to_device_buffer_for_host_views(void *block_tag,
+                                                                     double **state_host_key,
+                                                                     int state_count,
+                                                                     double *device_buffer,
+                                                                     int *ex,
+                                                                     int segment_count,
+                                                                     const int *segment_meta,
+                                                                     const double *state_soa);
+
+int z4c_cuda_prolong_state_segments_to_device_buffer(void *block_tag,
+                                                     int state_count,
+                                                     double *device_buffer,
+                                                     int *ex,
+                                                     int segment_count,
+                                                     const int *segment_meta,
+                                                     const double *state_soa);
+
+int z4c_cuda_prolong_state_segments_to_device_buffer_for_host_views(void *block_tag,
+                                                                    double **state_host_key,
+                                                                    int state_count,
+                                                                    double *device_buffer,
+                                                                    int *ex,
+                                                                    int segment_count,
+                                                                    const int *segment_meta,
+                                                                    const double *state_soa);
+
+int z4c_cuda_restrict_state_batch_to_device_buffer(void *block_tag,
+                                                   int state_count,
+                                                   double *device_buffer,
+                                                   int *ex,
+                                                   int sx, int sy, int sz,
+                                                   int fi0, int fj0, int fk0,
+                                                   const double *state_soa);
+
+int z4c_cuda_restrict_state_batch_to_device_buffer_for_host_views(void *block_tag,
+                                                                  double **state_host_key,
+                                                                  int state_count,
+                                                                  double *device_buffer,
+                                                                  int *ex,
+                                                                  int sx, int sy, int sz,
+                                                                  int fi0, int fj0, int fk0,
+                                                                  const double *state_soa);
+
+int z4c_cuda_prolong_state_batch_to_device_buffer(void *block_tag,
+                                                  int state_count,
+                                                  double *device_buffer,
+                                                  int *ex,
+                                                  int sx, int sy, int sz,
+                                                  int ii0, int jj0, int kk0,
+                                                  int lbc_i, int lbc_j, int lbc_k,
+                                                  const double *state_soa);
+
+int z4c_cuda_prolong_state_batch_to_device_buffer_for_host_views(void *block_tag,
+                                                                 double **state_host_key,
+                                                                 int state_count,
+                                                                 double *device_buffer,
+                                                                 int *ex,
+                                                                 int sx, int sy, int sz,
+                                                                 int ii0, int jj0, int kk0,
+                                                                 int lbc_i, int lbc_j, int lbc_k,
+                                                                 const double *state_soa);
+
 int z4c_cuda_download_state_subset(void *block_tag,
                                    int *ex,
                                    int subset_count,
@@ -72,7 +218,36 @@ int z4c_cuda_upload_state_subset(void *block_tag,
                                  const int *state_indices,
                                  double **state_host_in);
 
+int z4c_cuda_compute_constraints_resident(void *block_tag,
+                                          int *ex, double *X, double *Y, double *Z,
+                                          int Symmetry, double eps, int co,
+                                          double **constraint_host_out);
+
+int z4c_cuda_interp_state_point3(void *block_tag,
+                                 int *ex,
+                                 int state0,
+                                 int state1,
+                                 int state2,
+                                 double x0,
+                                 double y0,
+                                 double z0,
+                                 double dx,
+                                 double dy,
+                                 double dz,
+                                 double px,
+                                 double py,
+                                 double pz,
+                                 int ordn,
+                                 int symmetry,
+                                 const double *soa3,
+                                 double *out3);
+
+int z4c_cuda_download_constraint_outputs(int *ex,
+                                         double **constraint_host_out);
+
 int z4c_cuda_has_resident_state(void *block_tag);
+int z4c_cuda_resident_state_matches(void *block_tag,
+                                    double **state_host_key);
 
 void z4c_cuda_release_step_ctx(void *block_tag);
 

makefile_and_run.py

@@ -9,6 +9,8 @@
 
 
 import AMSS_NCKU_Input as input_data
+import os
+import shutil
 import subprocess
 import time
 
@@ -56,6 +58,124 @@ BUILD_JOBS = 64
 
 ##################################################################
 
+def _truthy(value, default=False):
+    if value is None:
+        return default
+    if isinstance(value, bool):
+        return value
+    text = str(value).strip().lower()
+    if text == "":
+        return default
+    return text in ("1", "yes", "y", "true", "on", "enable", "enabled")
+
+
+def _input_or_env(input_name, env_name, default=None):
+    if env_name in os.environ:
+        return os.environ[env_name]
+    return getattr(input_data, input_name, default)
+
+
+def _start_cuda_mps_if_requested(runtime_env):
+    if input_data.GPU_Calculation != "yes":
+        return False
+
+    default_auto_mps = int(getattr(input_data, "MPI_processes", 1)) > 1
+    auto_mps = _truthy(
+        _input_or_env("CUDA_Auto_MPS", "AMSS_CUDA_AUTO_MPS", default_auto_mps),
+        default=default_auto_mps,
+    )
+    if not auto_mps:
+        return False
+
+    mps_control = shutil.which("nvidia-cuda-mps-control")
+    if not mps_control:
+        print(" CUDA MPS control command was not found; running without MPS.")
+        return False
+
+    uid = os.getuid()
+    pipe_dir = str(_input_or_env("CUDA_MPS_PIPE_DIRECTORY", "CUDA_MPS_PIPE_DIRECTORY",
+                                 f"/tmp/amss-ncku-mps-{uid}"))
+    log_dir = str(_input_or_env("CUDA_MPS_LOG_DIRECTORY", "CUDA_MPS_LOG_DIRECTORY",
+                                f"/tmp/amss-ncku-mps-log-{uid}"))
+    os.makedirs(pipe_dir, exist_ok=True)
+    os.makedirs(log_dir, exist_ok=True)
+
+    mps_env = runtime_env.copy()
+    mps_env["CUDA_MPS_PIPE_DIRECTORY"] = pipe_dir
+    mps_env["CUDA_MPS_LOG_DIRECTORY"] = log_dir
+
+    if os.path.exists(os.path.join(pipe_dir, "control")):
+        runtime_env.update({
+            "CUDA_MPS_PIPE_DIRECTORY": pipe_dir,
+            "CUDA_MPS_LOG_DIRECTORY": log_dir,
+        })
+        print(f" Reusing CUDA MPS daemon: {pipe_dir}")
+        return False
+
+    print(f" Starting CUDA MPS daemon for this run: {pipe_dir}")
+    result = subprocess.run([mps_control, "-d"], env=mps_env, text=True,
+                            stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+    if result.returncode != 0:
+        print(" CUDA MPS daemon did not start; running without MPS.")
+        if result.stdout:
+            print(result.stdout, end="")
+        return False
+
+    runtime_env.update({
+        "CUDA_MPS_PIPE_DIRECTORY": pipe_dir,
+        "CUDA_MPS_LOG_DIRECTORY": log_dir,
+    })
+    return True
+
+
+def _stop_cuda_mps(runtime_env):
+    mps_control = shutil.which("nvidia-cuda-mps-control")
+    if not mps_control:
+        return
+    subprocess.run([mps_control], input="quit\n", env=runtime_env, text=True,
+                   stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+
+
+def _gpu_runtime_env():
+    runtime_env = os.environ.copy()
+
+    defaults = {
+        "AMSS_INTERP_FAST": "1",
+        "AMSS_INTERP_GPU": "1",
+        "AMSS_ANALYSIS_MAP_EVERY": "1000000",
+        "AMSS_CUDA_AWARE_MPI": "1",
+        "AMSS_CUDA_KEEP_RESIDENT_AFTER_STEP": "1",
+        "AMSS_CUDA_Z4C_KEEP_RESIDENT_AFTER_STEP": "1",
+        "AMSS_CUDA_KEEP_ALL_LEVELS": "1",
+        "AMSS_CUDA_Z4C_AMR_DEVICE": "0",
+        "AMSS_CUDA_AMR_RESTRICT_DEVICE": "1",
+        "AMSS_CUDA_AMR_RESTRICT_BATCH": "0",
+        "AMSS_CUDA_DEVICE_SEGMENT_BATCH": "0",
+        "AMSS_CUDA_PIN_ESCALAR_TRANSFERS": "0",
+        "AMSS_ESCALAR_GPU_RK": "0",
+    }
+    if getattr(input_data, "Equation_Class", "") == "Z4C":
+        defaults["AMSS_CUDA_Z4C_KEEP_RESIDENT_AFTER_STEP"] = "0"
+        defaults["AMSS_CUDA_KEEP_ALL_LEVELS"] = "0"
+    for key, value in defaults.items():
+        runtime_env.setdefault(key, value)
+
+    optional_overrides = {
+        "AMSS_INTERP_FAST_COMPARE": "AMSS_Interp_Fast_Compare",
+        "AMSS_INTERP_FAST_COMPARE_LIMIT": "AMSS_Interp_Fast_Compare_Limit",
+        "AMSS_INTERP_FAST_COMPARE_TOL": "AMSS_Interp_Fast_Compare_Tol",
+        "AMSS_GPU_STAGE_TIMING": "AMSS_GPU_Stage_Timing",
+        "AMSS_GPU_STAGE_TIMING_EVERY": "AMSS_GPU_Stage_Timing_Every",
+    }
+    for env_name, input_name in optional_overrides.items():
+        if env_name not in runtime_env and hasattr(input_data, input_name):
+            runtime_env[env_name] = str(getattr(input_data, input_name))
+
+    return runtime_env
+
+
+##################################################################
+
 
 
 ##################################################################
@@ -145,6 +265,8 @@ def run_ABE():
     print(                                                      )
 
     ## Define the command to run; cast other values to strings as needed
+    mpi_env = None
+    started_mps = False
     
     if (input_data.GPU_Calculation == "no"):
         mpi_command         = NUMACTL_CPU_BIND + " mpirun -np " + str(input_data.MPI_processes) + " ./ABE"
@@ -153,21 +275,45 @@ def run_ABE():
     elif (input_data.GPU_Calculation == "yes"):
         mpi_command         = NUMACTL_CPU_BIND + " mpirun -np " + str(input_data.MPI_processes) + " ./ABE_CUDA"
         mpi_command_outfile = "ABEGPU_out.log"
+        mpi_env = _gpu_runtime_env()
+        started_mps = _start_cuda_mps_if_requested(mpi_env)
+        print(" GPU optimized runtime switches:")
+        print(f"   AMSS_INTERP_FAST={mpi_env.get('AMSS_INTERP_FAST', '')}")
+        print(f"   AMSS_INTERP_GPU={mpi_env.get('AMSS_INTERP_GPU', '')}")
+        print(f"   AMSS_ANALYSIS_MAP_EVERY={mpi_env.get('AMSS_ANALYSIS_MAP_EVERY', '')}")
+        print(f"   AMSS_CUDA_AWARE_MPI={mpi_env.get('AMSS_CUDA_AWARE_MPI', '')}")
+        print(f"   AMSS_CUDA_KEEP_RESIDENT_AFTER_STEP={mpi_env.get('AMSS_CUDA_KEEP_RESIDENT_AFTER_STEP', '')}")
+        print(f"   AMSS_CUDA_Z4C_KEEP_RESIDENT_AFTER_STEP={mpi_env.get('AMSS_CUDA_Z4C_KEEP_RESIDENT_AFTER_STEP', '')}")
+        print(f"   AMSS_CUDA_KEEP_ALL_LEVELS={mpi_env.get('AMSS_CUDA_KEEP_ALL_LEVELS', '')}")
+        print(f"   AMSS_CUDA_Z4C_AMR_DEVICE={mpi_env.get('AMSS_CUDA_Z4C_AMR_DEVICE', '')}")
+        print(f"   AMSS_CUDA_AMR_RESTRICT_DEVICE={mpi_env.get('AMSS_CUDA_AMR_RESTRICT_DEVICE', '')}")
+        print(f"   AMSS_CUDA_AMR_RESTRICT_BATCH={mpi_env.get('AMSS_CUDA_AMR_RESTRICT_BATCH', '')}")
+        print(f"   AMSS_CUDA_DEVICE_SEGMENT_BATCH={mpi_env.get('AMSS_CUDA_DEVICE_SEGMENT_BATCH', '')}")
+        print(f"   AMSS_CUDA_PIN_ESCALAR_TRANSFERS={mpi_env.get('AMSS_CUDA_PIN_ESCALAR_TRANSFERS', '')}")
+        print(f"   AMSS_ESCALAR_GPU_RK={mpi_env.get('AMSS_ESCALAR_GPU_RK', '')}")
+        if "CUDA_MPS_PIPE_DIRECTORY" in mpi_env:
+            print(f"   CUDA_MPS_PIPE_DIRECTORY={mpi_env['CUDA_MPS_PIPE_DIRECTORY']}")
  
-    ## Execute the MPI command and stream output
+    try:
-    mpi_process = subprocess.Popen(mpi_command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)
+        ## Execute the MPI command and stream output
+        mpi_process = subprocess.Popen(mpi_command, shell=True, stdout=subprocess.PIPE,
+                                       stderr=subprocess.STDOUT, text=True, env=mpi_env)
 
-    ## Write ABE run output to file while printing to stdout
+        ## Write ABE run output to file while printing to stdout
-    with open(mpi_command_outfile, 'w') as file0:  
+        with open(mpi_command_outfile, 'w') as file0:
-        ## Read and print output lines; also write each line to file
+            ## Read and print output lines; also write each line to file
-        for line in mpi_process.stdout:
+            for line in mpi_process.stdout:
-            print(line, end='')  # stream output in real time
+                print(line, end='')  # stream output in real time
-            file0.write(line)    # write the line to file
+                file0.write(line)    # write the line to file
-            file0.flush()        # flush to ensure each line is written immediately (optional)            
+                file0.flush()        # flush to ensure each line is written immediately (optional)
-    file0.close()
 
-    ## Wait for the process to finish
+        ## Wait for the process to finish
-    mpi_return_code = mpi_process.wait()
+        mpi_return_code = mpi_process.wait()
+        if mpi_return_code != 0:
+            raise subprocess.CalledProcessError(mpi_return_code, mpi_command)
+    finally:
+        if started_mps:
+            _stop_cuda_mps(mpi_env)
     
     print(                                           )
     print( " The ABE/ABEGPU simulation is finished " )