Add opt-in BSSN CUDA resident AMR path

AMSS_NCKU_source/bssn_class.C

@@ -271,6 +271,24 @@ bool bssn_cuda_use_resident_sync(int lev)
 #endif
 }
 
+bool bssn_cuda_keep_resident_after_step(int lev, int trfls_in, int analysis_lev)
+{
+  static int enabled = -1;
+  if (enabled < 0)
+  {
+    const char *env = getenv("AMSS_CUDA_KEEP_RESIDENT_AFTER_STEP");
+    enabled = (env && atoi(env) != 0) ? 1 : 0;
+  }
+  if (!enabled)
+    return false;
+  // Levels at and above trfls can be read by CPU time interpolation through
+  // State/Old/corrector lists. Keep those conservative until multi-time-level
+  // resident storage is implemented.
+  if (lev == analysis_lev)
+    return false;
+  return lev < trfls_in;
+}
+
 bool bssn_constraint_recompute_from_state(int lev, bool level0_cache_valid)
 {
 #if USE_CUDA_BSSN
@@ -4131,6 +4149,12 @@ void bssn_class::Step(int lev, int YN)
   // Warning NOTE: the variables1 are used as temp storege room
   if (lev == a_lev)
   {
+#if USE_CUDA_BSSN
+    const bool need_analysis_state_before_predictor =
+        (LastAnas + dT_lev >= AnasTime);
+    if (use_cuda_resident_sync && need_analysis_state_before_predictor)
+      bssn_cuda_download_level_state(GH->PatL[lev], StateList, myrank, false);
+#endif
     AnalysisStuff(lev, dT_lev);
   }
 #endif
@@ -4623,7 +4647,10 @@ void bssn_class::Step(int lev, int YN)
   }
 #if USE_CUDA_BSSN
   if (use_cuda_resident_sync)
-    bssn_cuda_download_level_state(GH->PatL[lev], SynchList_cor, myrank, true);
+  {
+    if (!bssn_cuda_keep_resident_after_step(lev, trfls, a_lev))
+      bssn_cuda_download_level_state(GH->PatL[lev], SynchList_cor, myrank, true);
+  }
 #endif
 #if (RPS == 0)
   // mesh refinement boundary part