Stabilize EScalar CUDA fallback path

Rewritten done by Codex.
This still has errors, do not pick this one now.

.gitignore

@@ -0,0 +1,6 @@
+__pycache__
+GW150914
+GW150914*
+docs
+*.tmp
+.codex

AMSS_NCKU_Input.py

@@ -16,9 +16,9 @@ import numpy
 File_directory   = "GW150914"                    ## output file directory
 Output_directory = "binary_output"               ## binary data file directory
                                                  ## The file directory name should not be too long
-MPI_processes    = 64                             ## number of mpi processes used in the simulation
+MPI_processes    = 2                             ## number of mpi processes used in the simulation
 
-GPU_Calculation  = "no"                          ## Use GPU or not 
+GPU_Calculation  = "yes"                          ## Use GPU or not 
                                                  ## (prefer "no" in the current version, because the GPU part may have bugs when integrated in this Python interface)
 CPU_Part         = 1.0
 GPU_Part         = 0.0
@@ -31,7 +31,7 @@ GPU_Part         = 0.0
 ## Setting the physical system and numerical method
 
 Symmetry                 = "equatorial-symmetry"   ## Symmetry of System: choose equatorial-symmetry、no-symmetry、octant-symmetry
-Equation_Class           = "BSSN"                  ## Evolution Equation: choose "BSSN", "BSSN-EScalar", "BSSN-EM", "Z4C" 
+Equation_Class           = "BSSN-EScalar"                  ## Evolution Equation: choose "BSSN", "BSSN-EScalar", "BSSN-EM", "Z4C"
                                                    ## If "BSSN-EScalar" is chosen, it is necessary to set other parameters below
 Initial_Data_Method      = "Ansorg-TwoPuncture"    ## initial data method: choose "Ansorg-TwoPuncture", "Lousto-Analytical", "Cao-Analytical", "KerrSchild-Analytical"
 Time_Evolution_Method    = "runge-kutta-45"        ## time evolution method: choose "runge-kutta-45"

AMSS_NCKU_Program.py

@@ -8,6 +8,14 @@
 ##
 ##################################################################
 
+## Guard against re-execution by multiprocessing child processes.
+## Without this, using 'spawn' or 'forkserver' context would cause every
+## worker to re-run the entire script, spawning exponentially more
+## workers (fork bomb).
+if __name__ != '__main__':
+    import sys as _sys
+    _sys.exit(0)
+
 
 ##################################################################
 
@@ -50,31 +58,36 @@ File_directory = os.path.join(input_data.File_directory)
 
 ## If the specified output directory exists, ask the user whether to continue
 if os.path.exists(File_directory):
-    print( " Output dictionary has been existed !!!  "                                                              )
-    print( " If you want to overwrite the existing file directory, please input 'continue' in the terminal !! "     ) 
-    print( " If you want to retain the existing file directory, please input 'stop' in the terminal to stop the "   ) 
-    print( " simulation. Then you can reset the output dictionary in the input script file AMSS_NCKU_Input.py !!! " )
-    print(                                                                                                          )
-    ## Prompt whether to overwrite the existing directory
-    while True:
-        try:
-            inputvalue = input()
-            ## If the user agrees to overwrite, proceed and remove the existing directory
-            if ( inputvalue == "continue" ):
-                print( " Continue the calculation !!! " )
-                print(                                  )
-                break  
-            ## If the user chooses not to overwrite, exit and keep the existing directory
-            elif ( inputvalue == "stop" ):
-                print( " Stop the calculation !!! "    )
-                sys.exit() 
-            ## If the user input is invalid, prompt again
-            else:
+    auto_overwrite = str(getattr(input_data, "Auto_Overwrite_Output", "yes")).strip().lower()
+    if auto_overwrite in ("1", "yes", "y", "true", "on", "continue"):
+        print( " Output dictionary has been existed; Auto_Overwrite_Output=yes, continue the calculation. " )
+        print(                                                                                              )
+    else:
+        print( " Output dictionary has been existed !!!  "                                                              )
+        print( " If you want to overwrite the existing file directory, please input 'continue' in the terminal !! "     ) 
+        print( " If you want to retain the existing file directory, please input 'stop' in the terminal to stop the "   ) 
+        print( " simulation. Then you can reset the output dictionary in the input script file AMSS_NCKU_Input.py !!! " )
+        print(                                                                                                          )
+        ## Prompt whether to overwrite the existing directory
+        while True:
+            try:
+                inputvalue = input()
+                ## If the user agrees to overwrite, proceed and remove the existing directory
+                if ( inputvalue == "continue" ):
+                    print( " Continue the calculation !!! " )
+                    print(                                  )
+                    break  
+                ## If the user chooses not to overwrite, exit and keep the existing directory
+                elif ( inputvalue == "stop" ):
+                    print( " Stop the calculation !!! "    )
+                    sys.exit() 
+                ## If the user input is invalid, prompt again
+                else:
+                    print( " Please input your choice !!! "                   )
+                    print( " Input 'continue' or 'stop' in the terminal !!! " )
+            except ValueError:
                 print( " Please input your choice !!! "                   )
                 print( " Input 'continue' or 'stop' in the terminal !!! " )
-        except ValueError:
-            print( " Please input your choice !!! "                   )
-            print( " Input 'continue' or 'stop' in the terminal !!! " )
         
 ## Remove the existing output directory if present
 shutil.rmtree(File_directory, ignore_errors=True)
@@ -250,7 +263,7 @@ print()
 if (input_data.GPU_Calculation == "no"):
     ABE_file = os.path.join(AMSS_NCKU_source_copy, "ABE")
 elif (input_data.GPU_Calculation == "yes"):
-    ABE_file = os.path.join(AMSS_NCKU_source_copy, "ABEGPU")
+    ABE_file = os.path.join(AMSS_NCKU_source_copy, "ABE_CUDA")
 
 if not os.path.exists( ABE_file ):
     print(                                                                                                  )
@@ -262,6 +275,12 @@ if not os.path.exists( ABE_file ):
 ## Copy the executable ABE (or ABEGPU) into the run directory
 shutil.copy2(ABE_file, output_directory)
 
+## Copy interp load balance profile if present (for optimize pass)
+interp_lb_profile = os.path.join(AMSS_NCKU_source_copy, "interp_lb_profile.bin")
+if os.path.exists(interp_lb_profile):
+    shutil.copy2(interp_lb_profile, output_directory)
+    print( " Copied interp_lb_profile.bin to run directory " )
+
 ###########################
 
 ## If the initial-data method is TwoPuncture, copy the TwoPunctureABE executable to the run directory
@@ -424,26 +443,31 @@ print(
 
 import plot_xiaoqu
 import plot_GW_strain_amplitude_xiaoqu
+from parallel_plot_helper import run_plot_tasks_parallel
+
+plot_tasks = []
 
 ## Plot black hole trajectory
-plot_xiaoqu.generate_puncture_orbit_plot(   binary_results_directory, figure_directory )
-plot_xiaoqu.generate_puncture_orbit_plot3D( binary_results_directory, figure_directory )
+plot_tasks.append( ( plot_xiaoqu.generate_puncture_orbit_plot,   (binary_results_directory, figure_directory) ) )
+plot_tasks.append( ( plot_xiaoqu.generate_puncture_orbit_plot3D, (binary_results_directory, figure_directory) ) )
 
 ## Plot black hole separation vs. time
-plot_xiaoqu.generate_puncture_distence_plot( binary_results_directory, figure_directory )
+plot_tasks.append( ( plot_xiaoqu.generate_puncture_distence_plot, (binary_results_directory, figure_directory) ) )
 
 ## Plot gravitational waveforms (psi4 and strain amplitude)
 for i in range(input_data.Detector_Number):
-    plot_xiaoqu.generate_gravitational_wave_psi4_plot( binary_results_directory, figure_directory, i )
-    plot_GW_strain_amplitude_xiaoqu.generate_gravitational_wave_amplitude_plot( binary_results_directory, figure_directory, i )
+    plot_tasks.append( ( plot_xiaoqu.generate_gravitational_wave_psi4_plot, (binary_results_directory, figure_directory, i) ) )
+    plot_tasks.append( ( plot_GW_strain_amplitude_xiaoqu.generate_gravitational_wave_amplitude_plot, (binary_results_directory, figure_directory, i) ) )
 
 ## Plot ADM mass evolution
 for i in range(input_data.Detector_Number):
-    plot_xiaoqu.generate_ADMmass_plot( binary_results_directory, figure_directory, i )
+    plot_tasks.append( ( plot_xiaoqu.generate_ADMmass_plot, (binary_results_directory, figure_directory, i) ) )
 
 ## Plot Hamiltonian constraint violation over time
 for i in range(input_data.grid_level):
-    plot_xiaoqu.generate_constraint_check_plot( binary_results_directory, figure_directory, i )
+    plot_tasks.append( ( plot_xiaoqu.generate_constraint_check_plot, (binary_results_directory, figure_directory, i) ) )
+
+run_plot_tasks_parallel(plot_tasks)
 
 ## Plot stored binary data
 plot_xiaoqu.generate_binary_data_plot( binary_results_directory, figure_directory )

AMSS_NCKU_Verify_ASC26.py

@@ -0,0 +1,422 @@
+#!/usr/bin/env python3
+"""
+AMSS-NCKU GW150914 Simulation Regression Test Script (Comprehensive Version)
+
+Verification Requirements:
+1. RMS errors < 1% for:
+   - 3D Vector Total RMS
+   - X Component RMS
+   - Y Component RMS
+   - Z Component RMS
+2. ADM constraint violation < 2 (Grid Level 0)
+3. The following figure PDFs must match GW150914-origin exactly after rasterization:
+   - ADM_Constraint_Grid_Level_0.pdf
+   - BH_Trajectory_21_XY.pdf
+   - BH_Trajectory_XY.pdf
+   The script also reports the percentage of differing pixels for each figure.
+
+RMS Calculation Method:
+- Computes trajectory deviation on the XY plane independently for BH1 and BH2
+- For each black hole: RMS = sqrt((1/M) * sum((Δr_i / r_i^max)^2)) × 100%
+- Final RMS = max(RMS_BH1, RMS_BH2)
+
+Usage: python3 AMSS_NCKU_Verify_ASC26.py [output_dir]
+Default: output_dir = GW150914/AMSS_NCKU_output
+Reference: GW150914-origin (baseline simulation)
+"""
+
+import numpy as np
+import sys
+import os
+import shutil
+import subprocess
+import tempfile
+from PIL import Image
+
+# ANSI Color Codes
+class Color:
+    GREEN = '\033[92m'
+    RED = '\033[91m'
+    YELLOW = '\033[93m'
+    BLUE = '\033[94m'
+    BOLD = '\033[1m'
+    RESET = '\033[0m'
+
+def get_status_text(passed):
+    if passed:
+        return f"{Color.GREEN}{Color.BOLD}PASS{Color.RESET}"
+    else:
+        return f"{Color.RED}{Color.BOLD}FAIL{Color.RESET}"
+
+def load_bh_trajectory(filepath):
+    """Load black hole trajectory data"""
+    data = np.loadtxt(filepath)
+    return {
+        'time': data[:, 0],
+        'x1': data[:, 1], 'y1': data[:, 2], 'z1': data[:, 3],
+        'x2': data[:, 4], 'y2': data[:, 5], 'z2': data[:, 6]
+    }
+
+
+def load_constraint_data(filepath):
+    """Load constraint violation data"""
+    data = []
+    with open(filepath, 'r') as f:
+        for line in f:
+            if line.startswith('#'):
+                continue
+            parts = line.split()
+            if len(parts) >= 8:
+                data.append([float(x) for x in parts[:8]])
+    return np.array(data)
+
+
+def resolve_figure_dir(path):
+    """Resolve the sibling figure directory from an output or figure path."""
+    normalized = os.path.normpath(path)
+    if os.path.basename(normalized) == "figure":
+        return normalized
+    return os.path.join(os.path.dirname(normalized), "figure")
+
+
+def render_pdf_to_images(pdf_path, dpi=150):
+    """Render a PDF to RGB images using Ghostscript."""
+    gs_path = shutil.which("gs")
+    if gs_path is None:
+        raise RuntimeError("Ghostscript executable 'gs' was not found in PATH")
+
+    with tempfile.TemporaryDirectory(prefix="amss_verify_pdf_") as temp_dir:
+        output_pattern = os.path.join(temp_dir, "page-%03d.ppm")
+        cmd = [
+            gs_path,
+            "-q",
+            "-dSAFER",
+            "-dBATCH",
+            "-dNOPAUSE",
+            "-sDEVICE=ppmraw",
+            f"-r{dpi}",
+            f"-o{output_pattern}",
+            pdf_path
+        ]
+
+        try:
+            subprocess.run(cmd, check=True, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, text=True)
+        except subprocess.CalledProcessError as exc:
+            message = exc.stderr.strip() or str(exc)
+            raise RuntimeError(f"Failed to render PDF '{pdf_path}': {message}") from exc
+
+        ppm_files = sorted(
+            os.path.join(temp_dir, filename)
+            for filename in os.listdir(temp_dir)
+            if filename.endswith(".ppm")
+        )
+
+        if not ppm_files:
+            raise RuntimeError(f"No rendered pages were produced for '{pdf_path}'")
+
+        images = []
+        for ppm_file in ppm_files:
+            with Image.open(ppm_file) as img:
+                images.append(np.array(img.convert("RGB"), dtype=np.uint8))
+
+        return images
+
+
+def compare_rendered_pages(ref_img, target_img):
+    """Return (different_pixels, total_pixels) for two rendered RGB pages."""
+    ref_h, ref_w = ref_img.shape[:2]
+    tgt_h, tgt_w = target_img.shape[:2]
+    total_pixels = max(ref_h, tgt_h) * max(ref_w, tgt_w)
+
+    if ref_h == tgt_h and ref_w == tgt_w:
+        different_pixels = int(np.count_nonzero(np.any(ref_img != target_img, axis=2)))
+        return different_pixels, total_pixels
+
+    diff_mask = np.ones((max(ref_h, tgt_h), max(ref_w, tgt_w)), dtype=bool)
+    overlap_h = min(ref_h, tgt_h)
+    overlap_w = min(ref_w, tgt_w)
+    overlap_diff = np.any(ref_img[:overlap_h, :overlap_w] != target_img[:overlap_h, :overlap_w], axis=2)
+    diff_mask[:overlap_h, :overlap_w] = overlap_diff
+    different_pixels = int(np.count_nonzero(diff_mask))
+    return different_pixels, total_pixels
+
+
+def compare_pdf_images(ref_pdf, target_pdf, dpi=150, threshold_percent=0.001):
+    """Compare two PDFs by rasterizing them and counting differing pixels."""
+    ref_pages = render_pdf_to_images(ref_pdf, dpi=dpi)
+    target_pages = render_pdf_to_images(target_pdf, dpi=dpi)
+
+    total_pixels = 0
+    different_pixels = 0
+    max_pages = max(len(ref_pages), len(target_pages))
+
+    for page_idx in range(max_pages):
+        if page_idx < len(ref_pages) and page_idx < len(target_pages):
+            page_diff, page_total = compare_rendered_pages(ref_pages[page_idx], target_pages[page_idx])
+        else:
+            existing_page = ref_pages[page_idx] if page_idx < len(ref_pages) else target_pages[page_idx]
+            page_total = existing_page.shape[0] * existing_page.shape[1]
+            page_diff = page_total
+
+        total_pixels += page_total
+        different_pixels += page_diff
+
+    diff_percent = (different_pixels / total_pixels * 100.0) if total_pixels else 0.0
+    return {
+        "different_pixels": different_pixels,
+        "total_pixels": total_pixels,
+        "diff_percent": diff_percent,
+        "pages_ref": len(ref_pages),
+        "pages_target": len(target_pages),
+        "passed": diff_percent < threshold_percent
+    }
+
+
+def compare_required_figures(reference_figure_dir, target_figure_dir):
+    """Compare the required GW150914 figure PDFs."""
+    figure_names = [
+        "ADM_Constraint_Grid_Level_0.pdf",
+        "BH_Trajectory_21_XY.pdf",
+        "BH_Trajectory_XY.pdf"
+    ]
+
+    results = []
+    for figure_name in figure_names:
+        ref_pdf = os.path.join(reference_figure_dir, figure_name)
+        target_pdf = os.path.join(target_figure_dir, figure_name)
+
+        if not os.path.exists(ref_pdf):
+            raise FileNotFoundError(f"Reference figure not found: {ref_pdf}")
+        if not os.path.exists(target_pdf):
+            raise FileNotFoundError(f"Target figure not found: {target_pdf}")
+
+        comparison = compare_pdf_images(ref_pdf, target_pdf)
+        comparison["name"] = figure_name
+        results.append(comparison)
+
+    return results
+
+def calculate_all_rms_errors(bh_data_ref, bh_data_target):
+    """
+    Calculate 3D Vector RMS and component-wise RMS (X, Y, Z) independently.
+    Uses r = sqrt(x^2 + y^2) as the denominator for all error normalizations.
+    Returns the maximum error between BH1 and BH2 for each category.
+    """
+    M = min(len(bh_data_ref['time']), len(bh_data_target['time']))
+
+    if M < 10:
+        return None, "Insufficient data points for comparison"
+
+    results = {}
+
+    for bh in ['1', '2']:
+        x_r, y_r, z_r = bh_data_ref[f'x{bh}'][:M], bh_data_ref[f'y{bh}'][:M], bh_data_ref[f'z{bh}'][:M]
+        x_n, y_n, z_n = bh_data_target[f'x{bh}'][:M], bh_data_target[f'y{bh}'][:M], bh_data_target[f'z{bh}'][:M]
+
+        # 核心修改：根据组委会的邮件指示，分母统一使用 r = sqrt(x^2 + y^2)
+        r_ref = np.sqrt(x_r**2 + y_r**2)
+        r_new = np.sqrt(x_n**2 + y_n**2)
+        denom_max = np.maximum(r_ref, r_new)
+
+        valid = denom_max > 1e-15
+        if np.sum(valid) < 10:
+            results[f'BH{bh}'] = { '3D_Vector': 0.0, 'X_Component': 0.0, 'Y_Component': 0.0, 'Z_Component': 0.0 }
+            continue
+
+        def calc_rms(delta):
+            # 将对应分量的偏差除以统一的轨道半径分母 denom_max
+            return np.sqrt(np.mean((delta[valid] / denom_max[valid])**2)) * 100
+
+        # 1. Total 3D Vector RMS
+        delta_vec = np.sqrt((x_r - x_n)**2 + (y_r - y_n)**2 + (z_r - z_n)**2)
+        rms_3d = calc_rms(delta_vec)
+
+        # 2. Component-wise RMS (分离计算各轴，但共用半径分母)
+        rms_x = calc_rms(np.abs(x_r - x_n))
+        rms_y = calc_rms(np.abs(y_r - y_n))
+        rms_z = calc_rms(np.abs(z_r - z_n))
+
+        results[f'BH{bh}'] = {
+            '3D_Vector': rms_3d,
+            'X_Component': rms_x,
+            'Y_Component': rms_y,
+            'Z_Component': rms_z
+        }
+
+    # 获取 BH1 和 BH2 中的最大误差
+    max_rms = {
+        '3D_Vector': max(results['BH1']['3D_Vector'], results['BH2']['3D_Vector']),
+        'X_Component': max(results['BH1']['X_Component'], results['BH2']['X_Component']),
+        'Y_Component': max(results['BH1']['Y_Component'], results['BH2']['Y_Component']),
+        'Z_Component': max(results['BH1']['Z_Component'], results['BH2']['Z_Component'])
+    }
+
+    return max_rms, None
+
+def analyze_constraint_violation(constraint_data, n_levels=9):
+    """
+    Analyze ADM constraint violation
+    Return maximum constraint violation for Grid Level 0
+    """
+    # Extract Grid Level 0 data (first entry for each time step)
+    level0_data = constraint_data[::n_levels]
+
+    # Calculate maximum absolute value for each constraint
+    results = {
+        'Ham': np.max(np.abs(level0_data[:, 1])),
+        'Px': np.max(np.abs(level0_data[:, 2])),
+        'Py': np.max(np.abs(level0_data[:, 3])),
+        'Pz': np.max(np.abs(level0_data[:, 4])),
+        'Gx': np.max(np.abs(level0_data[:, 5])),
+        'Gy': np.max(np.abs(level0_data[:, 6])),
+        'Gz': np.max(np.abs(level0_data[:, 7]))
+    }
+
+    results['max_violation'] = max(results.values())
+    return results
+
+
+def print_header():
+    print("\n" + Color.BLUE + Color.BOLD + "=" * 65 + Color.RESET)
+    print(Color.BOLD + "   AMSS-NCKU GW150914 Comprehensive Regression Test" + Color.RESET)
+    print(Color.BLUE + Color.BOLD + "=" * 65 + Color.RESET)
+
+def print_rms_results(rms_dict, error, threshold=1.0):
+    print(f"\n{Color.BOLD}1. RMS Error Analysis (Maximums of BH1 & BH2){Color.RESET}")
+    print("-" * 65)
+
+    if error:
+        print(f"   {Color.RED}Error: {error}{Color.RESET}")
+        return False
+
+    all_passed = True
+    print(f"   Requirement: < {threshold}%\n")
+
+    for key, val in rms_dict.items():
+        passed = val < threshold
+        all_passed = all_passed and passed
+        status = get_status_text(passed)
+        print(f"   {key:15}: {val:8.4f}%   |   Status: {status}")
+
+    return all_passed
+
+def print_constraint_results(results, threshold=2.0):
+    print(f"\n{Color.BOLD}2. ADM Constraint Violation Analysis (Grid Level 0){Color.RESET}")
+    print("-" * 65)
+
+    names = ['Ham', 'Px', 'Py', 'Pz', 'Gx', 'Gy', 'Gz']
+    for i, name in enumerate(names):
+        print(f"   Max |{name:3}|: {results[name]:.6f}", end="   ")
+        if (i + 1) % 2 == 0: print()
+    if len(names) % 2 != 0: print()
+
+    passed = results['max_violation'] < threshold
+    
+    print(f"\n   Maximum violation:  {results['max_violation']:.6f}")
+    print(f"   Requirement:        < {threshold}")
+    print(f"   Status:             {get_status_text(passed)}")
+
+    return passed
+
+
+def print_figure_results(results, threshold_percent=0.001):
+    print(f"\n{Color.BOLD}3. Figure Pixel Comparison (PDF Rasterization){Color.RESET}")
+    print("-" * 65)
+    print(f"   Requirement: < {threshold_percent:.3f}% differing pixels\n")
+
+    all_passed = True
+    for result in results:
+        passed = result["passed"]
+        all_passed = all_passed and passed
+        status = get_status_text(passed)
+        print(f"   {result['name']:32}: {result['diff_percent']:10.6f}%   |   Status: {status}")
+
+        if result["pages_ref"] != result["pages_target"]:
+            print(f"   {'':32}  pages(ref/target): {result['pages_ref']}/{result['pages_target']}")
+
+    return all_passed
+
+
+def print_figure_error(error_message):
+    print(f"\n{Color.BOLD}3. Figure Pixel Comparison (PDF Rasterization){Color.RESET}")
+    print("-" * 65)
+    print(f"   {Color.RED}Error: {error_message}{Color.RESET}")
+    return False
+
+
+def print_summary(rms_passed, constraint_passed, figure_passed):
+    print("\n" + Color.BLUE + Color.BOLD + "=" * 65 + Color.RESET)
+    print(Color.BOLD + "Verification Summary" + Color.RESET)
+    print(Color.BLUE + Color.BOLD + "=" * 65 + Color.RESET)
+
+    all_passed = rms_passed and constraint_passed and figure_passed
+    
+    res_rms = get_status_text(rms_passed)
+    res_con = get_status_text(constraint_passed)
+    res_fig = get_status_text(figure_passed)
+
+    print(f"   [1] Comprehensive RMS check:      {res_rms}")
+    print(f"   [2] ADM constraint check:         {res_con}")
+    print(f"   [3] Figure pixel comparison:      {res_fig}")
+    
+    final_status = f"{Color.GREEN}{Color.BOLD}ALL CHECKS PASSED{Color.RESET}" if all_passed else f"{Color.RED}{Color.BOLD}SOME CHECKS FAILED{Color.RESET}"
+    print(f"\n   Overall result: {final_status}")
+    print(Color.BLUE + Color.BOLD + "=" * 65 + Color.RESET + "\n")
+
+    return all_passed
+
+def main():
+    if len(sys.argv) > 1:
+        target_dir = sys.argv[1]
+    else:
+        script_dir = os.path.dirname(os.path.abspath(__file__))
+        target_dir = os.path.join(script_dir, "GW150914/AMSS_NCKU_output")
+
+    script_dir = os.path.dirname(os.path.abspath(__file__))
+    reference_dir = os.path.join(script_dir, "GW150914-origin/AMSS_NCKU_output")
+    target_figure_dir = resolve_figure_dir(target_dir)
+    reference_figure_dir = os.path.join(script_dir, "GW150914-origin/figure")
+
+    bh_file_ref = os.path.join(reference_dir, "bssn_BH.dat")
+    bh_file_target = os.path.join(target_dir, "bssn_BH.dat")
+    constraint_file = os.path.join(target_dir, "bssn_constraint.dat")
+
+    if not os.path.exists(bh_file_ref):
+        print(f"{Color.RED}{Color.BOLD}Error:{Color.RESET} Baseline trajectory file not found: {bh_file_ref}")
+        sys.exit(1)
+    if not os.path.exists(bh_file_target):
+        print(f"{Color.RED}{Color.BOLD}Error:{Color.RESET} Target trajectory file not found: {bh_file_target}")
+        sys.exit(1)
+    if not os.path.exists(constraint_file):
+        print(f"{Color.RED}{Color.BOLD}Error:{Color.RESET} Constraint data file not found: {constraint_file}")
+        sys.exit(1)
+
+    print_header()
+    print(f"\n{Color.BOLD}Reference (Baseline):{Color.RESET} {Color.BLUE}{reference_dir}{Color.RESET}")
+    print(f"{Color.BOLD}Target (Optimized):  {Color.RESET} {Color.BLUE}{target_dir}{Color.RESET}")
+    print(f"{Color.BOLD}Reference Figures:   {Color.RESET} {Color.BLUE}{reference_figure_dir}{Color.RESET}")
+    print(f"{Color.BOLD}Target Figures:      {Color.RESET} {Color.BLUE}{target_figure_dir}{Color.RESET}")
+
+    bh_data_ref = load_bh_trajectory(bh_file_ref)
+    bh_data_target = load_bh_trajectory(bh_file_target)
+    constraint_data = load_constraint_data(constraint_file)
+
+    # Output modified RMS results
+    rms_dict, error = calculate_all_rms_errors(bh_data_ref, bh_data_target)
+    rms_passed = print_rms_results(rms_dict, error)
+
+    # Output constraint results
+    constraint_results = analyze_constraint_violation(constraint_data)
+    constraint_passed = print_constraint_results(constraint_results)
+
+    try:
+        figure_results = compare_required_figures(reference_figure_dir, target_figure_dir)
+        figure_passed = print_figure_results(figure_results)
+    except (FileNotFoundError, RuntimeError) as exc:
+        figure_passed = print_figure_error(str(exc))
+
+    all_passed = print_summary(rms_passed, constraint_passed, figure_passed)
+    sys.exit(0 if all_passed else 1)
+
+if __name__ == "__main__":
+    main()

AMSS_NCKU_source/Block.C

@@ -6,14 +6,68 @@
 #include <cstdio>
 #include <string>
 #include <cmath>
-#include <new>
-using namespace std;
-
-#include "Block.h"
-#include "misc.h"
-
-Block::Block(int DIM, int *shapei, double *bboxi, int ranki, int ingfsi, int fngfsi, int levi, const int cgpui) : rank(ranki), ingfs(ingfsi), fngfs(fngfsi), lev(levi), cgpu(cgpui)
-{
+#include <new>
+using namespace std;
+
+#include "Block.h"
+#include "misc.h"
+
+#if USE_CUDA_BSSN || USE_CUDA_Z4C
+#include <cuda_runtime_api.h>
+#endif
+
+namespace {
+
+bool cuda_pin_gridfuncs_enabled()
+{
+  static int enabled = -1;
+  if (enabled < 0)
+  {
+    const char *env = getenv("AMSS_CUDA_PIN_GRIDFUNCS");
+    enabled = (env && atoi(env) != 0) ? 1 : 0;
+  }
+  return enabled != 0;
+}
+
+double *alloc_gridfunc(size_t count, unsigned char &pinned)
+{
+  pinned = 0;
+#if USE_CUDA_BSSN || USE_CUDA_Z4C
+  if (cuda_pin_gridfuncs_enabled())
+  {
+    double *ptr = 0;
+    cudaError_t err = cudaMallocHost((void **)&ptr, count * sizeof(double));
+    if (err == cudaSuccess)
+    {
+      pinned = 1;
+      return ptr;
+    }
+    cudaGetLastError();
+  }
+#endif
+  return (double *)malloc(sizeof(double) * count);
+}
+
+void free_gridfunc(double *ptr, unsigned char pinned)
+{
+  if (!ptr)
+    return;
+#if USE_CUDA_BSSN || USE_CUDA_Z4C
+  if (pinned)
+  {
+    cudaFreeHost(ptr);
+    return;
+  }
+#else
+  (void)pinned;
+#endif
+  free(ptr);
+}
+
+}
+
+Block::Block(int DIM, int *shapei, double *bboxi, int ranki, int ingfsi, int fngfsi, int levi, const int cgpui) : rank(ranki), lev(levi), cgpu(cgpui), ingfs(ingfsi), fngfs(fngfsi), igfs(0), fgfs(0), fgfs_pinned(0)
+{
   for (int i = 0; i < dim; i++)
     X[i] = 0;
 
@@ -68,14 +122,15 @@ Block::Block(int DIM, int *shapei, double *bboxi, int ranki, int ingfsi, int fng
 #endif
     }
 
-    int nn = shape[0] * shape[1] * shape[2];
-    fgfs = new double *[fngfs];
-    for (int i = 0; i < fngfs; i++)
-    {
-      fgfs[i] = (double *)malloc(sizeof(double) * nn);
-      if (!(fgfs[i]))
-      {
-        cout << "on node#" << rank << ", out of memory when constructing Block." << endl;
+    int nn = shape[0] * shape[1] * shape[2];
+    fgfs = new double *[fngfs];
+    fgfs_pinned = new unsigned char[fngfs];
+    for (int i = 0; i < fngfs; i++)
+    {
+      fgfs[i] = alloc_gridfunc((size_t)nn, fgfs_pinned[i]);
+      if (!(fgfs[i]))
+      {
+        cout << "on node#" << rank << ", out of memory when constructing Block." << endl;
         MPI_Abort(MPI_COMM_WORLD, 1);
       }
       memset(fgfs[i], 0, sizeof(double) * nn);
@@ -103,17 +158,19 @@ Block::~Block()
   {
     for (int i = 0; i < dim; i++)
       delete[] X[i];
-    for (int i = 0; i < ingfs; i++)
-      free(igfs[i]);
-    delete[] igfs;
-    for (int i = 0; i < fngfs; i++)
-      free(fgfs[i]);
-    delete[] fgfs;
-    X[0] = X[1] = X[2] = 0;
-    igfs = 0;
-    fgfs = 0;
-  }
-}
+    for (int i = 0; i < ingfs; i++)
+      free(igfs[i]);
+    delete[] igfs;
+    for (int i = 0; i < fngfs; i++)
+      free_gridfunc(fgfs[i], fgfs_pinned ? fgfs_pinned[i] : 0);
+    delete[] fgfs;
+    delete[] fgfs_pinned;
+    X[0] = X[1] = X[2] = 0;
+    igfs = 0;
+    fgfs = 0;
+    fgfs_pinned = 0;
+  }
+}
 void Block::checkBlock()
 {
   int myrank;
@@ -184,12 +241,14 @@ void Block::swapList(MyList<var> *VarList1, MyList<var> *VarList2, int myrank)
   if (rank == myrank)
   {
     MyList<var> *varl1 = VarList1, *varl2 = VarList2;
-    while (varl1 && varl2)
-    {
-      misc::swap<double *>(fgfs[varl1->data->sgfn], fgfs[varl2->data->sgfn]);
-      varl1 = varl1->next;
-      varl2 = varl2->next;
-    }
+    while (varl1 && varl2)
+    {
+      misc::swap<double *>(fgfs[varl1->data->sgfn], fgfs[varl2->data->sgfn]);
+      if (fgfs_pinned)
+        misc::swap<unsigned char>(fgfs_pinned[varl1->data->sgfn], fgfs_pinned[varl2->data->sgfn]);
+      varl1 = varl1->next;
+      varl2 = varl2->next;
+    }
     if (varl1 || varl2)
     {
       cout << "error in Block::swaplist, var lists does not match." << endl;

AMSS_NCKU_source/Block.h

@@ -13,14 +13,15 @@ public:
    int shape[dim];
    double bbox[2 * dim];
    double *X[dim];
-   int rank; // where the real data locate in
-   int lev, cgpu;
-   int ingfs, fngfs;
-   int *(*igfs);
-   double *(*fgfs);
+   int rank; // where the real data locate in
+   int lev, cgpu;
+   int ingfs, fngfs;
+   int *(*igfs);
+   double *(*fgfs);
+   unsigned char *fgfs_pinned;
 
 public:
-   Block() {};
+   Block() : rank(0), lev(0), cgpu(0), ingfs(0), fngfs(0), igfs(0), fgfs(0), fgfs_pinned(0) {};
    Block(int DIM, int *shapei, double *bboxi, int ranki, int ingfsi, int fngfs, int levi, const int cgpui = 0);
 
    ~Block();

AMSS_NCKU_source/FFT.f90

@@ -37,57 +37,51 @@ close(77)
 end program checkFFT
 #endif
 
+!-------------
+! Optimized FFT using Intel oneMKL DFTI
+! Mathematical equivalence: Standard DFT definition
+!   Forward (isign=1):  X[k] = sum_{n=0}^{N-1} x[n] * exp(-2*pi*i*k*n/N)
+!   Backward (isign=-1): X[k] = sum_{n=0}^{N-1} x[n] * exp(+2*pi*i*k*n/N)
+! Input/Output: dataa is interleaved complex array [Re(0),Im(0),Re(1),Im(1),...]
 !-------------
 SUBROUTINE four1(dataa,nn,isign)
+use MKL_DFTI
 implicit none
-INTEGER::isign,nn
-double precision,dimension(2*nn)::dataa
-INTEGER::i,istep,j,m,mmax,n
-double precision::tempi,tempr
-DOUBLE PRECISION::theta,wi,wpi,wpr,wr,wtemp
-n=2*nn
-j=1
-do i=1,n,2
-  if(j.gt.i)then
-     tempr=dataa(j)
-     tempi=dataa(j+1)
-     dataa(j)=dataa(i)
-     dataa(j+1)=dataa(i+1)
-     dataa(i)=tempr
-     dataa(i+1)=tempi
-  endif
-  m=nn
-1 if ((m.ge.2).and.(j.gt.m)) then
-  j=j-m
-  m=m/2
-goto 1
-  endif
-j=j+m
-enddo
-mmax=2
-2  if (n.gt.mmax) then
-     istep=2*mmax
-     theta=6.28318530717959d0/(isign*mmax)
-     wpr=-2.d0*sin(0.5d0*theta)**2
-     wpi=sin(theta)
-     wr=1.d0
-     wi=0.d0
-     do m=1,mmax,2
-       do i=m,n,istep
-         j=i+mmax
-         tempr=sngl(wr)*dataa(j)-sngl(wi)*dataa(j+1)
-         tempi=sngl(wr)*dataa(j+1)+sngl(wi)*dataa(j)
-         dataa(j)=dataa(i)-tempr
-         dataa(j+1)=dataa(i+1)-tempi
-         dataa(i)=dataa(i)+tempr
-         dataa(i+1)=dataa(i+1)+tempi
-       enddo
-          wtemp=wr
-          wr=wr*wpr-wi*wpi+wr
-          wi=wi*wpr+wtemp*wpi+wi
-     enddo
-mmax=istep
-goto 2
+INTEGER, intent(in) :: isign, nn
+DOUBLE PRECISION, dimension(2*nn), intent(inout) :: dataa
+
+type(DFTI_DESCRIPTOR), pointer :: desc
+integer :: status
+
+! Create DFTI descriptor for 1D complex-to-complex transform
+status = DftiCreateDescriptor(desc, DFTI_DOUBLE, DFTI_COMPLEX, 1, nn)
+if (status /= 0) return
+
+! Set input/output storage as interleaved complex (default)
+status = DftiSetValue(desc, DFTI_PLACEMENT, DFTI_INPLACE)
+if (status /= 0) then
+   status = DftiFreeDescriptor(desc)
+   return
 endif
+
+! Commit the descriptor
+status = DftiCommitDescriptor(desc)
+if (status /= 0) then
+   status = DftiFreeDescriptor(desc)
+   return
+endif
+
+! Execute FFT based on direction
+if (isign == 1) then
+   ! Forward FFT: exp(-2*pi*i*k*n/N)
+   status = DftiComputeForward(desc, dataa)
+else
+   ! Backward FFT: exp(+2*pi*i*k*n/N)
+   status = DftiComputeBackward(desc, dataa)
+endif
+
+! Free descriptor
+status = DftiFreeDescriptor(desc)
+
 return
 END SUBROUTINE four1

AMSS_NCKU_source/MPatch.h

@@ -39,6 +39,10 @@ public:
 
    bool Find_Point(double *XX);
 
+   void Interp_Points(MyList<var> *VarList,
+                      int NN, double **XX,
+                      double *Shellf, int Symmetry,
+                      int Nmin_consumer, int Nmax_consumer);
    void Interp_Points(MyList<var> *VarList,
                       int NN, double **XX,
                       double *Shellf, int Symmetry, MPI_Comm Comm_here);

AMSS_NCKU_source/Parallel.h

@@ -32,6 +32,16 @@ namespace Parallel
   int partition2(int *nxy, int split_size, int *min_width, int cpusize, int *shape); // special for 2 diemnsions
   int partition3(int *nxyz, int split_size, int *min_width, int cpusize, int *shape);
   MyList<Block> *distribute(MyList<Patch> *PatchLIST, int cpusize, int ingfsi, int fngfs, bool periodic, int nodes = 0); // produce corresponding Blocks
+  MyList<Block> *distribute_optimize(MyList<Patch> *PatchLIST, int cpusize, int ingfsi, int fngfs, bool periodic, int nodes = 0);
+  Block* splitHotspotBlock(MyList<Block>* &BlL, int _dim,
+                           int ib0_orig, int ib3_orig,
+                           int jb1_orig, int jb4_orig,
+                           int kb2_orig, int kb5_orig,
+                           Patch* PP, int r_left, int r_right,
+                           int ingfsi, int fngfsi, bool periodic,
+                           Block* &split_first_block, Block* &split_last_block);
+  Block* createMappedBlock(MyList<Block>* &BlL, int _dim, int* shape, double* bbox,
+                           int block_id, int ingfsi, int fngfsi, int lev);
   void KillBlocks(MyList<Patch> *PatchLIST);
 
   void setfunction(MyList<Block> *BlL, var *vn, double func(double x, double y, double z));
@@ -81,6 +91,59 @@ namespace Parallel
                    int Symmetry);
   void Sync(Patch *Pat, MyList<var> *VarList, int Symmetry);
   void Sync(MyList<Patch> *PatL, MyList<var> *VarList, int Symmetry);
+  void Sync_merged(MyList<Patch> *PatL, MyList<var> *VarList, int Symmetry);
+
+  struct SyncCache {
+    bool valid;
+    int cpusize;
+    MyList<gridseg> **combined_src;
+    MyList<gridseg> **combined_dst;
+    int *send_lengths;
+    int *recv_lengths;
+    double **send_bufs;
+    double **recv_bufs;
+    int *send_buf_caps;
+    int *recv_buf_caps;
+    unsigned char *send_buf_pinned;
+    unsigned char *recv_buf_pinned;
+    unsigned char *send_buf_is_dev;
+    unsigned char *recv_buf_is_dev;
+    int *send_buf_caps_dev;
+    int *recv_buf_caps_dev;
+    double **send_bufs_dev;
+    double **recv_bufs_dev;
+    MPI_Request *reqs;
+    MPI_Status *stats;
+    int max_reqs;
+    bool lengths_valid;
+    int *tc_req_node;
+    int *tc_req_is_recv;
+    int *tc_completed;
+    bool cuda_aware_mode;
+    SyncCache();
+    void invalidate();
+    void destroy();
+  };
+
+  void Sync_cached(MyList<Patch> *PatL, MyList<var> *VarList, int Symmetry, SyncCache &cache);
+  void Sync_ensure_cache(MyList<Patch> *PatL, int Symmetry, SyncCache &cache);
+  void transfer_cached(MyList<gridseg> **src, MyList<gridseg> **dst,
+                       MyList<var> *VarList1, MyList<var> *VarList2,
+                       int Symmetry, SyncCache &cache);
+
+  struct AsyncSyncState {
+    int req_no;
+    bool active;
+    int *req_node;
+    int *req_is_recv;
+    int pending_recv;
+    AsyncSyncState() : req_no(0), active(false), req_node(0), req_is_recv(0), pending_recv(0) {}
+  };
+
+  void Sync_start(MyList<Patch> *PatL, MyList<var> *VarList, int Symmetry,
+                  SyncCache &cache, AsyncSyncState &state);
+  void Sync_finish(SyncCache &cache, AsyncSyncState &state,
+                   MyList<var> *VarList, int Symmetry);
   void OutBdLow2Hi(Patch *Patc, Patch *Patf,
                    MyList<var> *VarList1 /* source */, MyList<var> *VarList2 /* target */,
                    int Symmetry);
@@ -93,6 +156,15 @@ namespace Parallel
   void OutBdLow2Himix(MyList<Patch> *PatcL, MyList<Patch> *PatfL,
                       MyList<var> *VarList1 /* source */, MyList<var> *VarList2 /* target */,
                       int Symmetry);
+  void Restrict_cached(MyList<Patch> *PatcL, MyList<Patch> *PatfL,
+                       MyList<var> *VarList1, MyList<var> *VarList2,
+                       int Symmetry, SyncCache &cache);
+  void OutBdLow2Hi_cached(MyList<Patch> *PatcL, MyList<Patch> *PatfL,
+                          MyList<var> *VarList1, MyList<var> *VarList2,
+                          int Symmetry, SyncCache &cache);
+  void OutBdLow2Himix_cached(MyList<Patch> *PatcL, MyList<Patch> *PatfL,
+                             MyList<var> *VarList1, MyList<var> *VarList2,
+                             int Symmetry, SyncCache &cache);
   void Prolong(Patch *Patc, Patch *Patf,
                MyList<var> *VarList1 /* source */, MyList<var> *VarList2 /* target */,
                int Symmetry);
@@ -121,6 +193,7 @@ namespace Parallel
                          MyList<Parallel::gridseg> **out_src, MyList<Parallel::gridseg> **out_dst);
   void PeriodicBD(Patch *Pat, MyList<var> *VarList, int Symmetry);
   double L2Norm(Patch *Pat, var *vf);
+  void L2Norm7(Patch *Pat, var **vf, double *norms);
   void checkgsl(MyList<Parallel::gridseg> *pp, bool first_only);
   void checkvarl(MyList<var> *pp, bool first_only);
   MyList<Parallel::gridseg> *divide_gsl(MyList<Parallel::gridseg> *p, Patch *Pat);
@@ -156,6 +229,7 @@ namespace Parallel
   void checkpatchlist(MyList<Patch> *PatL, bool buflog);
 
   double L2Norm(Patch *Pat, var *vf, MPI_Comm Comm_here);
+  void L2Norm7(Patch *Pat, var **vf, double *norms, MPI_Comm Comm_here);
   bool PatList_Interp_Points(MyList<Patch> *PatL, MyList<var> *VarList,
                              int NN, double **XX,
                              double *Shellf, int Symmetry, MPI_Comm Comm_here);

AMSS_NCKU_source/ShellPatch.C

@@ -3439,10 +3439,10 @@ void ShellPatch::write_Pablo_file_ss(int *ext, double xmin, double xmax, double
   delete[] Z;
 }
 
-double ShellPatch::L2Norm(var *vf)
-{
-  double tvf, dtvf = 0;
-  int BDW = overghost;
+double ShellPatch::L2Norm(var *vf)
+{
+  double tvf, dtvf = 0;
+  int BDW = overghost;
 
   MyList<ss_patch> *sPp = PatL;
   while (sPp)
@@ -3469,13 +3469,50 @@ double ShellPatch::L2Norm(var *vf)
   MPI_Allreduce(&dtvf, &tvf, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD);
 
   tvf = sqrt(tvf);
-
-  return tvf;
-}
-
-// find maximum of abstract value, XX store position for maximum, Shellf store maximum themselvs
-void ShellPatch::Find_Maximum(MyList<var> *VarList, double *XX,
-                              double *Shellf)
+
+  return tvf;
+}
+void ShellPatch::L2Norm7(var **vf, double *norms)
+{
+  double tvf[7], dtvf[7];
+  int BDW = overghost;
+  for (int i = 0; i < 7; i++)
+    dtvf[i] = 0;
+
+  MyList<ss_patch> *sPp = PatL;
+  while (sPp)
+  {
+    MyList<Block> *Bp = sPp->data->blb;
+    while (Bp)
+    {
+      Block *cg = Bp->data;
+      if (myrank == cg->rank)
+      {
+        f_l2normhelper7(cg->shape, cg->X[0], cg->X[1], cg->X[2],
+                        sPp->data->bbox[0], sPp->data->bbox[1], sPp->data->bbox[2],
+                        sPp->data->bbox[3], sPp->data->bbox[4], sPp->data->bbox[5],
+                        cg->fgfs[vf[0]->sgfn], cg->fgfs[vf[1]->sgfn], cg->fgfs[vf[2]->sgfn],
+                        cg->fgfs[vf[3]->sgfn], cg->fgfs[vf[4]->sgfn], cg->fgfs[vf[5]->sgfn],
+                        cg->fgfs[vf[6]->sgfn], tvf, BDW);
+        for (int i = 0; i < 7; i++)
+          dtvf[i] += tvf[i];
+      }
+      if (Bp == sPp->data->ble)
+        break;
+      Bp = Bp->next;
+    }
+    sPp = sPp->next;
+  }
+
+  MPI_Allreduce(dtvf, tvf, 7, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD);
+
+  for (int i = 0; i < 7; i++)
+    norms[i] = sqrt(tvf[i]);
+}
+
+// find maximum of abstract value, XX store position for maximum, Shellf store maximum themselvs
+void ShellPatch::Find_Maximum(MyList<var> *VarList, double *XX,
+                              double *Shellf)
 {
   MyList<var> *varl;
   int num_var = 0;

AMSS_NCKU_source/ShellPatch.h

@@ -195,10 +195,11 @@ public:
    bool Interp_One_Point(MyList<var> *VarList,
                          double *XX, /*input global Cartesian coordinate*/
                          double *Shellf, int Symmetry);
-   void write_Pablo_file_ss(int *ext, double xmin, double xmax, double ymin, double ymax, double zmin, double zmax,
-                            char *filename, int sst);
-   double L2Norm(var *vf);
-   void Find_Maximum(MyList<var> *VarList, double *XX, double *Shellf);
-};
+   void write_Pablo_file_ss(int *ext, double xmin, double xmax, double ymin, double ymax, double zmin, double zmax,
+                            char *filename, int sst);
+   double L2Norm(var *vf);
+   void L2Norm7(var **vf, double *norms);
+   void Find_Maximum(MyList<var> *VarList, double *XX, double *Shellf);
+};
 
 #endif /* SHELLPATCH_H */

AMSS_NCKU_source/TwoPunctures.h

@@ -1,7 +1,8 @@
-
 #ifndef TWO_PUNCTURES_H
 #define TWO_PUNCTURES_H
 
+#include <omp.h>
+
 #define StencilSize 19
 #define N_PlaneRelax 1
 #define NRELAX 200
@@ -32,7 +33,7 @@ private:
        int npoints_A, npoints_B, npoints_phi;
 
        double target_M_plus, target_M_minus;
-       
+
        double admMass;
 
        double adm_tol;
@@ -42,6 +43,18 @@ private:
 
        int ntotal;
 
+       // ===== Precomputed spectral derivative matrices =====
+       double *D1_A, *D2_A;
+       double *D1_B, *D2_B;
+       double *DF1_phi, *DF2_phi;
+
+       // ===== Pre-allocated workspace for LineRelax (per-thread) =====
+       int max_threads;
+       double **ws_diag_be, **ws_e_be, **ws_f_be, **ws_b_be, **ws_x_be;
+       double **ws_l_be, **ws_u_be, **ws_d_be, **ws_y_be;
+       double **ws_diag_al, **ws_e_al, **ws_f_al, **ws_b_al, **ws_x_al;
+       double **ws_l_al, **ws_u_al, **ws_d_al, **ws_y_al;
+
        struct parameters
        {
               int nvar, n1, n2, n3;
@@ -58,6 +71,28 @@ public:
                     int Newtonmaxit);
        ~TwoPunctures();
 
+       // 02/07: New/modified methods
+       void allocate_workspace();
+       void free_workspace();
+       void precompute_derivative_matrices();
+       void build_cheb_deriv_matrices(int n, double *D1, double *D2);
+       void build_fourier_deriv_matrices(int N, double *DF1, double *DF2);
+       void Derivatives_AB3_MatMul(int nvar, int n1, int n2, int n3, derivs v);
+       void ThomasAlgorithm_ws(int N, double *b, double *a, double *c, double *x, double *q,
+                                double *l, double *u_ws, double *d, double *y);
+       void LineRelax_be_omp(double *dv,
+                         int const i, int const k, int const nvar,
+                         int const n1, int const n2, int const n3,
+                         double const *rhs, int const *ncols, int **cols,
+                         double **JFD, int tid);
+       void LineRelax_al_omp(double *dv,
+                         int const j, int const k, int const nvar,
+                         int const n1, int const n2, int const n3,
+                         double const *rhs, int const *ncols,
+                         int **cols, double **JFD, int tid);
+       void relax_omp(double *dv, int const nvar, int const n1, int const n2, int const n3,
+                  double const *rhs, int const *ncols, int **cols, double **JFD);
+
        void Solve();
        void set_initial_guess(derivs v);
        int index(int i, int j, int k, int l, int a, int b, int c, int d);
@@ -116,23 +151,11 @@ public:
        double BY_KKofxyz(double x, double y, double z);
        void SetMatrix_JFD(int nvar, int n1, int n2, int n3, derivs u, int *ncols, int **cols, double **Matrix);
        void J_times_dv(int nvar, int n1, int n2, int n3, derivs dv, double *Jdv, derivs u);
-       void relax(double *dv, int const nvar, int const n1, int const n2, int const n3,
-                  double const *rhs, int const *ncols, int **cols, double **JFD);
-       void LineRelax_be(double *dv,
-                         int const i, int const k, int const nvar,
-                         int const n1, int const n2, int const n3,
-                         double const *rhs, int const *ncols, int **cols,
-                         double **JFD);
        void JFD_times_dv(int i, int j, int k, int nvar, int n1, int n2,
                          int n3, derivs dv, derivs u, double *values);
        void LinEquations(double A, double B, double X, double R,
                          double x, double r, double phi,
                          double y, double z, derivs dU, derivs U, double *values);
-       void LineRelax_al(double *dv,
-                         int const j, int const k, int const nvar,
-                         int const n1, int const n2, int const n3,
-                         double const *rhs, int const *ncols,
-                         int **cols, double **JFD);
        void ThomasAlgorithm(int N, double *b, double *a, double *c, double *x, double *q);
        void Save(char *fname);
        // provided by Vasileios Paschalidis (vpaschal@illinois.edu)
@@ -141,4 +164,4 @@ public:
        void SpecCoef(parameters par, int ivar, double *v, double *cf);
 };
 
-#endif /* TWO_PUNCTURES_H */
+#endif /* TWO_PUNCTURES_H */

AMSS_NCKU_source/Z4c_class.C

@@ -1,9 +1,10 @@
 
-#ifdef newc
-#include <sstream>
-#include <cstdio>
-#include <map>
-using namespace std;
+#ifdef newc
+#include <sstream>
+#include <cstdio>
+#include <cstdlib>
+#include <map>
+using namespace std;
 #else
 #include <stdio.h>
 #include <map.h>
@@ -26,12 +27,20 @@ using namespace std;
 #include "shellfunctions.h"
 #include "cpbc.h"
 #include "kodiss.h"
-#include "parameters.h"
-
-#ifdef With_AHF
-#include "derivatives.h"
-#include "myglobal.h"
-#endif
+#include "parameters.h"
+
+#ifndef USE_CUDA_Z4C
+#define USE_CUDA_Z4C 0
+#endif
+
+#if USE_CUDA_Z4C && (ABEtype == 2)
+#include "z4c_rhs_cuda.h"
+#endif
+
+#ifdef With_AHF
+#include "derivatives.h"
+#include "myglobal.h"
+#endif
 
 //================================================================================================
 
@@ -105,20 +114,22 @@ void Z4c_class::Initialize()
   else
     GH->compose_cgh(nprocs);
 
-#ifdef WithShell
-  SH = new ShellPatch(0, ngfs, pname, Symmetry, myrank, ErrorMonitor);
-  if (!checkrun)
-    SH->matchcheck(GH->PatL[0]);
+#ifdef WithShell
+  SH = new ShellPatch(0, ngfs, pname, Symmetry, myrank, ErrorMonitor);
+  if (!checkrun)
+    SH->matchcheck(GH->PatL[0]);
   SH->compose_sh(nprocs);
   SH->setupcordtrans();
   SH->Dump_xyz(0, 0, 1);
   SH->setupintintstuff(nprocs, GH->PatL[0], Symmetry);
 
-  if (checkrun)
-    CheckPoint->readcheck_sh(SH, myrank);
-#endif
-
-  double h = GH->PatL[0]->data->blb->data->getdX(0);
+  if (checkrun)
+    CheckPoint->readcheck_sh(SH, myrank);
+#endif
+
+  Initialize_Level_Runtime();
+
+  double h = GH->PatL[0]->data->blb->data->getdX(0);
   for (int i = 1; i < dim; i++)
     h = Mymin(h, GH->PatL[0]->data->blb->data->getdX(i));
   dT = Courant * h;
@@ -167,12 +178,753 @@ Z4c_class::~Z4c_class()
 
 #define MRBD 0 // 0: fix BD for meshrefinement level; 1: sommerfeld_bam for them; 2: sommerfeld_yo for them
 
-#ifndef CPBC
-// for sommerfeld boundary
-
-void Z4c_class::Step(int lev, int YN)
-{
-  double dT_lev = dT * pow(0.5, Mymax(lev, trfls));
+#ifndef CPBC
+// for sommerfeld boundary
+
+#if USE_CUDA_Z4C && (ABEtype == 2)
+#ifdef WithShell
+#error "USE_CUDA_Z4C resident path currently supports Cartesian non-shell Z4C only"
+#endif
+#if (MRBD == 2)
+#error "USE_CUDA_Z4C resident path does not support MRBD == 2"
+#endif
+
+namespace {
+
+static const int k_z4c_cuda_bh_state_indices[3] = {18, 19, 20};
+
+bool fill_z4c_cuda_views(Block *cg, MyList<var> *vars,
+                         double **host_views,
+                         double *propspeeds = 0,
+                         double *soa_flat = 0)
+{
+  int idx = 0;
+  while (vars && idx < Z4C_CUDA_STATE_COUNT)
+  {
+    host_views[idx] = cg->fgfs[vars->data->sgfn];
+    if (propspeeds)
+      propspeeds[idx] = vars->data->propspeed;
+    if (soa_flat)
+    {
+      soa_flat[3 * idx + 0] = vars->data->SoA[0];
+      soa_flat[3 * idx + 1] = vars->data->SoA[1];
+      soa_flat[3 * idx + 2] = vars->data->SoA[2];
+    }
+    vars = vars->next;
+    ++idx;
+  }
+  return idx == Z4C_CUDA_STATE_COUNT && vars == 0;
+}
+
+bool z4c_cuda_keep_resident_after_step(int lev, int trfls_in, int analysis_lev)
+{
+  static int keep_all_levels = -1;
+  if (keep_all_levels < 0)
+  {
+    const char *env = getenv("AMSS_CUDA_KEEP_ALL_LEVELS");
+    keep_all_levels = (env && atoi(env) != 0) ? 1 : 0;
+  }
+  static int enabled = -1;
+  if (enabled < 0)
+  {
+    const char *env = getenv("AMSS_CUDA_Z4C_KEEP_RESIDENT_AFTER_STEP");
+    if (env)
+      enabled = (atoi(env) != 0) ? 1 : 0;
+    else
+    {
+      env = getenv("AMSS_CUDA_KEEP_RESIDENT_AFTER_STEP");
+      enabled = (env && atoi(env) != 0) ? 1 : 0;
+    }
+  }
+  if (!enabled)
+    return false;
+  if (lev == analysis_lev)
+    return false;
+  if (keep_all_levels)
+    return true;
+  return lev < trfls_in;
+}
+
+void z4c_cuda_download_level_state(MyList<Patch> *PatL, MyList<var> *vars, int myrank, bool release_ctx)
+{
+  MyList<Patch> *Pp = PatL;
+  while (Pp)
+  {
+    MyList<Block> *BP = Pp->data->blb;
+    while (BP)
+    {
+      Block *cg = BP->data;
+      if (myrank == cg->rank && z4c_cuda_has_resident_state(cg))
+      {
+        double *state_out[Z4C_CUDA_STATE_COUNT];
+        if (!fill_z4c_cuda_views(cg, vars, state_out))
+        {
+          cout << "CUDA Z4C state list mismatch on resident state download" << endl;
+          MPI_Abort(MPI_COMM_WORLD, 1);
+        }
+        if (z4c_cuda_download_resident_state(cg, cg->shape, state_out))
+        {
+          cout << "CUDA Z4C resident state download failed" << endl;
+          MPI_Abort(MPI_COMM_WORLD, 1);
+        }
+        if (release_ctx)
+          z4c_cuda_release_step_ctx(cg);
+      }
+      if (BP == Pp->data->ble)
+        break;
+      BP = BP->next;
+    }
+    Pp = Pp->next;
+  }
+}
+
+bool z4c_cuda_patch_contains_point(Patch *patch, const double *point)
+{
+  if (!patch)
+    return false;
+  for (int d = 0; d < dim; d++)
+  {
+    const double h = patch->getdX(d);
+    const double lo = patch->bbox[d] + patch->lli[d] * h;
+    const double hi = patch->bbox[dim + d] - patch->uui[d] * h;
+    if (point[d] < lo || point[d] > hi)
+      return false;
+  }
+  return true;
+}
+
+bool z4c_cuda_point_in_block(Patch *patch, Block *block,
+                             const double *point, const double *DH)
+{
+  if (!patch || !block)
+    return false;
+  for (int d = 0; d < dim; d++)
+  {
+    double llb;
+    double uub;
+#ifdef Vertex
+#ifdef Cell
+#error Both Cell and Vertex are defined
+#endif
+    llb = (feq(block->bbox[d], patch->bbox[d], DH[d] / 2))
+              ? block->bbox[d] + patch->lli[d] * DH[d]
+              : block->bbox[d] + (ghost_width - 0.5) * DH[d];
+    uub = (feq(block->bbox[dim + d], patch->bbox[dim + d], DH[d] / 2))
+              ? block->bbox[dim + d] - patch->uui[d] * DH[d]
+              : block->bbox[dim + d] - (ghost_width - 0.5) * DH[d];
+#else
+#ifdef Cell
+    llb = (feq(block->bbox[d], patch->bbox[d], DH[d] / 2))
+              ? block->bbox[d] + patch->lli[d] * DH[d]
+              : block->bbox[d] + ghost_width * DH[d];
+    uub = (feq(block->bbox[dim + d], patch->bbox[dim + d], DH[d] / 2))
+              ? block->bbox[dim + d] - patch->uui[d] * DH[d]
+              : block->bbox[dim + d] - ghost_width * DH[d];
+#else
+#error Not define Vertex nor Cell
+#endif
+#endif
+    if (point[d] - llb < -DH[d] / 2 || point[d] - uub > DH[d] / 2)
+      return false;
+  }
+  return true;
+}
+
+int z4c_cuda_interp_tile_start(const double *coords, int n, double x, double dx, int ordn)
+{
+  if (!coords || n <= ordn)
+    return 0;
+  int cxi = int((x - coords[0]) / dx + 0.4) + 1;
+  int start = cxi - ordn / 2;
+  if (start < 0)
+    start = 0;
+  const int max_start = n - ordn;
+  if (start > max_start)
+    start = max_start;
+  return start;
+}
+
+bool z4c_cuda_interp_bh_point_resident(MyList<Patch> *PatL,
+                                       int myrank,
+                                       const double *point,
+                                       var *forx, var *fory, var *forz,
+                                       int Symmetry,
+                                       double *shellf)
+{
+  const int ordn = 2 * ghost_width;
+  int owner_rank = -1;
+
+  shellf[0] = shellf[1] = shellf[2] = 0.0;
+
+  MyList<Patch> *PL = PatL;
+  while (PL)
+  {
+    Patch *patch = PL->data;
+    if (!z4c_cuda_patch_contains_point(patch, point))
+    {
+      PL = PL->next;
+      continue;
+    }
+
+    double DH[dim];
+    for (int d = 0; d < dim; d++)
+      DH[d] = patch->getdX(d);
+
+    MyList<Block> *BP = patch->blb;
+    while (BP)
+    {
+      Block *block = BP->data;
+      if (z4c_cuda_point_in_block(patch, block, point, DH))
+      {
+        owner_rank = block->rank;
+        if (myrank == owner_rank)
+        {
+          int interp_ordn = ordn;
+          int interp_sym = Symmetry;
+          double x = point[0];
+          double y = point[1];
+          double z = point[2];
+
+          if (z4c_cuda_has_resident_state(block) &&
+              block->shape[0] >= ordn && block->shape[1] >= ordn && block->shape[2] >= ordn)
+          {
+            var *vars[3] = {forx, fory, forz};
+            static int use_device_bh_interp = -1;
+            if (use_device_bh_interp < 0)
+            {
+              const char *env = getenv("AMSS_CUDA_Z4C_BH_INTERP_DEVICE");
+              use_device_bh_interp = (env && atoi(env) != 0) ? 1 : 0;
+            }
+            bool used_device_interp = false;
+            if (use_device_bh_interp)
+            {
+              double soa3[9];
+              for (int f = 0; f < 3; f++)
+              {
+                soa3[3 * f + 0] = vars[f]->SoA[0];
+                soa3[3 * f + 1] = vars[f]->SoA[1];
+                soa3[3 * f + 2] = vars[f]->SoA[2];
+              }
+              used_device_interp =
+                  (z4c_cuda_interp_state_point3(block, block->shape,
+                                                k_z4c_cuda_bh_state_indices[0],
+                                                k_z4c_cuda_bh_state_indices[1],
+                                                k_z4c_cuda_bh_state_indices[2],
+                                                block->X[0][0], block->X[1][0], block->X[2][0],
+                                                DH[0], DH[1], DH[2],
+                                                x, y, z,
+                                                interp_ordn, interp_sym,
+                                                soa3, shellf) == 0);
+            }
+            if (!used_device_interp)
+            {
+              double *shift_views[3] = {
+                  block->fgfs[forx->sgfn],
+                  block->fgfs[fory->sgfn],
+                  block->fgfs[forz->sgfn]};
+              if (z4c_cuda_download_state_subset(block, block->shape, 3,
+                                                 k_z4c_cuda_bh_state_indices,
+                                                 shift_views) != 0)
+              {
+                cout << "CUDA Z4C BH shift download failed" << endl;
+                MPI_Abort(MPI_COMM_WORLD, 1);
+              }
+              f_global_interp(block->shape, block->X[0], block->X[1], block->X[2],
+                              block->fgfs[forx->sgfn], shellf[0],
+                              x, y, z, interp_ordn, forx->SoA, interp_sym);
+              f_global_interp(block->shape, block->X[0], block->X[1], block->X[2],
+                              block->fgfs[fory->sgfn], shellf[1],
+                              x, y, z, interp_ordn, fory->SoA, interp_sym);
+              f_global_interp(block->shape, block->X[0], block->X[1], block->X[2],
+                              block->fgfs[forz->sgfn], shellf[2],
+                              x, y, z, interp_ordn, forz->SoA, interp_sym);
+            }
+          }
+          else
+          {
+            f_global_interp(block->shape, block->X[0], block->X[1], block->X[2],
+                            block->fgfs[forx->sgfn], shellf[0],
+                            x, y, z, interp_ordn, forx->SoA, interp_sym);
+            f_global_interp(block->shape, block->X[0], block->X[1], block->X[2],
+                            block->fgfs[fory->sgfn], shellf[1],
+                            x, y, z, interp_ordn, fory->SoA, interp_sym);
+            f_global_interp(block->shape, block->X[0], block->X[1], block->X[2],
+                            block->fgfs[forz->sgfn], shellf[2],
+                            x, y, z, interp_ordn, forz->SoA, interp_sym);
+          }
+        }
+        break;
+      }
+      if (BP == patch->ble)
+        break;
+      BP = BP->next;
+    }
+
+    if (owner_rank >= 0)
+      break;
+    PL = PL->next;
+  }
+
+  if (owner_rank < 0)
+    return false;
+
+  MPI_Bcast(shellf, 3, MPI_DOUBLE, owner_rank, MPI_COMM_WORLD);
+  return true;
+}
+
+bool z4c_cuda_compute_porg_rhs_resident(cgh *GH,
+                                        int ilev,
+                                        int myrank,
+                                        int BH_num,
+                                        double **BH_PS,
+                                        double **BH_RHS,
+                                        var *forx, var *fory, var *forz,
+                                        int Symmetry)
+{
+  for (int n = 0; n < BH_num; n++)
+  {
+    double shellf[3] = {0.0, 0.0, 0.0};
+    int lev = ilev;
+    while (lev >= 0 &&
+           !z4c_cuda_interp_bh_point_resident(GH->PatL[lev], myrank, BH_PS[n],
+                                              forx, fory, forz, Symmetry, shellf))
+    {
+      --lev;
+    }
+    if (lev < 0)
+      return false;
+    BH_RHS[n][0] = -shellf[0];
+    BH_RHS[n][1] = -shellf[1];
+    BH_RHS[n][2] = -shellf[2];
+  }
+  return true;
+}
+
+bool z4c_cuda_download_bh_shift_level(MyList<Patch> *PatL,
+                                      int myrank,
+                                      var *forx, var *fory, var *forz)
+{
+  MyList<Patch> *Pp = PatL;
+  while (Pp)
+  {
+    MyList<Block> *BP = Pp->data->blb;
+    while (BP)
+    {
+      Block *cg = BP->data;
+      if (myrank == cg->rank && z4c_cuda_has_resident_state(cg))
+      {
+        double *fields[3] = {
+            cg->fgfs[forx->sgfn],
+            cg->fgfs[fory->sgfn],
+            cg->fgfs[forz->sgfn]};
+        if (z4c_cuda_download_state_subset(cg, cg->shape, 3,
+                                           k_z4c_cuda_bh_state_indices,
+                                           fields))
+          return false;
+      }
+      if (BP == Pp->data->ble)
+        break;
+      BP = BP->next;
+    }
+    Pp = Pp->next;
+  }
+  return true;
+}
+
+bool z4c_cuda_refresh_constraint_level(MyList<Patch> *PatL,
+                                       int myrank,
+                                       var *Cons_Ham, var *Cons_Px,
+                                       var *Cons_Py, var *Cons_Pz,
+                                       var *Cons_Gx, var *Cons_Gy,
+                                       var *Cons_Gz, var *TZ0,
+                                       int Symmetry, int lev, double eps)
+{
+  bool all_resident = true;
+  const int tz_index = 24;
+  MyList<Patch> *Pp = PatL;
+  while (Pp)
+  {
+    MyList<Block> *BP = Pp->data->blb;
+    while (BP)
+    {
+      Block *cg = BP->data;
+      if (myrank == cg->rank)
+      {
+        if (!z4c_cuda_has_resident_state(cg))
+        {
+          all_resident = false;
+        }
+        else
+        {
+          double *constraints[7] = {
+              cg->fgfs[Cons_Ham->sgfn], cg->fgfs[Cons_Px->sgfn],
+              cg->fgfs[Cons_Py->sgfn], cg->fgfs[Cons_Pz->sgfn],
+              cg->fgfs[Cons_Gx->sgfn], cg->fgfs[Cons_Gy->sgfn],
+              cg->fgfs[Cons_Gz->sgfn]};
+          double *tz_out[1] = {cg->fgfs[TZ0->sgfn]};
+          int co = 0;
+          if (z4c_cuda_compute_constraints_resident(cg, cg->shape,
+                                                   cg->X[0], cg->X[1], cg->X[2],
+                                                   Symmetry, eps, co,
+                                                   constraints) ||
+              z4c_cuda_download_state_subset(cg, cg->shape, 1, &tz_index, tz_out))
+          {
+            cout << "CUDA Z4C resident constraint refresh failed" << endl;
+            MPI_Abort(MPI_COMM_WORLD, 1);
+          }
+        }
+      }
+      if (BP == Pp->data->ble)
+        break;
+      BP = BP->next;
+    }
+    Pp = Pp->next;
+  }
+  return all_resident;
+}
+
+long long &z4c_constraint_output_counter()
+{
+  static long long counter = 0;
+  return counter;
+}
+
+int z4c_constraint_output_every()
+{
+  static int every = -1;
+  if (every < 0)
+  {
+    const char *env = getenv("AMSS_CUDA_Z4C_CONSTRAINT_EVERY");
+    every = (env && atoi(env) > 0) ? atoi(env) : 1;
+  }
+  return every;
+}
+
+bool z4c_constraint_output_due_now()
+{
+  const int every = z4c_constraint_output_every();
+  return every <= 1 || (z4c_constraint_output_counter() % every) == 0;
+}
+
+void z4c_constraint_output_advance()
+{
+  z4c_constraint_output_counter()++;
+}
+
+} // namespace
+#endif
+
+void Z4c_class::Step(int lev, int YN)
+{
+#if USE_CUDA_Z4C && (ABEtype == 2)
+  double dT_lev = dT * pow(0.5, Mymax(lev, trfls));
+#ifdef With_AHF
+  AH_Step_Find(lev, dT_lev);
+#endif
+  bool BB = fgt(PhysTime, StartTime, dT_lev / 2);
+  double ndeps = numepss;
+  if (lev < GH->movls)
+    ndeps = numepsb;
+  double TRK4 = PhysTime;
+  int iter_count = 0;
+  int pre = 0, cor = 1;
+  int ERROR = 0;
+  const double dT_mon = dT * pow(0.5, Mymax(0, trfls));
+  const bool need_constraint_after_step =
+      (LastConsOut + dT_mon >= AnasTime) && z4c_constraint_output_due_now();
+
+  if (BH_num > 0 && lev == GH->levels - 1)
+  {
+    if (!z4c_cuda_download_bh_shift_level(GH->PatL[lev], myrank, Sfx0, Sfy0, Sfz0))
+    {
+      if (myrank == 0 && ErrorMonitor->outfile)
+        ErrorMonitor->outfile << "CUDA Z4C failed to download predictor black-hole shift at t = "
+                              << PhysTime << endl;
+      MPI_Abort(MPI_COMM_WORLD, 1);
+    }
+    compute_Porg_rhs(Porg0, Porg_rhs, Sfx0, Sfy0, Sfz0, lev);
+    for (int ithBH = 0; ithBH < BH_num; ithBH++)
+    {
+      f_rungekutta4_scalar(dT_lev, Porg0[ithBH][0], Porg[ithBH][0], Porg_rhs[ithBH][0], iter_count);
+      f_rungekutta4_scalar(dT_lev, Porg0[ithBH][1], Porg[ithBH][1], Porg_rhs[ithBH][1], iter_count);
+      f_rungekutta4_scalar(dT_lev, Porg0[ithBH][2], Porg[ithBH][2], Porg_rhs[ithBH][2], iter_count);
+      if (Symmetry > 0)
+        Porg[ithBH][2] = fabs(Porg[ithBH][2]);
+      if (Symmetry == 2)
+      {
+        Porg[ithBH][0] = fabs(Porg[ithBH][0]);
+        Porg[ithBH][1] = fabs(Porg[ithBH][1]);
+      }
+    }
+  }
+
+  MyList<Patch> *Pp = GH->PatL[lev];
+  while (Pp)
+  {
+    MyList<Block> *BP = Pp->data->blb;
+    while (BP)
+    {
+      Block *cg = BP->data;
+      if (myrank == cg->rank)
+      {
+        double *state_in[Z4C_CUDA_STATE_COUNT];
+        double *state_out[Z4C_CUDA_STATE_COUNT];
+        double propspeed[Z4C_CUDA_STATE_COUNT];
+        double soa_flat[3 * Z4C_CUDA_STATE_COUNT];
+        if (!fill_z4c_cuda_views(cg, StateList, state_in, propspeed, soa_flat) ||
+            !fill_z4c_cuda_views(cg, SynchList_pre, state_out))
+        {
+          cout << "CUDA Z4C state list mismatch on predictor step" << endl;
+          MPI_Abort(MPI_COMM_WORLD, 1);
+        }
+        int apply_bam_bc = 0;
+#if (MRBD == 0)
+#if (SommerType == 0)
+        apply_bam_bc = (lev == 0) ? 1 : 0;
+#endif
+#elif (MRBD == 1)
+        apply_bam_bc = 1;
+#endif
+        int keep_resident_state = 1;
+        int apply_enforce_ga = 0;
+#if (AGM == 0)
+        apply_enforce_ga = 1;
+#endif
+        if (z4c_cuda_rk4_substep(cg,
+                                 cg->shape, cg->X[0], cg->X[1], cg->X[2],
+                                 state_in, state_out,
+                                 propspeed, soa_flat, Pp->data->bbox,
+                                 dT_lev, TRK4, iter_count, apply_bam_bc,
+                                 Symmetry, lev, ndeps, pre,
+                                 keep_resident_state, apply_enforce_ga, chitiny))
+        {
+          cout << "CUDA Z4C predictor substep failed in domain: ("
+               << cg->bbox[0] << ":" << cg->bbox[3] << ","
+               << cg->bbox[1] << ":" << cg->bbox[4] << ","
+               << cg->bbox[2] << ":" << cg->bbox[5] << ")" << endl;
+          ERROR = 1;
+        }
+      }
+      if (BP == Pp->data->ble)
+        break;
+      BP = BP->next;
+    }
+    Pp = Pp->next;
+  }
+
+  {
+    int erh = ERROR;
+    MPI_Allreduce(&erh, &ERROR, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD);
+  }
+  if (ERROR)
+  {
+    if (myrank == 0 && ErrorMonitor->outfile)
+      ErrorMonitor->outfile << "CUDA Z4C failed in predictor at t = " << PhysTime
+                            << ", lev = " << lev << endl;
+    MPI_Abort(MPI_COMM_WORLD, 1);
+  }
+
+  {
+    Parallel::AsyncSyncState async_pre;
+    Parallel::Sync_start(GH->PatL[lev], SynchList_pre, Symmetry, sync_cache_pre[lev], async_pre);
+    Parallel::Sync_finish(sync_cache_pre[lev], async_pre, SynchList_pre, Symmetry);
+  }
+
+  if ((lev == a_lev) && (LastAnas + dT_lev >= AnasTime))
+    z4c_cuda_download_level_state(GH->PatL[lev], SynchList_pre, myrank, false);
+  if (lev == a_lev)
+    AnalysisStuff(lev, dT_lev);
+
+  for (iter_count = 1; iter_count < 4; iter_count++)
+  {
+    if (iter_count == 1 || iter_count == 3)
+      TRK4 += dT_lev / 2;
+    Pp = GH->PatL[lev];
+    while (Pp)
+    {
+      MyList<Block> *BP = Pp->data->blb;
+      while (BP)
+      {
+        Block *cg = BP->data;
+        if (myrank == cg->rank)
+        {
+          double *state_in[Z4C_CUDA_STATE_COUNT];
+          double *state_out[Z4C_CUDA_STATE_COUNT];
+          double propspeed[Z4C_CUDA_STATE_COUNT];
+          double soa_flat[3 * Z4C_CUDA_STATE_COUNT];
+          if (!fill_z4c_cuda_views(cg, SynchList_pre, state_in, propspeed, soa_flat) ||
+              !fill_z4c_cuda_views(cg, SynchList_cor, state_out))
+          {
+            cout << "CUDA Z4C state list mismatch on corrector step" << endl;
+            MPI_Abort(MPI_COMM_WORLD, 1);
+          }
+          int apply_bam_bc = 0;
+#if (MRBD == 0)
+#if (SommerType == 0)
+          apply_bam_bc = (lev == 0) ? 1 : 0;
+#endif
+#elif (MRBD == 1)
+          apply_bam_bc = 1;
+#endif
+          int keep_resident_state = 1;
+          int apply_enforce_ga = 0;
+#if (AGM == 0)
+          apply_enforce_ga = 1;
+#elif (AGM == 1)
+          apply_enforce_ga = (iter_count == 3) ? 1 : 0;
+#endif
+          if (z4c_cuda_rk4_substep(cg,
+                                   cg->shape, cg->X[0], cg->X[1], cg->X[2],
+                                   state_in, state_out,
+                                   propspeed, soa_flat, Pp->data->bbox,
+                                   dT_lev, TRK4, iter_count, apply_bam_bc,
+                                   Symmetry, lev, ndeps, cor,
+                                   keep_resident_state, apply_enforce_ga, chitiny))
+          {
+            cout << "CUDA Z4C corrector substep failed in domain: ("
+                 << cg->bbox[0] << ":" << cg->bbox[3] << ","
+                 << cg->bbox[1] << ":" << cg->bbox[4] << ","
+                 << cg->bbox[2] << ":" << cg->bbox[5] << ")" << endl;
+            ERROR = 1;
+          }
+          if (!ERROR && iter_count == 3 && need_constraint_after_step)
+          {
+            double *constraints[7] = {
+                cg->fgfs[Cons_Ham->sgfn], cg->fgfs[Cons_Px->sgfn],
+                cg->fgfs[Cons_Py->sgfn], cg->fgfs[Cons_Pz->sgfn],
+                cg->fgfs[Cons_Gx->sgfn], cg->fgfs[Cons_Gy->sgfn],
+                cg->fgfs[Cons_Gz->sgfn]};
+            double *tz_out[1] = {cg->fgfs[TZ0->sgfn]};
+            const int tz_index = 24;
+            if (z4c_cuda_download_constraint_outputs(cg->shape, constraints) ||
+                z4c_cuda_download_state_subset(cg, cg->shape, 1, &tz_index, tz_out))
+            {
+              cout << "CUDA Z4C constraint download failed in domain: ("
+                   << cg->bbox[0] << ":" << cg->bbox[3] << ","
+                   << cg->bbox[1] << ":" << cg->bbox[4] << ","
+                   << cg->bbox[2] << ":" << cg->bbox[5] << ")" << endl;
+              ERROR = 1;
+            }
+          }
+        }
+        if (BP == Pp->data->ble)
+          break;
+        BP = BP->next;
+      }
+      Pp = Pp->next;
+    }
+
+    {
+      int erh = ERROR;
+      MPI_Allreduce(&erh, &ERROR, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD);
+    }
+    if (ERROR)
+    {
+      if (myrank == 0 && ErrorMonitor->outfile)
+        ErrorMonitor->outfile << "CUDA Z4C failed in RK4 substep#" << iter_count
+                              << " at t = " << PhysTime
+                              << ", lev = " << lev << endl;
+      MPI_Abort(MPI_COMM_WORLD, 1);
+    }
+
+    {
+      Parallel::AsyncSyncState async_cor;
+      Parallel::Sync_start(GH->PatL[lev], SynchList_cor, Symmetry, sync_cache_cor[lev], async_cor);
+      Parallel::Sync_finish(sync_cache_cor[lev], async_cor, SynchList_cor, Symmetry);
+    }
+
+    if (BH_num > 0 && lev == GH->levels - 1)
+    {
+      if (!z4c_cuda_compute_porg_rhs_resident(GH, lev, myrank, BH_num,
+                                              Porg, Porg1,
+                                              Sfx, Sfy, Sfz, Symmetry))
+      {
+        if (myrank == 0 && ErrorMonitor->outfile)
+          ErrorMonitor->outfile << "CUDA Z4C failed to interpolate black-hole shift at t = "
+                                << PhysTime << endl;
+        MPI_Abort(MPI_COMM_WORLD, 1);
+      }
+      for (int ithBH = 0; ithBH < BH_num; ithBH++)
+      {
+        f_rungekutta4_scalar(dT_lev, Porg0[ithBH][0], Porg1[ithBH][0], Porg_rhs[ithBH][0], iter_count);
+        f_rungekutta4_scalar(dT_lev, Porg0[ithBH][1], Porg1[ithBH][1], Porg_rhs[ithBH][1], iter_count);
+        f_rungekutta4_scalar(dT_lev, Porg0[ithBH][2], Porg1[ithBH][2], Porg_rhs[ithBH][2], iter_count);
+        if (Symmetry > 0)
+          Porg1[ithBH][2] = fabs(Porg1[ithBH][2]);
+        if (Symmetry == 2)
+        {
+          Porg1[ithBH][0] = fabs(Porg1[ithBH][0]);
+          Porg1[ithBH][1] = fabs(Porg1[ithBH][1]);
+        }
+      }
+    }
+
+    if (iter_count < 3)
+    {
+      Pp = GH->PatL[lev];
+      while (Pp)
+      {
+        MyList<Block> *BP = Pp->data->blb;
+        while (BP)
+        {
+          Block *cg = BP->data;
+          cg->swapList(SynchList_pre, SynchList_cor, myrank);
+          if (BP == Pp->data->ble)
+            break;
+          BP = BP->next;
+        }
+        Pp = Pp->next;
+      }
+      if (BH_num > 0 && lev == GH->levels - 1)
+      {
+        for (int ithBH = 0; ithBH < BH_num; ithBH++)
+        {
+          Porg[ithBH][0] = Porg1[ithBH][0];
+          Porg[ithBH][1] = Porg1[ithBH][1];
+          Porg[ithBH][2] = Porg1[ithBH][2];
+        }
+      }
+    }
+  }
+
+  {
+    const bool keep_resident = z4c_cuda_keep_resident_after_step(lev, trfls, a_lev);
+    const bool need_host_after_step =
+        ((lev == a_lev) && (LastAnas + dT_lev >= AnasTime));
+    if (!keep_resident || need_host_after_step)
+      z4c_cuda_download_level_state(GH->PatL[lev], SynchList_cor, myrank, !keep_resident);
+  }
+
+#if (RPS == 0)
+  RestrictProlong(lev, YN, BB);
+#endif
+
+  Pp = GH->PatL[lev];
+  while (Pp)
+  {
+    MyList<Block> *BP = Pp->data->blb;
+    while (BP)
+    {
+      Block *cg = BP->data;
+      cg->swapList(StateList, SynchList_cor, myrank);
+      cg->swapList(OldStateList, SynchList_cor, myrank);
+      if (BP == Pp->data->ble)
+        break;
+      BP = BP->next;
+    }
+    Pp = Pp->next;
+  }
+  if (BH_num > 0 && lev == GH->levels - 1)
+  {
+    for (int ithBH = 0; ithBH < BH_num; ithBH++)
+    {
+      Porg0[ithBH][0] = Porg1[ithBH][0];
+      Porg0[ithBH][1] = Porg1[ithBH][1];
+      Porg0[ithBH][2] = Porg1[ithBH][2];
+    }
+  }
+#else
+  double dT_lev = dT * pow(0.5, Mymax(lev, trfls));
 #ifdef With_AHF
   AH_Step_Find(lev, dT_lev);
 #endif
@@ -1039,15 +1791,19 @@ void Z4c_class::Step(int lev, int YN)
     {
       Porg0[ithBH][0] = Porg1[ithBH][0];
       Porg0[ithBH][1] = Porg1[ithBH][1];
-      Porg0[ithBH][2] = Porg1[ithBH][2];
-    }
-  }
-}
-#else
-// for constraint preserving boundary (CPBC)
-#ifndef WithShell
-#error "CPBC only supports Shell"
-#endif
+      Porg0[ithBH][2] = Porg1[ithBH][2];
+    }
+  }
+#endif
+}
+#else
+// for constraint preserving boundary (CPBC)
+#if USE_CUDA_Z4C && (ABEtype == 2)
+#error "USE_CUDA_Z4C resident path does not support CPBC"
+#endif
+#ifndef WithShell
+#error "CPBC only supports Shell"
+#endif
 
 // 0: extroplate rhs, 1: extroplate variable
 // 2: extroplate variable but before RHS calculation
@@ -2408,17 +3164,23 @@ void Z4c_class::Check_extrop()
 
 //================================================================================================
 
-void Z4c_class::Constraint_Out()
-{
-  // here we have to use the same variable name as in the parent class
-  LastConsOut += dT * pow(0.5, Mymax(0, trfls));
-  
-  if (LastConsOut >= AnasTime)
-  // Constraint violation
-  {
-    // recompute least the constraint data lost for moved new grid
-    for (int lev = 0; lev < GH->levels; lev++)
-    {
+void Z4c_class::Constraint_Out()
+{
+  // here we have to use the same variable name as in the parent class
+  LastConsOut += dT * pow(0.5, Mymax(0, trfls));
+  
+  if (LastConsOut >= AnasTime)
+  // Constraint violation
+  {
+#if USE_CUDA_Z4C && (ABEtype == 2)
+    bool cuda_constraints_ready = true;
+#else
+    const bool cuda_constraints_ready = false;
+#endif
+    // recompute least the constraint data lost for moved new grid
+    if (!cuda_constraints_ready)
+      for (int lev = 0; lev < GH->levels; lev++)
+    {
       // make sure the data consistent for higher levels
       if (lev > 0)
       {

AMSS_NCKU_source/Z4c_rhs.f90

@@ -94,29 +94,31 @@
                Hcon,Mxcon,Mycon,Mzcon,Gmxcon,Gmycon,Gmzcon,                    &            
                Symmetry,Lev,eps,co)
 
-#if (ABV == 0)  
-  call ricci_gamma(ex, X, Y, Z,                                      &
-               chi,                                                  &
-               dxx    ,   gxy    ,   gxz    ,   dyy    ,   gyz    ,   dzz,&
-               Gamx   ,  Gamy    ,  Gamz    , &
-               Gamxxx,Gamxxy,Gamxxz,Gamxyy,Gamxyz,Gamxzz,&
-               Gamyxx,Gamyxy,Gamyxz,Gamyyy,Gamyyz,Gamyzz,&
-               Gamzxx,Gamzxy,Gamzxz,Gamzyy,Gamzyz,Gamzzz,&
-               Rxx,Rxy,Rxz,Ryy,Ryz,Rzz,&
-               Symmetry)
-#endif
-  call constraint_bssn(ex, X, Y, Z,&
-               chi,trK, &
-               dxx,gxy,gxz,dyy,gyz,dzz, &
-               Axx,Axy,Axz,Ayy,Ayz,Azz, &
-               Gamx,Gamy,Gamz,&
-               Lap,betax,betay,betaz,rho,Sx,Sy,Sz,&
-               Gamxxx, Gamxxy, Gamxxz,Gamxyy, Gamxyz, Gamxzz, &
-               Gamyxx, Gamyxy, Gamyxz,Gamyyy, Gamyyz, Gamyzz, &
-               Gamzxx, Gamzxy, Gamzxz,Gamzyy, Gamzyz, Gamzzz, &
-               Rxx,Rxy,Rxz,Ryy,Ryz,Rzz, &
-               Hcon,Mxcon,Mycon,Mzcon,Gmxcon,Gmycon,Gmzcon, &
-               Symmetry)
+  if (co == 0) then
+#if (ABV == 0)  
+    call ricci_gamma(ex, X, Y, Z,                                      &
+                 chi,                                                  &
+                 dxx    ,   gxy    ,   gxz    ,   dyy    ,   gyz    ,   dzz,&
+                 Gamx   ,  Gamy    ,  Gamz    , &
+                 Gamxxx,Gamxxy,Gamxxz,Gamxyy,Gamxyz,Gamxzz,&
+                 Gamyxx,Gamyxy,Gamyxz,Gamyyy,Gamyyz,Gamyzz,&
+                 Gamzxx,Gamzxy,Gamzxz,Gamzyy,Gamzyz,Gamzzz,&
+                 Rxx,Rxy,Rxz,Ryy,Ryz,Rzz,&
+                 Symmetry)
+#endif
+    call constraint_bssn(ex, X, Y, Z,&
+                 chi,trK, &
+                 dxx,gxy,gxz,dyy,gyz,dzz, &
+                 Axx,Axy,Axz,Ayy,Ayz,Azz, &
+                 Gamx,Gamy,Gamz,&
+                 Lap,betax,betay,betaz,rho,Sx,Sy,Sz,&
+                 Gamxxx, Gamxxy, Gamxxz,Gamxyy, Gamxyz, Gamxzz, &
+                 Gamyxx, Gamyxy, Gamyxz,Gamyyy, Gamyyz, Gamyzz, &
+                 Gamzxx, Gamzxy, Gamzxz,Gamzyy, Gamzyz, Gamzzz, &
+                 Rxx,Rxy,Rxz,Ryy,Ryz,Rzz, &
+                 Hcon,Mxcon,Mycon,Mzcon,Gmxcon,Gmycon,Gmzcon, &
+                 Symmetry)
+  endif
 
   return
 
@@ -226,11 +228,12 @@
 
   call get_Z4cparameters(kappa1,kappa2,kappa3,FF,eta)
 
-!!! sanity check
-  dX = sum(chi)+sum(trK)+sum(dxx)+sum(gxy)+sum(gxz)+sum(dyy)+sum(gyz)+sum(dzz) &
-      +sum(Axx)+sum(Axy)+sum(Axz)+sum(Ayy)+sum(Ayz)+sum(Azz)                   &
-      +sum(Gamx)+sum(Gamy)+sum(Gamz)                                           &
-      +sum(Lap)+sum(betax)+sum(betay)+sum(betaz)+sum(dtSfx)+sum(dtSfy)+sum(dtSfz) &
+!!! sanity check
+#ifdef DEBUG
+  dX = sum(chi)+sum(trK)+sum(dxx)+sum(gxy)+sum(gxz)+sum(dyy)+sum(gyz)+sum(dzz) &
+      +sum(Axx)+sum(Axy)+sum(Axz)+sum(Ayy)+sum(Ayz)+sum(Azz)                   &
+      +sum(Gamx)+sum(Gamy)+sum(Gamz)                                           &
+      +sum(Lap)+sum(betax)+sum(betay)+sum(betaz)+sum(dtSfx)+sum(dtSfy)+sum(dtSfz) &
       +sum(TZ)
   if(dX.ne.dX) then
      if(sum(chi).ne.sum(chi))write(*,*)"Z4c_rhs.f90: find NaN in chi"
@@ -257,10 +260,11 @@
      if(sum(dtSfx).ne.sum(dtSfx))write(*,*)"Z4c_rhs.f90: find NaN in dtSfx"
      if(sum(dtSfy).ne.sum(dtSfy))write(*,*)"Z4c_rhs.f90: find NaN in dtSfy"
      if(sum(dtSfz).ne.sum(dtSfz))write(*,*)"Z4c_rhs.f90: find NaN in dtSfz"
-     if(sum(TZ).ne.sum(Tz))write(*,*)"Z4c_rhs.f90: find NaN in TZ"
-     gont = 1
-     return
-  endif
+     if(sum(TZ).ne.sum(Tz))write(*,*)"Z4c_rhs.f90: find NaN in TZ"
+     gont = 1
+     return
+  endif
+#endif
 
   PI = dacos(-ONE)
 
@@ -1263,30 +1267,32 @@
 
   endif
 
-#if (ABV == 0)  
-  call ricci_gamma(ex, X, Y, Z,                                      &
-               chi,                                                  &
-               dxx    ,   gxy    ,   gxz    ,   dyy    ,   gyz    ,   dzz,&
-               Gamx   ,  Gamy    ,  Gamz    , &
-               Gamxxx,Gamxxy,Gamxxz,Gamxyy,Gamxyz,Gamxzz,&
-               Gamyxx,Gamyxy,Gamyxz,Gamyyy,Gamyyz,Gamyzz,&
-               Gamzxx,Gamzxy,Gamzxz,Gamzyy,Gamzyz,Gamzzz,&
-               Rxx,Rxy,Rxz,Ryy,Ryz,Rzz,&
-               Symmetry)
-#endif
-
-  call constraint_bssn(ex, X, Y, Z,&
-               chi,trK, &
-               dxx,gxy,gxz,dyy,gyz,dzz, &
-               Axx,Axy,Axz,Ayy,Ayz,Azz, &
-               Gamx,Gamy,Gamz,&
-               Lap,betax,betay,betaz,rho,Sx,Sy,Sz,&
-               Gamxxx, Gamxxy, Gamxxz,Gamxyy, Gamxyz, Gamxzz, &
-               Gamyxx, Gamyxy, Gamyxz,Gamyyy, Gamyyz, Gamyzz, &
-               Gamzxx, Gamzxy, Gamzxz,Gamzyy, Gamzyz, Gamzzz, &
-               Rxx,Rxy,Rxz,Ryy,Ryz,Rzz, &
-               Hcon,Mxcon,Mycon,Mzcon,Gmxcon,Gmycon,Gmzcon, &
-               Symmetry)
+  if (co == 0) then
+#if (ABV == 0)  
+    call ricci_gamma(ex, X, Y, Z,                                      &
+                 chi,                                                  &
+                 dxx    ,   gxy    ,   gxz    ,   dyy    ,   gyz    ,   dzz,&
+                 Gamx   ,  Gamy    ,  Gamz    , &
+                 Gamxxx,Gamxxy,Gamxxz,Gamxyy,Gamxyz,Gamxzz,&
+                 Gamyxx,Gamyxy,Gamyxz,Gamyyy,Gamyyz,Gamyzz,&
+                 Gamzxx,Gamzxy,Gamzxz,Gamzyy,Gamzyz,Gamzzz,&
+                 Rxx,Rxy,Rxz,Ryy,Ryz,Rzz,&
+                 Symmetry)
+#endif
+
+    call constraint_bssn(ex, X, Y, Z,&
+                 chi,trK, &
+                 dxx,gxy,gxz,dyy,gyz,dzz, &
+                 Axx,Axy,Axz,Ayy,Ayz,Azz, &
+                 Gamx,Gamy,Gamz,&
+                 Lap,betax,betay,betaz,rho,Sx,Sy,Sz,&
+                 Gamxxx, Gamxxy, Gamxxz,Gamxyy, Gamxyz, Gamxzz, &
+                 Gamyxx, Gamyxy, Gamyxz,Gamyyy, Gamyyz, Gamyzz, &
+                 Gamzxx, Gamzxy, Gamzxz,Gamzyy, Gamzyz, Gamzzz, &
+                 Rxx,Rxy,Rxz,Ryy,Ryz,Rzz, &
+                 Hcon,Mxcon,Mycon,Mzcon,Gmxcon,Gmycon,Gmzcon, &
+                 Symmetry)
+  endif
 
   gont = 0
 

AMSS_NCKU_source/Z4c_rhs_ss.f90

@@ -121,11 +121,12 @@
 
   call get_Z4cparameters(kappa1,kappa2,kappa3,FF,eta)
 
-!!! sanity check
-  dX = sum(chi)+sum(trK)+sum(dxx)+sum(gxy)+sum(gxz)+sum(dyy)+sum(gyz)+sum(dzz) &
-      +sum(Axx)+sum(Axy)+sum(Axz)+sum(Ayy)+sum(Ayz)+sum(Azz)                   &
-      +sum(Gamx)+sum(Gamy)+sum(Gamz)                                           &
-      +sum(Lap)+sum(betax)+sum(betay)+sum(betaz)+sum(dtSfx)+sum(dtSfy)+sum(dtSfz) &
+!!! sanity check
+#ifdef DEBUG
+  dX = sum(chi)+sum(trK)+sum(dxx)+sum(gxy)+sum(gxz)+sum(dyy)+sum(gyz)+sum(dzz) &
+      +sum(Axx)+sum(Axy)+sum(Axz)+sum(Ayy)+sum(Ayz)+sum(Azz)                   &
+      +sum(Gamx)+sum(Gamy)+sum(Gamz)                                           &
+      +sum(Lap)+sum(betax)+sum(betay)+sum(betaz)+sum(dtSfx)+sum(dtSfy)+sum(dtSfz) &
       +sum(TZ)
   if(dX.ne.dX) then
      if(sum(chi).ne.sum(chi))write(*,*)"Z4c_rhs_ss.f90: find NaN in chi"
@@ -152,10 +153,11 @@
      if(sum(dtSfx).ne.sum(dtSfx))write(*,*)"Z4c_rhs_ss.f90: find NaN in dtSfx"
      if(sum(dtSfy).ne.sum(dtSfy))write(*,*)"Z4c_rhs_ss.f90: find NaN in dtSfy"
      if(sum(dtSfz).ne.sum(dtSfz))write(*,*)"Z4c_rhs_ss.f90: find NaN in dtSfz"
-     if(sum(TZ).ne.sum(Tz))write(*,*)"Z4c_rhs_ss.f90: find NaN in TZ"
-     gont = 1
-     return
-  endif
+     if(sum(TZ).ne.sum(Tz))write(*,*)"Z4c_rhs_ss.f90: find NaN in TZ"
+     gont = 1
+     return
+  endif
+#endif
 
   PI = dacos(-ONE)
 
@@ -1388,41 +1390,43 @@
   call kodis_sh(ex,crho,sigma,R,TZ,TZ_rhs,SSS,Symmetry,eps,sst)
   endif
 
-#if (ABV == 1)  
-  call ricci_gamma_ss(ex,crho,sigma,R,X, Y, Z,                                 &
-               drhodx, drhody, drhodz,                                         &
-               dsigmadx,dsigmady,dsigmadz,                                     &
-               dRdx,dRdy,dRdz,                                                 &
-               drhodxx,drhodxy,drhodxz,drhodyy,drhodyz,drhodzz,                &
-               dsigmadxx,dsigmadxy,dsigmadxz,dsigmadyy,dsigmadyz,dsigmadzz,    &
-               dRdxx,dRdxy,dRdxz,dRdyy,dRdyz,dRdzz,                            &
-               chi,                                                  &
-               dxx    ,   gxy    ,   gxz    ,   dyy    ,   gyz    ,   dzz,&
-               Gamx   ,  Gamy    ,  Gamz    , &
-               Gamxxx,Gamxxy,Gamxxz,Gamxyy,Gamxyz,Gamxzz,&
-               Gamyxx,Gamyxy,Gamyxz,Gamyyy,Gamyyz,Gamyzz,&
-               Gamzxx,Gamzxy,Gamzxz,Gamzyy,Gamzyz,Gamzzz,&
-               Rxx,Rxy,Rxz,Ryy,Ryz,Rzz,&
-               Symmetry,Lev,sst)
-  call constraint_bssn_ss(ex,crho,sigma,R,X, Y, Z,  &
-               drhodx, drhody, drhodz,                                         &
-               dsigmadx,dsigmady,dsigmadz,                                     &
-               dRdx,dRdy,dRdz,                                                 &
-               drhodxx,drhodxy,drhodxz,drhodyy,drhodyz,drhodzz,                &
-               dsigmadxx,dsigmadxy,dsigmadxz,dsigmadyy,dsigmadyz,dsigmadzz,    &
-               dRdxx,dRdxy,dRdxz,dRdyy,dRdyz,dRdzz,                            &
-               chi,trK, &
-               dxx,gxy,gxz,dyy,gyz,dzz, &
-               Axx,Axy,Axz,Ayy,Ayz,Azz, &
-               Gamx,Gamy,Gamz,&
-               Lap,betax,betay,betaz,rho,Sx,Sy,Sz,&
-               Gamxxx, Gamxxy, Gamxxz,Gamxyy, Gamxyz, Gamxzz, &
-               Gamyxx, Gamyxy, Gamyxz,Gamyyy, Gamyyz, Gamyzz, &
-               Gamzxx, Gamzxy, Gamzxz,Gamzyy, Gamzyz, Gamzzz, &
-               Rxx,Rxy,Rxz,Ryy,Ryz,Rzz, &
-               Hcon,Mxcon,Mycon,Mzcon,Gmxcon,Gmycon,Gmzcon, &
-               Symmetry,Lev,sst)
-#endif
+  if (co == 0) then
+#if (ABV == 1)  
+    call ricci_gamma_ss(ex,crho,sigma,R,X, Y, Z,                               &
+                 drhodx, drhody, drhodz,                                       &
+                 dsigmadx,dsigmady,dsigmadz,                                   &
+                 dRdx,dRdy,dRdz,                                               &
+                 drhodxx,drhodxy,drhodxz,drhodyy,drhodyz,drhodzz,              &
+                 dsigmadxx,dsigmadxy,dsigmadxz,dsigmadyy,dsigmadyz,dsigmadzz,  &
+                 dRdxx,dRdxy,dRdxz,dRdyy,dRdyz,dRdzz,                          &
+                 chi,                                                          &
+                 dxx    ,   gxy    ,   gxz    ,   dyy    ,   gyz    ,   dzz,&
+                 Gamx   ,  Gamy    ,  Gamz    , &
+                 Gamxxx,Gamxxy,Gamxxz,Gamxyy,Gamxyz,Gamxzz,&
+                 Gamyxx,Gamyxy,Gamyxz,Gamyyy,Gamyyz,Gamyzz,&
+                 Gamzxx,Gamzxy,Gamzxz,Gamzyy,Gamzyz,Gamzzz,&
+                 Rxx,Rxy,Rxz,Ryy,Ryz,Rzz,&
+                 Symmetry,Lev,sst)
+#endif
+    call constraint_bssn_ss(ex,crho,sigma,R,X, Y, Z,  &
+                 drhodx, drhody, drhodz,                                       &
+                 dsigmadx,dsigmady,dsigmadz,                                   &
+                 dRdx,dRdy,dRdz,                                               &
+                 drhodxx,drhodxy,drhodxz,drhodyy,drhodyz,drhodzz,              &
+                 dsigmadxx,dsigmadxy,dsigmadxz,dsigmadyy,dsigmadyz,dsigmadzz,  &
+                 dRdxx,dRdxy,dRdxz,dRdyy,dRdyz,dRdzz,                          &
+                 chi,trK, &
+                 dxx,gxy,gxz,dyy,gyz,dzz, &
+                 Axx,Axy,Axz,Ayy,Ayz,Azz, &
+                 Gamx,Gamy,Gamz,&
+                 Lap,betax,betay,betaz,rho,Sx,Sy,Sz,&
+                 Gamxxx, Gamxxy, Gamxxz,Gamxyy, Gamxyz, Gamxzz, &
+                 Gamyxx, Gamyxy, Gamyxz,Gamyyy, Gamyyz, Gamyzz, &
+                 Gamzxx, Gamzxy, Gamzxz,Gamzyy, Gamzyz, Gamzzz, &
+                 Rxx,Rxy,Rxz,Ryy,Ryz,Rzz, &
+                 Hcon,Mxcon,Mycon,Mzcon,Gmxcon,Gmycon,Gmzcon, &
+                 Symmetry,Lev,sst)
+  endif
 
   gont = 0
 

AMSS_NCKU_source/bssnEM_class.C

@@ -15,10 +15,13 @@ using namespace std;
 #include "misc.h"
 #include "Ansorg.h"
 #include "fmisc.h"
-#include "Parallel.h"
-#include "bssnEM_class.h"
-#include "bssn_rhs.h"
-#include "empart.h"
+#include "Parallel.h"
+#include "bssnEM_class.h"
+#include "bssn_rhs.h"
+#if USE_CUDA_BSSN
+#include "bssn_rhs_cuda.h"
+#endif
+#include "empart.h"
 #include "initial_puncture.h"
 #include "initial_maxwell.h"
 #include "enforce_algebra.h"
@@ -32,11 +35,111 @@ using namespace std;
 #ifdef With_AHF
 #include "derivatives.h"
 #include "myglobal.h"
-#endif
-
-//================================================================================================
-
-// Define bssnEM_class
+#endif
+
+//================================================================================================
+
+#if USE_CUDA_BSSN
+namespace {
+
+bool fill_bssn_cuda_views_prefix(Block *cg, MyList<var> *vars,
+                                 double **host_views,
+                                 double *propspeeds = nullptr,
+                                 double *soa_flat = nullptr)
+{
+  int idx = 0;
+  while (vars && idx < BSSN_CUDA_STATE_COUNT)
+  {
+    host_views[idx] = cg->fgfs[vars->data->sgfn];
+    if (propspeeds)
+      propspeeds[idx] = vars->data->propspeed;
+    if (soa_flat)
+    {
+      soa_flat[3 * idx + 0] = vars->data->SoA[0];
+      soa_flat[3 * idx + 1] = vars->data->SoA[1];
+      soa_flat[3 * idx + 2] = vars->data->SoA[2];
+    }
+    vars = vars->next;
+    ++idx;
+  }
+  return idx == BSSN_CUDA_STATE_COUNT;
+}
+
+void skip_bssn_cuda_prefix(MyList<var> *&a, MyList<var> *&b, MyList<var> *&c)
+{
+  for (int i = 0; i < BSSN_CUDA_STATE_COUNT && a && b && c; ++i)
+  {
+    a = a->next;
+    b = b->next;
+    c = c->next;
+  }
+}
+
+void skip_bssn_cuda_prefix(MyList<var> *&a, MyList<var> *&b,
+                           MyList<var> *&c, MyList<var> *&d)
+{
+  for (int i = 0; i < BSSN_CUDA_STATE_COUNT && a && b && c && d; ++i)
+  {
+    a = a->next;
+    b = b->next;
+    c = c->next;
+    d = d->next;
+  }
+}
+
+int run_bssn_em_cuda_substep(Block *cg,
+                             MyList<var> *state_in_list,
+                             MyList<var> *state_out_list,
+                             Patch *patch,
+                             double &dT_lev,
+                             double &TRK4,
+                             int &iter_count,
+                             int &Symmetry,
+                             int lev,
+                             double &ndeps,
+                             int &co,
+                             double &chitiny,
+                             var *rho, var *Sx, var *Sy, var *Sz,
+                             var *Sxx, var *Sxy, var *Sxz,
+                             var *Syy, var *Syz, var *Szz)
+{
+  double *state_in[BSSN_CUDA_STATE_COUNT];
+  double *state_out[BSSN_CUDA_STATE_COUNT];
+  double *matter[BSSN_CUDA_MATTER_COUNT] = {
+      cg->fgfs[rho->sgfn], cg->fgfs[Sx->sgfn], cg->fgfs[Sy->sgfn], cg->fgfs[Sz->sgfn],
+      cg->fgfs[Sxx->sgfn], cg->fgfs[Sxy->sgfn], cg->fgfs[Sxz->sgfn],
+      cg->fgfs[Syy->sgfn], cg->fgfs[Syz->sgfn], cg->fgfs[Szz->sgfn]};
+  double propspeed[BSSN_CUDA_STATE_COUNT];
+  double soa_flat[3 * BSSN_CUDA_STATE_COUNT];
+  if (!fill_bssn_cuda_views_prefix(cg, state_in_list, state_in, propspeed, soa_flat) ||
+      !fill_bssn_cuda_views_prefix(cg, state_out_list, state_out))
+    return 1;
+
+  int apply_bam_bc = 0;
+#if (SommerType == 0)
+#ifndef WithShell
+  apply_bam_bc = (lev == 0) ? 1 : 0;
+#endif
+#endif
+  int use_zero_matter = 0;
+  int keep_resident_state = 0;
+  int apply_enforce_ga = 0;
+  return bssn_cuda_rk4_substep(cg,
+                               cg->shape, cg->X[0], cg->X[1], cg->X[2],
+                               state_in, state_out, matter,
+                               propspeed, soa_flat, patch->bbox,
+                               dT_lev, TRK4, iter_count, apply_bam_bc,
+                               Symmetry, lev, ndeps, co,
+                               use_zero_matter,
+                               keep_resident_state, apply_enforce_ga, chitiny);
+}
+
+}
+#endif
+
+//================================================================================================
+
+// Define bssnEM_class
 
 // It inherits some members and methods from the parent class bssn_class and modifies others.
 // The modified members and methods are defined below (and in the header bssnEM_class.h).
@@ -232,19 +335,21 @@ void bssnEM_class::Initialize()
   else
     GH->compose_cgh(nprocs);
 
-#ifdef WithShell
-  SH = new ShellPatch(0, ngfs, pname, Symmetry, myrank, ErrorMonitor);
-  SH->matchcheck(GH->PatL[0]);
+#ifdef WithShell
+  SH = new ShellPatch(0, ngfs, pname, Symmetry, myrank, ErrorMonitor);
+  SH->matchcheck(GH->PatL[0]);
   SH->compose_sh(nprocs);
   SH->setupcordtrans();
   SH->Dump_xyz(0, 0, 1);
   SH->setupintintstuff(nprocs, GH->PatL[0], Symmetry);
 
-  if (checkrun)
-    CheckPoint->readcheck_sh(SH, myrank);
-#endif
-
-  double h = GH->PatL[0]->data->blb->data->getdX(0);
+  if (checkrun)
+    CheckPoint->readcheck_sh(SH, myrank);
+#endif
+
+  Initialize_Level_Runtime();
+
+  double h = GH->PatL[0]->data->blb->data->getdX(0);
   for (int i = 1; i < dim; i++)
     h = Mymin(h, GH->PatL[0]->data->blb->data->getdX(i));
   dT = Courant * h;
@@ -851,10 +956,11 @@ void bssnEM_class::Step(int lev, int YN)
                      cg->fgfs[gyy0->sgfn], cg->fgfs[gyz0->sgfn], cg->fgfs[gzz0->sgfn],
                      cg->fgfs[Axx0->sgfn], cg->fgfs[Axy0->sgfn], cg->fgfs[Axz0->sgfn], 
                      cg->fgfs[Ayy0->sgfn], cg->fgfs[Ayz0->sgfn], cg->fgfs[Azz0->sgfn]);
-#endif
-
-        if (
-            f_compute_rhs_empart(cg->shape, cg->X[0], cg->X[1], cg->X[2],
+#endif
+
+        bool used_gpu_substep = false;
+        if (
+            f_compute_rhs_empart(cg->shape, cg->X[0], cg->X[1], cg->X[2],
                                  cg->fgfs[phi0->sgfn],
                                  cg->fgfs[gxx0->sgfn], cg->fgfs[gxy0->sgfn], cg->fgfs[gxz0->sgfn], 
                                  cg->fgfs[gyy0->sgfn], cg->fgfs[gyz0->sgfn], cg->fgfs[gzz0->sgfn],
@@ -871,11 +977,20 @@ void bssnEM_class::Step(int lev, int YN)
                                  cg->fgfs[Kpsi_rhs->sgfn], cg->fgfs[Kphi_rhs->sgfn],
                                  cg->fgfs[rho->sgfn], 
                                  cg->fgfs[Sx->sgfn], cg->fgfs[Sy->sgfn], cg->fgfs[Sz->sgfn],
-                                 cg->fgfs[Sxx->sgfn], cg->fgfs[Sxy->sgfn], cg->fgfs[Sxz->sgfn], 
-                                 cg->fgfs[Syy->sgfn], cg->fgfs[Syz->sgfn], cg->fgfs[Szz->sgfn],
-                                 Symmetry, lev, ndeps) ||
-            f_compute_rhs_bssn(cg->shape, TRK4, cg->X[0], cg->X[1], cg->X[2],
-                               cg->fgfs[phi0->sgfn], cg->fgfs[trK0->sgfn],
+                                 cg->fgfs[Sxx->sgfn], cg->fgfs[Sxy->sgfn], cg->fgfs[Sxz->sgfn],
+                                 cg->fgfs[Syy->sgfn], cg->fgfs[Syz->sgfn], cg->fgfs[Szz->sgfn],
+                                 Symmetry, lev, ndeps) ||
+#if USE_CUDA_BSSN
+            ((used_gpu_substep =
+                  (run_bssn_em_cuda_substep(cg, StateList, SynchList_pre, Pp->data,
+                                            dT_lev, TRK4, iter_count, Symmetry, lev,
+                                            ndeps, pre, chitiny,
+                                            rho, Sx, Sy, Sz, Sxx, Sxy, Sxz, Syy, Syz, Szz) == 0))
+                 ? 0
+                 : 1) ||
+#endif
+            (!used_gpu_substep && f_compute_rhs_bssn(cg->shape, TRK4, cg->X[0], cg->X[1], cg->X[2],
+                               cg->fgfs[phi0->sgfn], cg->fgfs[trK0->sgfn],
                                cg->fgfs[gxx0->sgfn], cg->fgfs[gxy0->sgfn], cg->fgfs[gxz0->sgfn], 
                                cg->fgfs[gyy0->sgfn], cg->fgfs[gyz0->sgfn], cg->fgfs[gzz0->sgfn],
                                cg->fgfs[Axx0->sgfn], cg->fgfs[Axy0->sgfn], cg->fgfs[Axz0->sgfn], 
@@ -904,10 +1019,10 @@ void bssnEM_class::Step(int lev, int YN)
                                cg->fgfs[Gamzyy->sgfn], cg->fgfs[Gamzyz->sgfn], cg->fgfs[Gamzzz->sgfn],
                                cg->fgfs[Rxx->sgfn], cg->fgfs[Rxy->sgfn], cg->fgfs[Rxz->sgfn], 
                                cg->fgfs[Ryy->sgfn], cg->fgfs[Ryz->sgfn], cg->fgfs[Rzz->sgfn],
-                               cg->fgfs[Cons_Ham->sgfn],
-                               cg->fgfs[Cons_Px->sgfn], cg->fgfs[Cons_Py->sgfn], cg->fgfs[Cons_Pz->sgfn],
-                               cg->fgfs[Cons_Gx->sgfn], cg->fgfs[Cons_Gy->sgfn], cg->fgfs[Cons_Gz->sgfn],
-                               Symmetry, lev, ndeps, pre))
+                               cg->fgfs[Cons_Ham->sgfn],
+                               cg->fgfs[Cons_Px->sgfn], cg->fgfs[Cons_Py->sgfn], cg->fgfs[Cons_Pz->sgfn],
+                               cg->fgfs[Cons_Gx->sgfn], cg->fgfs[Cons_Gy->sgfn], cg->fgfs[Cons_Gz->sgfn],
+                               Symmetry, lev, ndeps, pre)))
         {
           cout << "find NaN in domain: (" 
                << cg->bbox[0] << ":" << cg->bbox[3] << "," 
@@ -917,11 +1032,15 @@ void bssnEM_class::Step(int lev, int YN)
         }
 
         // rk4 substep and boundary
-        {
-          MyList<var> *varl0 = StateList, *varl = SynchList_pre, *varlrhs = RHSList; 
-          // we do not check the correspondence here
-          
-          while (varl0)
+        {
+          MyList<var> *varl0 = StateList, *varl = SynchList_pre, *varlrhs = RHSList;
+          // we do not check the correspondence here
+#if USE_CUDA_BSSN
+          if (used_gpu_substep)
+            skip_bssn_cuda_prefix(varl0, varl, varlrhs);
+#endif
+
+          while (varl0)
           {
 #ifndef WithShell
             if (lev == 0) // sommerfeld indeed
@@ -1221,7 +1340,7 @@ void bssnEM_class::Step(int lev, int YN)
   }
 #endif
 
-  Parallel::Sync(GH->PatL[lev], SynchList_pre, Symmetry);
+  Parallel::Sync_cached(GH->PatL[lev], SynchList_pre, Symmetry, sync_cache_pre[lev]);
 
 #ifdef WithShell
   if (lev == 0)
@@ -1307,10 +1426,11 @@ void bssnEM_class::Step(int lev, int YN)
                          cg->fgfs[gyy->sgfn], cg->fgfs[gyz->sgfn], cg->fgfs[gzz->sgfn],
                          cg->fgfs[Axx->sgfn], cg->fgfs[Axy->sgfn], cg->fgfs[Axz->sgfn], 
                          cg->fgfs[Ayy->sgfn], cg->fgfs[Ayz->sgfn], cg->fgfs[Azz->sgfn]);
-#endif
-
-          if (
-              f_compute_rhs_empart(cg->shape, cg->X[0], cg->X[1], cg->X[2],
+#endif
+
+          bool used_gpu_substep = false;
+          if (
+              f_compute_rhs_empart(cg->shape, cg->X[0], cg->X[1], cg->X[2],
                                    cg->fgfs[phi->sgfn],
                                    cg->fgfs[gxx->sgfn], cg->fgfs[gxy->sgfn], cg->fgfs[gxz->sgfn], 
                                    cg->fgfs[gyy->sgfn], cg->fgfs[gyz->sgfn], cg->fgfs[gzz->sgfn],
@@ -1327,11 +1447,20 @@ void bssnEM_class::Step(int lev, int YN)
                                    cg->fgfs[Kpsi1->sgfn], cg->fgfs[Kphi1->sgfn],
                                    cg->fgfs[rho->sgfn], 
                                    cg->fgfs[Sx->sgfn], cg->fgfs[Sy->sgfn], cg->fgfs[Sz->sgfn],
-                                   cg->fgfs[Sxx->sgfn], cg->fgfs[Sxy->sgfn], cg->fgfs[Sxz->sgfn], 
-                                   cg->fgfs[Syy->sgfn], cg->fgfs[Syz->sgfn], cg->fgfs[Szz->sgfn],
-                                   Symmetry, lev, ndeps) ||
-              f_compute_rhs_bssn(cg->shape, TRK4, cg->X[0], cg->X[1], cg->X[2],
-                                 cg->fgfs[phi->sgfn], cg->fgfs[trK->sgfn],
+                                   cg->fgfs[Sxx->sgfn], cg->fgfs[Sxy->sgfn], cg->fgfs[Sxz->sgfn],
+                                   cg->fgfs[Syy->sgfn], cg->fgfs[Syz->sgfn], cg->fgfs[Szz->sgfn],
+                                   Symmetry, lev, ndeps) ||
+#if USE_CUDA_BSSN
+              ((used_gpu_substep =
+                    (run_bssn_em_cuda_substep(cg, SynchList_pre, SynchList_cor, Pp->data,
+                                              dT_lev, TRK4, iter_count, Symmetry, lev,
+                                              ndeps, cor, chitiny,
+                                              rho, Sx, Sy, Sz, Sxx, Sxy, Sxz, Syy, Syz, Szz) == 0))
+                   ? 0
+                   : 1) ||
+#endif
+              (!used_gpu_substep && f_compute_rhs_bssn(cg->shape, TRK4, cg->X[0], cg->X[1], cg->X[2],
+                                 cg->fgfs[phi->sgfn], cg->fgfs[trK->sgfn],
                                  cg->fgfs[gxx->sgfn], cg->fgfs[gxy->sgfn], cg->fgfs[gxz->sgfn], 
                                  cg->fgfs[gyy->sgfn], cg->fgfs[gyz->sgfn], cg->fgfs[gzz->sgfn],
                                  cg->fgfs[Axx->sgfn], cg->fgfs[Axy->sgfn], cg->fgfs[Axz->sgfn], 
@@ -1359,10 +1488,10 @@ void bssnEM_class::Step(int lev, int YN)
                                  cg->fgfs[Gamzyy->sgfn], cg->fgfs[Gamzyz->sgfn], cg->fgfs[Gamzzz->sgfn],
                                  cg->fgfs[Rxx->sgfn], cg->fgfs[Rxy->sgfn], cg->fgfs[Rxz->sgfn], 
                                  cg->fgfs[Ryy->sgfn], cg->fgfs[Ryz->sgfn], cg->fgfs[Rzz->sgfn],
-                                 cg->fgfs[Cons_Ham->sgfn],
-                                 cg->fgfs[Cons_Px->sgfn], cg->fgfs[Cons_Py->sgfn], cg->fgfs[Cons_Pz->sgfn],
-                                 cg->fgfs[Cons_Gx->sgfn], cg->fgfs[Cons_Gy->sgfn], cg->fgfs[Cons_Gz->sgfn],
-                                 Symmetry, lev, ndeps, cor))
+                                 cg->fgfs[Cons_Ham->sgfn],
+                                 cg->fgfs[Cons_Px->sgfn], cg->fgfs[Cons_Py->sgfn], cg->fgfs[Cons_Pz->sgfn],
+                                 cg->fgfs[Cons_Gx->sgfn], cg->fgfs[Cons_Gy->sgfn], cg->fgfs[Cons_Gz->sgfn],
+                                 Symmetry, lev, ndeps, cor)))
           {
             cout << "find NaN in domain: (" 
                  << cg->bbox[0] << ":" << cg->bbox[3] << "," 
@@ -1371,11 +1500,15 @@ void bssnEM_class::Step(int lev, int YN)
             ERROR = 1;
           }
           // rk4 substep and boundary
-          {
-            MyList<var> *varl0 = StateList, *varl = SynchList_pre, *varl1 = SynchList_cor, *varlrhs = RHSList; 
-            // we do not check the correspondence here
-            
-            while (varl0)
+          {
+            MyList<var> *varl0 = StateList, *varl = SynchList_pre, *varl1 = SynchList_cor, *varlrhs = RHSList;
+            // we do not check the correspondence here
+#if USE_CUDA_BSSN
+            if (used_gpu_substep)
+              skip_bssn_cuda_prefix(varl0, varl, varl1, varlrhs);
+#endif
+
+            while (varl0)
             {
 #ifndef WithShell
               if (lev == 0) // sommerfeld indeed
@@ -1683,7 +1816,7 @@ void bssnEM_class::Step(int lev, int YN)
     }
 #endif
 
-    Parallel::Sync(GH->PatL[lev], SynchList_cor, Symmetry);
+    Parallel::Sync_cached(GH->PatL[lev], SynchList_cor, Symmetry, sync_cache_cor[lev]);
 
 #ifdef WithShell
     if (lev == 0)

AMSS_NCKU_source/bssnEScalar_class.h

@@ -54,17 +54,21 @@ public:
      void Interp_Constraint();
      void Constraint_Out(); 
 
-protected:
-     var *Sphio, *Spio;
-     var *Sphi0, *Spi0;
+protected:
+     var *Sphio, *Spio;
+     var *Sphi0, *Spi0;
      var *Sphi,  *Spi;
      var *Sphi1, *Spi1;
      var *Sphi_rhs, *Spi_rhs;
-
-     var *Cons_fR;
-
-     monitor *MaxScalar_Monitor;
-};
+
+     var *Cons_fR;
+
+     MyList<var> *BSSNStateList, *BSSNSynchList_pre, *BSSNSynchList_cor;
+     MyList<var> *ScalarSynchList_pre, *ScalarSynchList_cor;
+     Parallel::SyncCache *sync_cache_scalar_pre, *sync_cache_scalar_cor;
+
+     monitor *MaxScalar_Monitor;
+};
 
 #endif /* BSSNESCALAR_CLASS_H */
 

AMSS_NCKU_source/bssnEScalar_rhs.f90

@@ -3,11 +3,143 @@
 !! note that the potential for scalar field in F(R) gravity
 !!       is defined in the file Set_Rho_ADM.f90
 
-#include "macrodef.fh"
-
-! rhs for scalar and GR variables
-! here we consider vacuum spacetime only
-  function compute_rhs_bssn_escalar(ex, T,X, Y, Z,                             &
+#include "macrodef.fh"
+
+! scalar RHS and stress-energy only; BSSN RHS can be supplied by CUDA.
+  function compute_rhs_bssn_escalar_matter(ex, T, X, Y, Z,                    &
+               chi    ,   trK    ,                                            &
+               dxx    ,   gxy    ,   gxz    ,   dyy    ,   gyz    ,   dzz,    &
+               Axx    ,   Axy    ,   Axz    ,   Ayy    ,   Ayz    ,   Azz,    &
+               Gamx   ,  Gamy    ,  Gamz    ,                                 &
+               Lap    ,  betax   ,  betay   ,  betaz   ,                      &
+               dtSfx  ,  dtSfy   ,  dtSfz   ,                                 &
+               Sphi   ,   Spi    ,                                            &
+               Sphi_rhs ,   Spi_rhs ,                                         &
+               rho,Sx,Sy,Sz,Sxx,Sxy,Sxz,Syy,Syz,Szz,                          &
+               Symmetry,Lev,eps) result(gont)
+  implicit none
+
+  integer,intent(in ):: ex(1:3), Symmetry,Lev
+  real*8, intent(in ):: T
+  real*8, intent(in ):: X(1:ex(1)),Y(1:ex(2)),Z(1:ex(3))
+  real*8, dimension(ex(1),ex(2),ex(3)),intent(inout) :: chi,dxx,dyy,dzz
+  real*8, dimension(ex(1),ex(2),ex(3)),intent(in ) :: trK
+  real*8, dimension(ex(1),ex(2),ex(3)),intent(in ) :: gxy,gxz,gyz
+  real*8, dimension(ex(1),ex(2),ex(3)),intent(in ) :: Axx,Axy,Axz,Ayy,Ayz,Azz
+  real*8, dimension(ex(1),ex(2),ex(3)),intent(in ) :: Gamx,Gamy,Gamz
+  real*8, dimension(ex(1),ex(2),ex(3)),intent(inout) :: Lap, betax, betay, betaz
+  real*8, dimension(ex(1),ex(2),ex(3)),intent(in ) :: dtSfx,  dtSfy,  dtSfz
+  real*8, dimension(ex(1),ex(2),ex(3)),intent(in ) :: Sphi,Spi
+  real*8, dimension(ex(1),ex(2),ex(3)),intent(out) :: Sphi_rhs,Spi_rhs
+  real*8, dimension(ex(1),ex(2),ex(3)),intent(inout) :: rho,Sx,Sy,Sz
+  real*8, dimension(ex(1),ex(2),ex(3)),intent(inout) :: Sxx,Sxy,Sxz,Syy,Syz,Szz
+  real*8,intent(in) :: eps
+  integer::gont
+
+  real*8, dimension(ex(1),ex(2),ex(3)) :: gxx,gyy,gzz
+  real*8, dimension(ex(1),ex(2),ex(3)) :: chix,chiy,chiz
+  real*8, dimension(ex(1),ex(2),ex(3)) :: Lapx,Lapy,Lapz
+  real*8, dimension(ex(1),ex(2),ex(3)) :: Kx,Ky,Kz,S
+  real*8, dimension(ex(1),ex(2),ex(3)) :: f,fxx,fxy,fxz,fyy,fyz,fzz
+  real*8, dimension(ex(1),ex(2),ex(3)) :: alpn1,chin1
+  real*8, dimension(ex(1),ex(2),ex(3)) :: gupxx,gupxy,gupxz
+  real*8, dimension(ex(1),ex(2),ex(3)) :: gupyy,gupyz,gupzz
+
+  real*8            :: dX
+  real*8, parameter :: ZEO=0.d0, ONE = 1.D0, TWO = 2.D0, HALF = 0.5D0
+  real*8, parameter :: SYM = 1.D0
+
+  dX = sum(chi)+sum(trK)+sum(dxx)+sum(gxy)+sum(gxz)+sum(dyy)+sum(gyz)+sum(dzz) &
+      +sum(Gamx)+sum(Gamy)+sum(Gamz)                                           &
+      +sum(Lap)+sum(Sphi)+sum(Spi)
+  if(dX.ne.dX) then
+     if(sum(chi).ne.sum(chi))write(*,*)"bssn_escalar_matter: find NaN in chi"
+     if(sum(trK).ne.sum(trK))write(*,*)"bssn_escalar_matter: find NaN in trk"
+     if(sum(dxx).ne.sum(dxx))write(*,*)"bssn_escalar_matter: find NaN in dxx"
+     if(sum(gxy).ne.sum(gxy))write(*,*)"bssn_escalar_matter: find NaN in gxy"
+     if(sum(gxz).ne.sum(gxz))write(*,*)"bssn_escalar_matter: find NaN in gxz"
+     if(sum(dyy).ne.sum(dyy))write(*,*)"bssn_escalar_matter: find NaN in dyy"
+     if(sum(gyz).ne.sum(gyz))write(*,*)"bssn_escalar_matter: find NaN in gyz"
+     if(sum(dzz).ne.sum(dzz))write(*,*)"bssn_escalar_matter: find NaN in dzz"
+     if(sum(Gamx).ne.sum(Gamx))write(*,*)"bssn_escalar_matter: find NaN in Gamx"
+     if(sum(Gamy).ne.sum(Gamy))write(*,*)"bssn_escalar_matter: find NaN in Gamy"
+     if(sum(Gamz).ne.sum(Gamz))write(*,*)"bssn_escalar_matter: find NaN in Gamz"
+     if(sum(Lap).ne.sum(Lap))write(*,*)"bssn_escalar_matter: find NaN in Lap"
+     if(sum(Sphi).ne.sum(Sphi))write(*,*)"bssn_escalar_matter: find NaN in Sphi"
+     if(sum(Spi).ne.sum(Spi))write(*,*)"bssn_escalar_matter: find NaN in Spi"
+     gont = 1
+     return
+  endif
+
+  alpn1 = Lap + ONE
+  chin1 = chi + ONE
+  gxx = dxx + ONE
+  gyy = dyy + ONE
+  gzz = dzz + ONE
+
+  call fderivs(ex,chi,chix,chiy,chiz,X,Y,Z,SYM,SYM,SYM,Symmetry,Lev)
+  call fderivs(ex,Lap,Lapx,Lapy,Lapz,X,Y,Z,SYM,SYM,SYM,Symmetry,Lev)
+
+  gupzz =  gxx * gyy * gzz + gxy * gyz * gxz + gxz * gxy * gyz - &
+           gxz * gyy * gxz - gxy * gxy * gzz - gxx * gyz * gyz
+  gupxx =   ( gyy * gzz - gyz * gyz ) / gupzz
+  gupxy = - ( gxy * gzz - gyz * gxz ) / gupzz
+  gupxz =   ( gxy * gyz - gyy * gxz ) / gupzz
+  gupyy =   ( gxx * gzz - gxz * gxz ) / gupzz
+  gupyz = - ( gxx * gyz - gxy * gxz ) / gupzz
+  gupzz =   ( gxx * gyy - gxy * gxy ) / gupzz
+
+#if 1
+  Sphi_rhs = alpn1 * Spi
+  call fderivs(ex,Sphi,Kx,Ky,Kz,X,Y,Z,SYM,SYM,SYM,Symmetry,Lev)
+  call fdderivs(ex,Sphi,fxx,fxy,fxz,fyy,fyz,fzz,X,Y,Z,SYM,SYM,SYM,Symmetry,Lev)
+  Spi_rhs =   gupxx * fxx + gupyy * fyy + gupzz * fzz +           &
+             ( gupxy * fxy + gupxz * fxz + gupyz * fyz ) * TWO -  &
+             ((Gamx+(gupxx*chix+gupxy*chiy+gupxz*chiz)/TWO/chin1)*Kx  &
+            + (Gamy+(gupxy*chix+gupyy*chiy+gupyz*chiz)/TWO/chin1)*Ky  &
+            + (Gamz+(gupxz*chix+gupyz*chiy+gupzz*chiz)/TWO/chin1)*Kz)
+  Spi_rhs = Spi_rhs*alpn1 +                                      &
+            (gupxx*Lapx*Kx + gupxy*Lapx*Ky + gupxz*Lapx*Kz       &
+            +gupxy*Lapy*Kx + gupyy*Lapy*Ky + gupyz*Lapy*Kz       &
+            +gupxz*Lapz*Kx + gupyz*Lapz*Ky + gupzz*Lapz*Kz)
+
+  call frpotential(ex,Sphi,f,S)
+  Spi_rhs = Spi_rhs*chin1 + alpn1*(trK*Spi - S)
+  rho = chin1*((gupxx * Kx * Kx + gupyy * Ky * Ky + gupzz * Kz * Kz)/TWO + &
+                gupxy * Kx * Ky + gupxz * Kx * Kz + gupyz * Ky * Kz )      &
+        + Spi*Spi/TWO+f
+  Sx = -Spi*Kx
+  Sy = -Spi*Ky
+  Sz = -Spi*Kz
+  f = (rho - Spi*Spi)/chin1
+  Sxx = Kx*Kx-f*gxx
+  Sxy = Kx*Ky-f*gxy
+  Sxz = Kx*Kz-f*gxz
+  Syy = Ky*Ky-f*gyy
+  Syz = Ky*Kz-f*gyz
+  Szz = Kz*Kz-f*gzz
+#else
+  Sphi_rhs = ZEO
+  Spi_rhs = ZEO
+  rho = ZEO
+  Sx = ZEO
+  Sy = ZEO
+  Sz = ZEO
+  Sxx = ZEO
+  Sxy = ZEO
+  Sxz = ZEO
+  Syy = ZEO
+  Syz = ZEO
+  Szz = ZEO
+#endif
+
+  gont = 0
+  return
+end function compute_rhs_bssn_escalar_matter
+
+! rhs for scalar and GR variables
+! here we consider vacuum spacetime only
+  function compute_rhs_bssn_escalar(ex, T,X, Y, Z,                             &
                chi    ,   trK    ,                                             &
                dxx    ,   gxy    ,   gxz    ,   dyy    ,   gyz    ,   dzz,     &
                Axx    ,   Axy    ,   Axz    ,   Ayy    ,   Ayz    ,   Azz,     &

AMSS_NCKU_source/bssn_class.h

@@ -45,10 +45,12 @@ public:
        int checkrun;
        char checkfilename[50];
        int Steps;
-       double StartTime, TotalTime;
-       double AnasTime, DumpTime, d2DumpTime, CheckTime;
-       double LastAnas, LastConsOut;
-       double Courant;
+       double StartTime, TotalTime;
+       double AnasTime, DumpTime, d2DumpTime, CheckTime;
+       double LastAnas, LastConsOut;
+       bool cuda_level0_constraint_cache_valid;
+       int *ConstraintRefreshLevels;
+       double Courant;
        double numepss, numepsb, numepsh;
        int Symmetry;
        int maxl, decn;
@@ -126,9 +128,16 @@ public:
        MyList<var> *OldStateList, *DumpList;
        MyList<var> *ConstraintList;
 
-       monitor *ErrorMonitor, *Psi4Monitor, *BHMonitor, *MAPMonitor;
-       monitor *ConVMonitor;
-       surface_integral *Waveshell;
+       Parallel::SyncCache *sync_cache_pre;  // per-level cache for predictor sync
+       Parallel::SyncCache *sync_cache_cor;  // per-level cache for corrector sync
+       Parallel::SyncCache *sync_cache_rp_coarse;  // RestrictProlong sync on PatL[lev-1]
+       Parallel::SyncCache *sync_cache_rp_fine;    // RestrictProlong sync on PatL[lev]
+       Parallel::SyncCache *sync_cache_restrict;   // cached Restrict in RestrictProlong
+       Parallel::SyncCache *sync_cache_outbd;      // cached OutBdLow2Hi in RestrictProlong
+
+       monitor *ErrorMonitor, *Psi4Monitor, *BHMonitor, *MAPMonitor;
+       monitor *ConVMonitor, *TimingMonitor;
+       surface_integral *Waveshell;
        checkpoint *CheckPoint;
 
 public:
@@ -169,14 +178,17 @@ public:
        virtual void Initialize();
        virtual void Read_Ansorg();
        virtual void Read_Pablo() {};
-       virtual void Compute_Psi4(int lev);
-       virtual void Step(int lev, int YN);
-       virtual void Interp_Constraint(bool infg);
-       virtual void Constraint_Out();
-       virtual void Compute_Constraint();
-
-#ifdef With_AHF
-protected:
+       virtual void Compute_Psi4(int lev);
+       virtual void Step(int lev, int YN);
+       virtual void Interp_Constraint(bool infg);
+       virtual void Constraint_Out();
+       virtual void Compute_Constraint();
+
+protected:
+       void Initialize_Level_Runtime();
+
+#ifdef With_AHF
+protected:
        MyList<var> *AHList, *AHDList, *GaugeList;
        int AHfindevery;
        double AHdumptime;

AMSS_NCKU_source/bssn_gpu.h

@@ -1,73 +0,0 @@
-
-#ifndef BSSN_GPU_H_
-#define BSSN_GPU_H_
-#include "bssn_macro.h"
-#include "macrodef.fh"
-
-#define DEVICE_ID 0
-// #define DEVICE_ID_BY_MPI_RANK
-#define GRID_DIM 256
-#define BLOCK_DIM 128
-
-#define _FH2_(i, j, k) fh[(i) + (j) * _1D_SIZE[2] + (k) * _2D_SIZE[2]]
-#define _FH3_(i, j, k) fh[(i) + (j) * _1D_SIZE[3] + (k) * _2D_SIZE[3]]
-#define pow2(x) ((x) * (x))
-#define TimeBetween(a, b) ((b.tv_sec - a.tv_sec) + (b.tv_usec - a.tv_usec) / 1000000.0f)
-#define M_ metac.
-#define Mh_ meta->
-#define Ms_ metassc.
-#define Msh_ metass->
-
-// #define TIMING
-
-#define RHS_SS_PARA int calledby, int mpi_rank, int *ex, double &T, double *crho, double *sigma, double *R, double *X, double *Y, double *Z, double *drhodx, double *drhody, double *drhodz, double *dsigmadx, double *dsigmady, double *dsigmadz, double *dRdx, double *dRdy, double *dRdz, double *drhodxx, double *drhodxy, double *drhodxz, double *drhodyy, double *drhodyz, double *drhodzz, double *dsigmadxx, double *dsigmadxy, double *dsigmadxz, double *dsigmadyy, double *dsigmadyz, double *dsigmadzz, double *dRdxx, double *dRdxy, double *dRdxz, double *dRdyy, double *dRdyz, double *dRdzz, double *chi, double *trK, double *dxx, double *gxy, double *gxz, double *dyy, double *gyz, double *dzz, double *Axx, double *Axy, double *Axz, double *Ayy, double *Ayz, double *Azz, double *Gamx, double *Gamy, double *Gamz, double *Lap, double *betax, double *betay, double *betaz, double *dtSfx, double *dtSfy, double *dtSfz, double *chi_rhs, double *trK_rhs, double *gxx_rhs, double *gxy_rhs, double *gxz_rhs, double *gyy_rhs, double *gyz_rhs, double *gzz_rhs, double *Axx_rhs, double *Axy_rhs, double *Axz_rhs, double *Ayy_rhs, double *Ayz_rhs, double *Azz_rhs, double *Gamx_rhs, double *Gamy_rhs, double *Gamz_rhs, double *Lap_rhs, double *betax_rhs, double *betay_rhs, double *betaz_rhs, double *dtSfx_rhs, double *dtSfy_rhs, double *dtSfz_rhs, double *rho, double *Sx, double *Sy, double *Sz, double *Sxx, double *Sxy, double *Sxz, double *Syy, double *Syz, double *Szz, double *Gamxxx, double *Gamxxy, double *Gamxxz, double *Gamxyy, double *Gamxyz, double *Gamxzz, double *Gamyxx, double *Gamyxy, double *Gamyxz, double *Gamyyy, double *Gamyyz, double *Gamyzz, double *Gamzxx, double *Gamzxy, double *Gamzxz, double *Gamzyy, double *Gamzyz, double *Gamzzz, double *Rxx, double *Rxy, double *Rxz, double *Ryy, double *Ryz, double *Rzz, double *ham_Res, double *movx_Res, double *movy_Res, double *movz_Res, double *Gmx_Res, double *Gmy_Res, double *Gmz_Res, int &Symmetry, int &Lev, double &eps, int &sst, int &co
-
-/**  main function */
-int gpu_rhs(int calledby, int mpi_rank, int *ex, double &T,
-            double *X, double *Y, double *Z,
-
-            double *chi, double *trK,
-
-            double *dxx, double *gxy, double *gxz, double *dyy, double *gyz, double *dzz,
-
-            double *Axx, double *Axy, double *Axz, double *Ayy, double *Ayz, double *Azz,
-
-            double *Gamx, double *Gamy, double *Gamz,
-
-            double *Lap, double *betax, double *betay, double *betaz,
-
-            double *dtSfx, double *dtSfy, double *dtSfz,
-
-            double *chi_rhs, double *trK_rhs,
-
-            double *gxx_rhs, double *gxy_rhs, double *gxz_rhs, double *gyy_rhs, double *gyz_rhs, double *gzz_rhs,
-
-            double *Axx_rhs, double *Axy_rhs, double *Axz_rhs, double *Ayy_rhs, double *Ayz_rhs, double *Azz_rhs,
-
-            double *Gamx_rhs, double *Gamy_rhs, double *Gamz_rhs,
-
-            double *Lap_rhs, double *betax_rhs, double *betay_rhs, double *betaz_rhs,
-
-            double *dtSfx_rhs, double *dtSfy_rhs, double *dtSfz_rhs,
-
-            double *rho, double *Sx, double *Sy, double *Sz, double *Sxx,
-            double *Sxy, double *Sxz, double *Syy, double *Syz, double *Szz,
-
-            double *Gamxxx, double *Gamxxy, double *Gamxxz, double *Gamxyy, double *Gamxyz, double *Gamxzz,
-
-            double *Gamyxx, double *Gamyxy, double *Gamyxz, double *Gamyyy, double *Gamyyz, double *Gamyzz,
-
-            double *Gamzxx, double *Gamzxy, double *Gamzxz, double *Gamzyy, double *Gamzyz, double *Gamzzz,
-
-            double *Rxx, double *Rxy, double *Rxz, double *Ryy, double *Ryz, double *Rzz,
-
-            double *ham_Res, double *movx_Res, double *movy_Res, double *movz_Res,
-            double *Gmx_Res, double *Gmy_Res, double *Gmz_Res,
-            int &Symmetry, int &Lev, double &eps, int &co);
-
-int gpu_rhs_ss(RHS_SS_PARA);
-
-/** Init GPU side data in GPUMeta. */
-// void init_fluid_meta_gpu(GPUMeta *gpu_meta);
-
-#endif

AMSS_NCKU_source/bssn_gpu_class.h

@@ -1,210 +0,0 @@
-
-#ifndef BSSN_GPU_CLASS_H
-#define BSSN_GPU_CLASS_H
-
-#ifdef newc
-#include <iostream>
-#include <iomanip>
-#include <fstream>
-#include <cstdlib>
-#include <string>
-#include <cmath>
-using namespace std;
-#else
-#include <iostream.h>
-#include <iomanip.h>
-#include <fstream.h>
-#include <stdlib.h>
-#include <string.h>
-#include <math.h>
-#endif
-
-#include <mpi.h>
-
-#include "macrodef.h"
-#include "cgh.h"
-#include "ShellPatch.h"
-#include "misc.h"
-#include "var.h"
-#include "MyList.h"
-#include "monitor.h"
-#include "surface_integral.h"
-#include "checkpoint.h"
-
-// added by yangquan
-#include "bssn_macro.h"
-
-extern void setpbh(int iBHN, double **iPBH, double *iMass, int rBHN);
-
-class bssn_class
-{
-public:
-       // added by yangquan
-       //----------------------
-       int gpu_num_mynode;
-       int cpu_core_num_mynode;
-       int mpi_process_num_mynode;
-       int my_sequence_mynode;
-       int mynode_id;
-       int use_gpu;
-
-       virtual void Step_GPU(int lev, int YN);
-       virtual void Get_runtime_envirment();
-       // virtual void Step_OPENMP(int lev,int YN);
-       //----------------------
-
-       int ngfs;
-       int nprocs, myrank;
-       cgh *GH;
-       ShellPatch *SH;
-       double PhysTime;
-
-       int checkrun;
-       char checkfilename[50];
-       int Steps;
-       double StartTime, TotalTime;
-       double AnasTime, DumpTime, d2DumpTime, CheckTime;
-       double LastAnas, LastConsOut;
-       double Courant;
-       double numepss, numepsb, numepsh;
-       int Symmetry;
-       int maxl, decn;
-       double maxrex, drex;
-       int trfls, a_lev;
-
-       double dT;
-       double chitiny;
-
-       double **Porg0, **Porgbr, **Porg, **Porg1, **Porg_rhs;
-       int BH_num, BH_num_input;
-       double *Mass, *Pmom, *Spin;
-       double ADMMass;
-
-       var *phio, *trKo;
-       var *gxxo, *gxyo, *gxzo, *gyyo, *gyzo, *gzzo;
-       var *Axxo, *Axyo, *Axzo, *Ayyo, *Ayzo, *Azzo;
-       var *Gmxo, *Gmyo, *Gmzo;
-       var *Lapo, *Sfxo, *Sfyo, *Sfzo;
-       var *dtSfxo, *dtSfyo, *dtSfzo;
-
-       var *phi0, *trK0;
-       var *gxx0, *gxy0, *gxz0, *gyy0, *gyz0, *gzz0;
-       var *Axx0, *Axy0, *Axz0, *Ayy0, *Ayz0, *Azz0;
-       var *Gmx0, *Gmy0, *Gmz0;
-       var *Lap0, *Sfx0, *Sfy0, *Sfz0;
-       var *dtSfx0, *dtSfy0, *dtSfz0;
-
-       var *phi, *trK;
-       var *gxx, *gxy, *gxz, *gyy, *gyz, *gzz;
-       var *Axx, *Axy, *Axz, *Ayy, *Ayz, *Azz;
-       var *Gmx, *Gmy, *Gmz;
-       var *Lap, *Sfx, *Sfy, *Sfz;
-       var *dtSfx, *dtSfy, *dtSfz;
-
-       var *phi1, *trK1;
-       var *gxx1, *gxy1, *gxz1, *gyy1, *gyz1, *gzz1;
-       var *Axx1, *Axy1, *Axz1, *Ayy1, *Ayz1, *Azz1;
-       var *Gmx1, *Gmy1, *Gmz1;
-       var *Lap1, *Sfx1, *Sfy1, *Sfz1;
-       var *dtSfx1, *dtSfy1, *dtSfz1;
-
-       var *phi_rhs, *trK_rhs;
-       var *gxx_rhs, *gxy_rhs, *gxz_rhs, *gyy_rhs, *gyz_rhs, *gzz_rhs;
-       var *Axx_rhs, *Axy_rhs, *Axz_rhs, *Ayy_rhs, *Ayz_rhs, *Azz_rhs;
-       var *Gmx_rhs, *Gmy_rhs, *Gmz_rhs;
-       var *Lap_rhs, *Sfx_rhs, *Sfy_rhs, *Sfz_rhs;
-       var *dtSfx_rhs, *dtSfy_rhs, *dtSfz_rhs;
-
-       var *rho, *Sx, *Sy, *Sz, *Sxx, *Sxy, *Sxz, *Syy, *Syz, *Szz;
-
-       var *Gamxxx, *Gamxxy, *Gamxxz, *Gamxyy, *Gamxyz, *Gamxzz;
-       var *Gamyxx, *Gamyxy, *Gamyxz, *Gamyyy, *Gamyyz, *Gamyzz;
-       var *Gamzxx, *Gamzxy, *Gamzxz, *Gamzyy, *Gamzyz, *Gamzzz;
-
-       var *Rxx, *Rxy, *Rxz, *Ryy, *Ryz, *Rzz;
-
-       var *Rpsi4, *Ipsi4;
-       var *t1Rpsi4, *t1Ipsi4, *t2Rpsi4, *t2Ipsi4;
-
-       var *Cons_Ham, *Cons_Px, *Cons_Py, *Cons_Pz, *Cons_Gx, *Cons_Gy, *Cons_Gz;
-
-#ifdef Point_Psi4
-       var *phix, *phiy, *phiz;
-       var *trKx, *trKy, *trKz;
-       var *Axxx, *Axxy, *Axxz;
-       var *Axyx, *Axyy, *Axyz;
-       var *Axzx, *Axzy, *Axzz;
-       var *Ayyx, *Ayyy, *Ayyz;
-       var *Ayzx, *Ayzy, *Ayzz;
-       var *Azzx, *Azzy, *Azzz;
-#endif
-       // FIXME: uc = StateList, up = OldStateList, upp = SynchList_cor; so never touch these three data
-       MyList<var> *StateList, *SynchList_pre, *SynchList_cor, *RHSList;
-       MyList<var> *OldStateList, *DumpList;
-       MyList<var> *ConstraintList;
-
-       monitor *ErrorMonitor, *Psi4Monitor, *BHMonitor, *MAPMonitor;
-       monitor *ConVMonitor;
-       surface_integral *Waveshell;
-       checkpoint *CheckPoint;
-
-public:
-       bssn_class(double Couranti, double StartTimei, double TotalTimei, double DumpTimei, double d2DumpTimei, double CheckTimei, double AnasTimei,
-                  int Symmetryi, int checkruni, char *checkfilenamei, double numepssi, double numepsbi, double numepshi,
-                  int a_levi, int maxli, int decni, double maxrexi, double drexi);
-       ~bssn_class();
-
-       void Evolve(int Steps);
-       void RecursiveStep(int lev);
-#if (PSTR == 1)
-       void ParallelStep();
-       void SHStep();
-#endif
-       void RestrictProlong(int lev, int YN, bool BB, MyList<var> *SL, MyList<var> *OL, MyList<var> *corL);
-       void RestrictProlong_aux(int lev, int YN, bool BB, MyList<var> *SL, MyList<var> *OL, MyList<var> *corL);
-       void RestrictProlong(int lev, int YN, bool BB);
-       void ProlongRestrict(int lev, int YN, bool BB);
-       void Setup_Black_Hole_position();
-       void compute_Porg_rhs(double **BH_PS, double **BH_RHS, var *forx, var *fory, var *forz, int lev);
-       bool read_Pablo_file(int *ext, double *datain, char *filename);
-       void write_Pablo_file(int *ext, double xmin, double xmax, double ymin, double ymax, double zmin, double zmax,
-                             char *filename);
-       void AnalysisStuff(int lev, double dT_lev);
-       void Setup_KerrSchild();
-       void Enforce_algcon(int lev, int fg);
-
-       void testRestrict();
-       void testOutBd();
-
-       virtual void Setup_Initial_Data_Lousto();
-       virtual void Setup_Initial_Data_Cao();
-       virtual void Initialize();
-       virtual void Read_Ansorg();
-       virtual void Read_Pablo() {};
-       virtual void Compute_Psi4(int lev);
-       virtual void Step(int lev, int YN);
-       virtual void Interp_Constraint(bool infg);
-       virtual void Constraint_Out();
-       virtual void Compute_Constraint();
-
-#ifdef With_AHF
-protected:
-       MyList<var> *AHList, *AHDList, *GaugeList;
-       int AHfindevery;
-       double AHdumptime;
-       int *lastahdumpid, HN_num; // number of possible horizons
-       int *findeveryl;
-       double *xc, *yc, *zc, *xr, *yr, *zr;
-       bool *trigger;
-       double *dTT;
-       int *dumpid;
-
-public:
-       void AH_Prepare_derivatives();
-       bool AH_Interp_Points(MyList<var> *VarList,
-                             int NN, double **XX,
-                             double *Shellf, int Symmetryi);
-       void AH_Step_Find(int lev, double dT_lev);
-#endif
-};
-#endif /* BSSN_GPU_CLASS_H */

AMSS_NCKU_source/bssn_rhs.h

@@ -5,8 +5,9 @@
 #ifdef fortran1
 #define f_compute_rhs_bssn compute_rhs_bssn
 #define f_compute_rhs_bssn_ss compute_rhs_bssn_ss
-#define f_compute_rhs_bssn_escalar compute_rhs_bssn_escalar
-#define f_compute_rhs_bssn_escalar_ss compute_rhs_bssn_escalar_ss
+#define f_compute_rhs_bssn_escalar compute_rhs_bssn_escalar
+#define f_compute_rhs_bssn_escalar_matter compute_rhs_bssn_escalar_matter
+#define f_compute_rhs_bssn_escalar_ss compute_rhs_bssn_escalar_ss
 #define f_compute_rhs_Z4c compute_rhs_z4c
 #define f_compute_rhs_Z4cnot compute_rhs_z4cnot
 #define f_compute_rhs_Z4c_ss compute_rhs_z4c_ss
@@ -15,26 +16,41 @@
 #ifdef fortran2
 #define f_compute_rhs_bssn COMPUTE_RHS_BSSN
 #define f_compute_rhs_bssn_ss COMPUTE_RHS_BSSN_SS
-#define f_compute_rhs_bssn_escalar COMPUTE_RHS_BSSN_ESCALAR
-#define f_compute_rhs_bssn_escalar_ss COMPUTE_RHS_BSSN_ESCALAR_SS
+#define f_compute_rhs_bssn_escalar COMPUTE_RHS_BSSN_ESCALAR
+#define f_compute_rhs_bssn_escalar_matter COMPUTE_RHS_BSSN_ESCALAR_MATTER
+#define f_compute_rhs_bssn_escalar_ss COMPUTE_RHS_BSSN_ESCALAR_SS
 #define f_compute_rhs_Z4c COMPUTE_RHS_Z4C
 #define f_compute_rhs_Z4cnot COMPUTE_RHS_Z4CNOT
 #define f_compute_rhs_Z4c_ss COMPUTE_RHS_Z4C_SS
 #define f_compute_constraint_fr COMPUTE_CONSTRAINT_FR
 #endif
-#ifdef fortran3
-#define f_compute_rhs_bssn compute_rhs_bssn_
+#ifdef fortran3
+#define f_compute_rhs_bssn compute_rhs_bssn_
 #define f_compute_rhs_bssn_ss compute_rhs_bssn_ss_
-#define f_compute_rhs_bssn_escalar compute_rhs_bssn_escalar_
-#define f_compute_rhs_bssn_escalar_ss compute_rhs_bssn_escalar_ss_
+#define f_compute_rhs_bssn_escalar compute_rhs_bssn_escalar_
+#define f_compute_rhs_bssn_escalar_matter compute_rhs_bssn_escalar_matter_
+#define f_compute_rhs_bssn_escalar_ss compute_rhs_bssn_escalar_ss_
 #define f_compute_rhs_Z4c compute_rhs_z4c_
 #define f_compute_rhs_Z4cnot compute_rhs_z4cnot_
 #define f_compute_rhs_Z4c_ss compute_rhs_z4c_ss_
-#define f_compute_constraint_fr compute_constraint_fr_
-#endif
-extern "C"
-{
-        int f_compute_rhs_bssn(int *, double &, double *, double *, double *,                                                      // ex,T,X,Y,Z
+#define f_compute_constraint_fr compute_constraint_fr_
+#endif
+
+#ifdef __cplusplus
+extern "C"
+{
+#endif
+        void f_bssn_rhs_kernel_timing_reset();
+        int f_bssn_rhs_kernel_timing_bucket_count();
+        const double *f_bssn_rhs_kernel_timing_local_seconds();
+        const char *f_bssn_rhs_kernel_timing_label(int);
+#ifdef __cplusplus
+}
+#endif
+
+extern "C"
+{
+        int f_compute_rhs_bssn(int *, double &, double *, double *, double *,                                                      // ex,T,X,Y,Z
                                double *, double *,                                                                                 // chi, trK
                                double *, double *, double *, double *, double *, double *,                                         // gij
                                double *, double *, double *, double *, double *, double *,                                         // Aij
@@ -83,10 +99,24 @@ extern "C"
                                   int &, int &, double &, int &, int &);
 }
 
-extern "C"
-{
-        int f_compute_rhs_bssn_escalar(int *, double &, double *, double *, double *,                                                      // ex,T,X,Y,Z
-                                       double *, double *,                                                                                 // chi, trK
+extern "C"
+{
+        int f_compute_rhs_bssn_escalar_matter(int *, double &, double *, double *, double *,                                             // ex,T,X,Y,Z
+                                              double *, double *,                                                                        // chi, trK
+                                              double *, double *, double *, double *, double *, double *,                                // gij
+                                              double *, double *, double *, double *, double *, double *,                                // Aij
+                                              double *, double *, double *,                                                              // Gam
+                                              double *, double *, double *, double *, double *, double *, double *,                      // Gauge
+                                              double *, double *,                                                                        // Sphi, Spi
+                                              double *, double *,                                                                        // Sphi, Spi rhs
+                                              double *, double *, double *, double *, double *, double *, double *, double *, double *, double *, // stress-energy
+                                              int &, int &, double &);
+}
+
+extern "C"
+{
+        int f_compute_rhs_bssn_escalar(int *, double &, double *, double *, double *,                                                      // ex,T,X,Y,Z
+                                       double *, double *,                                                                                 // chi, trK
                                        double *, double *, double *, double *, double *, double *,                                         // gij
                                        double *, double *, double *, double *, double *, double *,                                         // Aij
                                        double *, double *, double *,                                                                       // Gam

AMSS_NCKU_source/bssn_rhs_cuda.h

@@ -0,0 +1,470 @@
+#ifndef BSSN_RHS_CUDA_H
+#define BSSN_RHS_CUDA_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+enum {
+    BSSN_CUDA_STATE_COUNT = 24,
+    BSSN_CUDA_MATTER_COUNT = 10
+};
+
+int f_compute_rhs_bssn(int *ex, double &T,
+                       double *X, double *Y, double *Z,
+                       double *chi, double *trK,
+                       double *dxx, double *gxy, double *gxz, double *dyy, double *gyz, double *dzz,
+                       double *Axx, double *Axy, double *Axz, double *Ayy, double *Ayz, double *Azz,
+                       double *Gamx, double *Gamy, double *Gamz,
+                       double *Lap, double *betax, double *betay, double *betaz,
+                       double *dtSfx, double *dtSfy, double *dtSfz,
+                       double *chi_rhs, double *trK_rhs,
+                       double *gxx_rhs, double *gxy_rhs, double *gxz_rhs, double *gyy_rhs, double *gyz_rhs, double *gzz_rhs,
+                       double *Axx_rhs, double *Axy_rhs, double *Axz_rhs, double *Ayy_rhs, double *Ayz_rhs, double *Azz_rhs,
+                       double *Gamx_rhs, double *Gamy_rhs, double *Gamz_rhs,
+                       double *Lap_rhs, double *betax_rhs, double *betay_rhs, double *betaz_rhs,
+                       double *dtSfx_rhs, double *dtSfy_rhs, double *dtSfz_rhs,
+                       double *rho, double *Sx, double *Sy, double *Sz,
+                       double *Sxx, double *Sxy, double *Sxz, double *Syy, double *Syz, double *Szz,
+                       double *Gamxxx, double *Gamxxy, double *Gamxxz, double *Gamxyy, double *Gamxyz, double *Gamxzz,
+                       double *Gamyxx, double *Gamyxy, double *Gamyxz, double *Gamyyy, double *Gamyyz, double *Gamyzz,
+                       double *Gamzxx, double *Gamzxy, double *Gamzxz, double *Gamzyy, double *Gamzyz, double *Gamzzz,
+                       double *Rxx, double *Rxy, double *Rxz, double *Ryy, double *Ryz, double *Rzz,
+                       double *ham_Res, double *movx_Res, double *movy_Res, double *movz_Res,
+                       double *Gmx_Res, double *Gmy_Res, double *Gmz_Res,
+                       int &Symmetry, int &Lev, double &eps, int &co);
+
+int bssn_cuda_rk4_substep(void *block_tag,
+                          int *ex, double *X, double *Y, double *Z,
+                          double **state_host_in,
+                          double **state_host_out,
+                          double **matter_host,
+                          const double *propspeed,
+                          const double *soa_flat,
+                          const double *bbox,
+                          double &dT,
+                          double &T,
+                          int &RK4,
+                          int &apply_bam_bc,
+                          int &Symmetry,
+                          int &Lev,
+                          double &eps,
+                          int &co,
+                          int &use_zero_matter,
+                          int &keep_resident_state,
+                          int &apply_enforce_ga,
+                          double &chitiny);
+
+int bssn_cuda_compute_escalar_matter(void *block_tag,
+                                     int *ex, double *X, double *Y, double *Z,
+                                     double **state_host_in,
+                                     double *Sphi_host,
+                                     double *Spi_host,
+                                     double *Sphi_rhs_host,
+                                     double *Spi_rhs_host,
+                                     double a2,
+                                     int &Symmetry,
+                                     int &Lev,
+                                     double &eps,
+                                     int &co,
+                                     int &apply_enforce_ga);
+
+int bssn_cuda_escalar_finalize_scalar_fields(void *block_tag,
+                                             int *ex, double *X, double *Y, double *Z,
+                                             double *Sphi_out_host,
+                                             double *Spi_out_host,
+                                             const double *propspeed,
+                                             const double *soa_flat,
+                                             const double *bbox,
+                                             double &dT,
+                                             int &RK4,
+                                             int &apply_bam_bc,
+                                             int &Symmetry,
+                                             int &Lev,
+                                             double &eps,
+                                             int &precor);
+
+int bssn_cuda_escalar_has_resident_fields(void *block_tag,
+                                          double *Sphi_host,
+                                          double *Spi_host);
+
+int bssn_cuda_escalar_has_any_resident_fields(void *block_tag);
+
+int bssn_cuda_escalar_download_fields_if_present(void *block_tag,
+                                                 int *ex,
+                                                 double *Sphi_host,
+                                                 double *Spi_host);
+
+int bssn_cuda_pack_escalar_batch_to_host_buffer(void *block_tag,
+                                                double **scalar_host_key,
+                                                double *host_buffer,
+                                                int *ex,
+                                                int i0, int j0, int k0,
+                                                int sx, int sy, int sz);
+
+int bssn_cuda_unpack_escalar_batch_from_host_buffer(void *block_tag,
+                                                    double **scalar_host_key,
+                                                    double *host_buffer,
+                                                    int *ex,
+                                                    int i0, int j0, int k0,
+                                                    int sx, int sy, int sz);
+
+int bssn_cuda_pack_escalar_batch_to_device_buffer(void *block_tag,
+                                                  double **scalar_host_key,
+                                                  double *device_buffer,
+                                                  int *ex,
+                                                  int i0, int j0, int k0,
+                                                  int sx, int sy, int sz);
+
+int bssn_cuda_unpack_escalar_batch_from_device_buffer(void *block_tag,
+                                                      double **scalar_host_key,
+                                                      double *device_buffer,
+                                                      int *ex,
+                                                      int i0, int j0, int k0,
+                                                      int sx, int sy, int sz);
+
+int bssn_cuda_restrict_escalar_batch_to_host_buffer(void *block_tag,
+                                                    double **scalar_host_key,
+                                                    double *host_buffer,
+                                                    int *ex,
+                                                    int sx, int sy, int sz,
+                                                    int fi0, int fj0, int fk0,
+                                                    const double *scalar_soa);
+
+int bssn_cuda_prolong_escalar_batch_to_host_buffer(void *block_tag,
+                                                   double **scalar_host_key,
+                                                   double *host_buffer,
+                                                   int *ex,
+                                                   int sx, int sy, int sz,
+                                                   int ii0, int jj0, int kk0,
+                                                   int lbc_i, int lbc_j, int lbc_k,
+                                                   const double *scalar_soa);
+
+int bssn_cuda_restrict_escalar_batch_to_device_buffer(void *block_tag,
+                                                      double **scalar_host_key,
+                                                      double *device_buffer,
+                                                      int *ex,
+                                                      int sx, int sy, int sz,
+                                                      int fi0, int fj0, int fk0,
+                                                      const double *scalar_soa);
+
+int bssn_cuda_prolong_escalar_batch_to_device_buffer(void *block_tag,
+                                                     double **scalar_host_key,
+                                                     double *device_buffer,
+                                                     int *ex,
+                                                     int sx, int sy, int sz,
+                                                     int ii0, int jj0, int kk0,
+                                                     int lbc_i, int lbc_j, int lbc_k,
+                                                     const double *scalar_soa);
+
+int bssn_cuda_prepare_escalar_inter_time_level(void *block_tag,
+                                               int *ex,
+                                               double **src1_host_key,
+                                               double **src2_host_key,
+                                               double **src3_host_key,
+                                               double **dst_host_key,
+                                               int source_count,
+                                               int tindex);
+
+int bssn_cuda_copy_state_region_to_host(void *block_tag,
+                                        int state_index,
+                                        double *host_state,
+                                        int *ex,
+                                        int i0, int j0, int k0,
+                                        int sx, int sy, int sz);
+
+int bssn_cuda_copy_state_region_from_host(void *block_tag,
+                                          int state_index,
+                                          double *host_state,
+                                          int *ex,
+                                          int i0, int j0, int k0,
+                                          int sx, int sy, int sz);
+
+int bssn_cuda_download_resident_state(void *block_tag,
+                                      int *ex,
+                                      double **state_host_out);
+
+int bssn_cuda_download_resident_state_if_present(void *block_tag,
+                                                int *ex,
+                                                double **state_host_out);
+
+int bssn_cuda_resident_state_matches(void *block_tag,
+                                     double **state_host_key);
+
+int bssn_cuda_download_constraint_outputs(int *ex,
+                                          double **constraint_host_out);
+
+int bssn_cuda_pack_state_region_to_host_buffer(void *block_tag,
+                                               int state_index,
+                                               double *host_buffer,
+                                               int *ex,
+                                               int i0, int j0, int k0,
+                                               int sx, int sy, int sz);
+
+int bssn_cuda_interp_state_point3(void *block_tag,
+                                  int *ex,
+                                  int state0,
+                                  int state1,
+                                  int state2,
+                                  double x0,
+                                  double y0,
+                                  double z0,
+                                  double dx,
+                                  double dy,
+                                  double dz,
+                                  double px,
+                                  double py,
+                                  double pz,
+                                  int ordn,
+                                  int symmetry,
+                                  const double *soa3,
+                                  double *out3);
+
+int bssn_cuda_interp_host_two_fields(void *block_tag,
+                                     int *ex,
+                                     double *field0,
+                                     double *field1,
+                                     double x0,
+                                     double y0,
+                                     double z0,
+                                     double dx,
+                                     double dy,
+                                     double dz,
+                                     const double *px,
+                                     const double *py,
+                                     const double *pz,
+                                     int npoints,
+                                     int ordn,
+                                     int symmetry,
+                                     const double *soa6,
+                                     double *out_interleaved);
+
+int bssn_cuda_unpack_state_region_from_host_buffer(void *block_tag,
+                                                   int state_index,
+                                                   double *host_buffer,
+                                                   int *ex,
+                                                   int i0, int j0, int k0,
+                                                   int sx, int sy, int sz);
+
+int bssn_cuda_pack_state_batch_to_host_buffer(void *block_tag,
+                                              int state_count,
+                                              double *host_buffer,
+                                              int *ex,
+                                              int i0, int j0, int k0,
+                                              int sx, int sy, int sz);
+
+int bssn_cuda_pack_state_batch_to_host_buffer_for_host_views(void *block_tag,
+                                                             double **state_host_key,
+                                                             int state_count,
+                                                             double *host_buffer,
+                                                             int *ex,
+                                                             int i0, int j0, int k0,
+                                                             int sx, int sy, int sz);
+
+int bssn_cuda_unpack_state_batch_from_host_buffer(void *block_tag,
+                                                  int state_count,
+                                                  double *host_buffer,
+                                                  int *ex,
+                                                  int i0, int j0, int k0,
+                                                  int sx, int sy, int sz);
+
+int bssn_cuda_unpack_state_batch_from_host_buffer_for_host_views(void *block_tag,
+                                                                 double **state_host_key,
+                                                                 int state_count,
+                                                                 double *host_buffer,
+                                                                 int *ex,
+                                                                 int i0, int j0, int k0,
+                                                                 int sx, int sy, int sz);
+
+int bssn_cuda_restrict_state_batch_to_host_buffer_for_host_views(void *block_tag,
+                                                                 double **state_host_key,
+                                                                 int state_count,
+                                                                 double *host_buffer,
+                                                                 int *ex,
+                                                                 int sx, int sy, int sz,
+                                                                 int fi0, int fj0, int fk0,
+                                                                 const double *state_soa);
+
+int bssn_cuda_restrict_state_batch_to_host_buffer(void *block_tag,
+                                                  int state_count,
+                                                  double *host_buffer,
+                                                  int *ex,
+                                                  int sx, int sy, int sz,
+                                                  int fi0, int fj0, int fk0,
+                                                  const double *state_soa);
+
+int bssn_cuda_prolong_state_batch_to_host_buffer_for_host_views(void *block_tag,
+                                                                double **state_host_key,
+                                                                int state_count,
+                                                                double *host_buffer,
+                                                                int *ex,
+                                                                int sx, int sy, int sz,
+                                                                int ii0, int jj0, int kk0,
+                                                                int lbc_i, int lbc_j, int lbc_k,
+                                                                const double *state_soa);
+
+int bssn_cuda_prolong_state_batch_to_host_buffer(void *block_tag,
+                                                 int state_count,
+                                                 double *host_buffer,
+                                                 int *ex,
+                                                 int sx, int sy, int sz,
+                                                 int ii0, int jj0, int kk0,
+                                                 int lbc_i, int lbc_j, int lbc_k,
+                                                 const double *state_soa);
+
+int bssn_cuda_pack_state_batch_to_device_buffer(void *block_tag,
+                                                 int state_count,
+                                                double *device_buffer,
+                                                int *ex,
+                                                int i0, int j0, int k0,
+                                                int sx, int sy, int sz);
+
+int bssn_cuda_pack_state_batch_to_device_buffer_for_host_views(void *block_tag,
+                                                               double **state_host_key,
+                                                               int state_count,
+                                                               double *device_buffer,
+                                                               int *ex,
+                                                               int i0, int j0, int k0,
+                                                               int sx, int sy, int sz);
+
+int bssn_cuda_unpack_state_batch_from_device_buffer(void *block_tag,
+                                                    int state_count,
+                                                    double *device_buffer,
+                                                    int *ex,
+                                                    int i0, int j0, int k0,
+                                                    int sx, int sy, int sz);
+
+int bssn_cuda_unpack_state_batch_from_device_buffer_for_host_views(void *block_tag,
+                                                                   double **state_host_key,
+                                                                   int state_count,
+                                                                   double *device_buffer,
+                                                                   int *ex,
+                                                                   int i0, int j0, int k0,
+                                                                   int sx, int sy, int sz);
+
+int bssn_cuda_pack_state_segments_to_device_buffer(void *block_tag,
+                                                   int state_count,
+                                                   double *device_buffer,
+                                                   int *ex,
+                                                   int segment_count,
+                                                   const int *segment_meta);
+
+int bssn_cuda_pack_state_segments_to_device_buffer_for_host_views(void *block_tag,
+                                                                  double **state_host_key,
+                                                                  int state_count,
+                                                                  double *device_buffer,
+                                                                  int *ex,
+                                                                  int segment_count,
+                                                                  const int *segment_meta);
+
+int bssn_cuda_unpack_state_segments_from_device_buffer(void *block_tag,
+                                                       int state_count,
+                                                       double *device_buffer,
+                                                       int *ex,
+                                                       int segment_count,
+                                                       const int *segment_meta);
+
+int bssn_cuda_unpack_state_segments_from_device_buffer_for_host_views(void *block_tag,
+                                                                      double **state_host_key,
+                                                                      int state_count,
+                                                                      double *device_buffer,
+                                                                      int *ex,
+                                                                      int segment_count,
+                                                                      const int *segment_meta);
+
+int bssn_cuda_restrict_state_segments_to_device_buffer(void *block_tag,
+                                                       int state_count,
+                                                       double *device_buffer,
+                                                       int *ex,
+                                                       int segment_count,
+                                                       const int *segment_meta);
+
+int bssn_cuda_restrict_state_segments_to_device_buffer_for_host_views(void *block_tag,
+                                                                      double **state_host_key,
+                                                                      int state_count,
+                                                                      double *device_buffer,
+                                                                      int *ex,
+                                                                      int segment_count,
+                                                                      const int *segment_meta,
+                                                                      const double *state_soa);
+
+int bssn_cuda_prolong_state_segments_to_device_buffer(void *block_tag,
+                                                      int state_count,
+                                                      double *device_buffer,
+                                                      int *ex,
+                                                      int segment_count,
+                                                      const int *segment_meta);
+
+int bssn_cuda_prolong_state_segments_to_device_buffer_for_host_views(void *block_tag,
+                                                                     double **state_host_key,
+                                                                     int state_count,
+                                                                     double *device_buffer,
+                                                                     int *ex,
+                                                                     int segment_count,
+                                                                     const int *segment_meta,
+                                                                     const double *state_soa);
+
+int bssn_cuda_restrict_state_batch_to_device_buffer(void *block_tag,
+                                                    int state_count,
+                                                    double *device_buffer,
+                                                    int *ex,
+                                                    int sx, int sy, int sz,
+                                                    int fi0, int fj0, int fk0);
+
+int bssn_cuda_restrict_state_batch_to_device_buffer_for_host_views(void *block_tag,
+                                                                   double **state_host_key,
+                                                                   int state_count,
+                                                                   double *device_buffer,
+                                                                   int *ex,
+                                                                   int sx, int sy, int sz,
+                                                                   int fi0, int fj0, int fk0,
+                                                                   const double *state_soa);
+
+int bssn_cuda_prolong_state_batch_to_device_buffer(void *block_tag,
+                                                   int state_count,
+                                                   double *device_buffer,
+                                                   int *ex,
+                                                   int sx, int sy, int sz,
+                                                   int ii0, int jj0, int kk0,
+                                                   int lbc_i, int lbc_j, int lbc_k);
+
+int bssn_cuda_prolong_state_batch_to_device_buffer_for_host_views(void *block_tag,
+                                                                  double **state_host_key,
+                                                                  int state_count,
+                                                                  double *device_buffer,
+                                                                  int *ex,
+                                                                  int sx, int sy, int sz,
+                                                                  int ii0, int jj0, int kk0,
+                                                                  int lbc_i, int lbc_j, int lbc_k,
+                                                                  const double *state_soa);
+
+int bssn_cuda_download_state_subset(void *block_tag,
+                                    int *ex,
+                                    int subset_count,
+                                    const int *state_indices,
+                                    double **state_host_out);
+
+int bssn_cuda_upload_state_subset(void *block_tag,
+                                  int *ex,
+                                  int subset_count,
+                                  const int *state_indices,
+                                  double **state_host_in);
+
+int bssn_cuda_prepare_inter_time_level(void *block_tag,
+                                       int *ex,
+                                       double **src1_host_key,
+                                       double **src2_host_key,
+                                       double **src3_host_key,
+                                       double **dst_host_key,
+                                       int source_count,
+                                       int tindex);
+
+int bssn_cuda_has_resident_state(void *block_tag);
+
+void bssn_cuda_release_step_ctx(void *block_tag);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

AMSS_NCKU_source/cgh.C

@@ -130,7 +130,11 @@ void cgh::compose_cgh(int nprocs)
   for (int lev = 0; lev < levels; lev++)
   {
     checkPatchList(PatL[lev], false);
+#ifdef INTERP_LB_OPTIMIZE
+    Parallel::distribute_optimize(PatL[lev], nprocs, ingfs, fngfs, false);
+#else
     Parallel::distribute(PatL[lev], nprocs, ingfs, fngfs, false);
+#endif
 #if (RPB == 1)
     // we need distributed box of PatL[lev] and PatL[lev-1]
     if (lev > 0)
@@ -1301,13 +1305,13 @@ bool cgh::Interp_One_Point(MyList<var> *VarList,
 }
 
 
-void cgh::Regrid_Onelevel(int lev, int Symmetry, int BH_num, double **Porgbr, double **Porg0,
+bool cgh::Regrid_Onelevel(int lev, int Symmetry, int BH_num, double **Porgbr, double **Porg0,
                           MyList<var> *OldList, MyList<var> *StateList,
                           MyList<var> *FutureList, MyList<var> *tmList, bool BB,
                           monitor *ErrorMonitor)
 {
   if (lev < movls)
-    return;
+    return false;
 
 #if (0)
   // #if (PSTR == 1 || PSTR == 2)
@@ -1396,7 +1400,7 @@ void cgh::Regrid_Onelevel(int lev, int Symmetry, int BH_num, double **Porgbr, do
       for (bhi = 0; bhi < BH_num; bhi++)
         delete[] tmpPorg[bhi];
       delete[] tmpPorg;
-      return;
+      return false;
     }
     // x direction
     rr = (Porg0[bhi][0] - handle[lev][grd][0]) / dX;
@@ -1500,6 +1504,7 @@ void cgh::Regrid_Onelevel(int lev, int Symmetry, int BH_num, double **Porgbr, do
   for (int bhi = 0; bhi < BH_num; bhi++)
     delete[] tmpPorg[bhi];
   delete[] tmpPorg;
+  return tot_flag;
 }
 
 

AMSS_NCKU_source/cgh.h

@@ -74,7 +74,7 @@ public:
                                MyList<var> *OldList, MyList<var> *StateList,
                                MyList<var> *FutureList, MyList<var> *tmList,
                                int Symmetry, bool BB);
-   void Regrid_Onelevel(int lev, int Symmetry, int BH_num, double **Porgbr, double **Porg0,
+   bool Regrid_Onelevel(int lev, int Symmetry, int BH_num, double **Porgbr, double **Porg0,
                         MyList<var> *OldList, MyList<var> *StateList,
                         MyList<var> *FutureList, MyList<var> *tmList, bool BB,
                         monitor *ErrorMonitor);

AMSS_NCKU_source/diff_new.f90

@@ -69,10 +69,12 @@
   fy = ZEO
   fz = ZEO
 
+!DIR$ SIMD VECTORLENGTHFOR(KNOWN_INTEGER=8)
+!DIR$ UNROLL PARTIAL(4)
   do k=1,ex(3)-1
   do j=1,ex(2)-1
   do i=1,ex(1)-1
-! x direction   
+! x direction
         if(i+1 <= imax .and. i-1 >= imin)then
 !
 !              - f(i-1) + f(i+1)
@@ -371,6 +373,8 @@
   fxz = ZEO
   fyz = ZEO
 
+!DIR$ SIMD VECTORLENGTHFOR(KNOWN_INTEGER=8)
+!DIR$ UNROLL PARTIAL(4)
   do k=1,ex(3)-1
   do j=1,ex(2)-1
   do i=1,ex(1)-1

AMSS_NCKU_source/diff_newwb.f90

@@ -33,7 +33,7 @@
   real*8 :: dX,dY,dZ
   real*8,dimension(0:ex(1),0:ex(2),0:ex(3))   :: fh
   real*8, dimension(3) :: SoA
-  integer :: imin,jmin,kmin,imax,jmax,kmax,i,j,k
+  integer :: imin,jmin,kmin,imax,jmax,kmax,i,j,k
   real*8 :: d2dx,d2dy,d2dz
   integer, parameter :: NO_SYMM = 0, EQ_SYMM = 1, OCTANT = 2
   real*8,  parameter :: ZEO=0.d0,ONE=1.d0, F60=6.d1
@@ -137,7 +137,7 @@
   real*8 :: dX
   real*8,dimension(0:ex(1),0:ex(2),0:ex(3))   :: fh
   real*8, dimension(3) :: SoA
-  integer :: imin,jmin,kmin,imax,jmax,kmax,i,j,k
+  integer :: imin,jmin,kmin,imax,jmax,kmax,i,j,k
   real*8 :: d2dx
   integer, parameter :: NO_SYMM = 0, EQ_SYMM = 1, OCTANT = 2
   real*8,  parameter :: ZEO=0.d0,ONE=1.d0, F60=6.d1
@@ -1512,8 +1512,9 @@
   real*8 :: dX,dY,dZ
   real*8,dimension(-1:ex(1),-1:ex(2),-1:ex(3))   :: fh
   real*8, dimension(3) :: SoA
-  integer :: imin,jmin,kmin,imax,jmax,kmax,i,j,k
-  real*8  :: Sdxdx,Sdydy,Sdzdz,Fdxdx,Fdydy,Fdzdz
+  integer :: imin,jmin,kmin,imax,jmax,kmax,i,j,k
+  integer :: i_core_min,i_core_max,j_core_min,j_core_max,k_core_min,k_core_max
+  real*8  :: Sdxdx,Sdydy,Sdzdz,Fdxdx,Fdydy,Fdzdz
   real*8  :: Sdxdy,Sdxdz,Sdydz,Fdxdy,Fdxdz,Fdydz
   integer, parameter :: NO_SYMM = 0, EQ_SYMM = 1, OCTANT = 2
   real*8, parameter :: ZEO=0.d0, ONE=1.d0, TWO=2.d0, F1o4=2.5d-1, F9=9.d0,  F45=4.5d1
@@ -1560,17 +1561,55 @@
 
   fxx = ZEO
   fyy = ZEO
-  fzz = ZEO
-  fxy = ZEO
-  fxz = ZEO
-  fyz = ZEO
-
-  do k=1,ex(3)
-  do j=1,ex(2)
-  do i=1,ex(1)
-!~~~~~~ fxx
-        if(i+2 <= imax .and. i-2 >= imin)then
-!
+  fzz = ZEO
+  fxy = ZEO
+  fxz = ZEO
+  fyz = ZEO
+
+  i_core_min = max(1, imin+2)
+  i_core_max = min(ex(1), imax-2)
+  j_core_min = max(1, jmin+2)
+  j_core_max = min(ex(2), jmax-2)
+  k_core_min = max(1, kmin+2)
+  k_core_max = min(ex(3), kmax-2)
+
+  if(i_core_min <= i_core_max .and. j_core_min <= j_core_max .and. k_core_min <= k_core_max)then
+   do k=k_core_min,k_core_max
+   do j=j_core_min,j_core_max
+   do i=i_core_min,i_core_max
+! interior points always use 4th-order stencils without branch checks
+      fxx(i,j,k) = Fdxdx*(-fh(i-2,j,k)+F16*fh(i-1,j,k)-F30*fh(i,j,k) &
+                          -fh(i+2,j,k)+F16*fh(i+1,j,k)              )
+      fyy(i,j,k) = Fdydy*(-fh(i,j-2,k)+F16*fh(i,j-1,k)-F30*fh(i,j,k) &
+                          -fh(i,j+2,k)+F16*fh(i,j+1,k)              )
+      fzz(i,j,k) = Fdzdz*(-fh(i,j,k-2)+F16*fh(i,j,k-1)-F30*fh(i,j,k) &
+                          -fh(i,j,k+2)+F16*fh(i,j,k+1)              )
+      fxy(i,j,k) = Fdxdy*(     (fh(i-2,j-2,k)-F8*fh(i-1,j-2,k)+F8*fh(i+1,j-2,k)-fh(i+2,j-2,k))  &
+                          -F8 *(fh(i-2,j-1,k)-F8*fh(i-1,j-1,k)+F8*fh(i+1,j-1,k)-fh(i+2,j-1,k))  &
+                          +F8 *(fh(i-2,j+1,k)-F8*fh(i-1,j+1,k)+F8*fh(i+1,j+1,k)-fh(i+2,j+1,k))  &
+                          -    (fh(i-2,j+2,k)-F8*fh(i-1,j+2,k)+F8*fh(i+1,j+2,k)-fh(i+2,j+2,k)))
+      fxz(i,j,k) = Fdxdz*(     (fh(i-2,j,k-2)-F8*fh(i-1,j,k-2)+F8*fh(i+1,j,k-2)-fh(i+2,j,k-2))  &
+                          -F8 *(fh(i-2,j,k-1)-F8*fh(i-1,j,k-1)+F8*fh(i+1,j,k-1)-fh(i+2,j,k-1))  &
+                          +F8 *(fh(i-2,j,k+1)-F8*fh(i-1,j,k+1)+F8*fh(i+1,j,k+1)-fh(i+2,j,k+1))  &
+                          -    (fh(i-2,j,k+2)-F8*fh(i-1,j,k+2)+F8*fh(i+1,j,k+2)-fh(i+2,j,k+2)))
+      fyz(i,j,k) = Fdydz*(     (fh(i,j-2,k-2)-F8*fh(i,j-1,k-2)+F8*fh(i,j+1,k-2)-fh(i,j+2,k-2))  &
+                          -F8 *(fh(i,j-2,k-1)-F8*fh(i,j-1,k-1)+F8*fh(i,j+1,k-1)-fh(i,j+2,k-1))  &
+                          +F8 *(fh(i,j-2,k+1)-F8*fh(i,j-1,k+1)+F8*fh(i,j+1,k+1)-fh(i,j+2,k+1))  &
+                          -    (fh(i,j-2,k+2)-F8*fh(i,j-1,k+2)+F8*fh(i,j+1,k+2)-fh(i,j+2,k+2)))
+   enddo
+   enddo
+   enddo
+  endif
+
+  do k=1,ex(3)
+  do j=1,ex(2)
+  do i=1,ex(1)
+      if(i>=i_core_min .and. i<=i_core_max .and. &
+         j>=j_core_min .and. j<=j_core_max .and. &
+         k>=k_core_min .and. k<=k_core_max) cycle
+!~~~~~~ fxx
+        if(i+2 <= imax .and. i-2 >= imin)then
+!
 !               - f(i-2) + 16 f(i-1) - 30 f(i) + 16 f(i+1) - f(i+2)
 !  fxx(i) = ----------------------------------------------------------
 !                                  12 dx^2 

AMSS_NCKU_source/enforce_algebra.f90

@@ -18,49 +18,61 @@
   real*8, dimension(ex(1),ex(2),ex(3)), intent(inout) :: Ayy,Ayz,Azz
 
 !~~~~~~~> Local variable:
-  
-  real*8, dimension(ex(1),ex(2),ex(3)) :: trA,detg
-  real*8, dimension(ex(1),ex(2),ex(3)) :: gxx,gyy,gzz 
-  real*8, dimension(ex(1),ex(2),ex(3)) :: gupxx,gupxy,gupxz,gupyy,gupyz,gupzz
+
+  integer :: i,j,k
+  real*8 :: lgxx,lgyy,lgzz,ldetg
+  real*8 :: lgupxx,lgupxy,lgupxz,lgupyy,lgupyz,lgupzz
+  real*8 :: ltrA,lscale
   real*8, parameter :: F1o3 = 1.D0 / 3.D0, ONE = 1.D0, TWO = 2.D0
 
 !~~~~~~>
 
-  gxx = dxx + ONE
-  gyy = dyy + ONE
-  gzz = dzz + ONE
+  do k=1,ex(3)
+  do j=1,ex(2)
+  do i=1,ex(1)
 
-  detg =  gxx * gyy * gzz + gxy * gyz * gxz + gxz * gxy * gyz - &
-          gxz * gyy * gxz - gxy * gxy * gzz - gxx * gyz * gyz
-  gupxx =   ( gyy * gzz - gyz * gyz ) / detg
-  gupxy = - ( gxy * gzz - gyz * gxz ) / detg
-  gupxz =   ( gxy * gyz - gyy * gxz ) / detg
-  gupyy =   ( gxx * gzz - gxz * gxz ) / detg
-  gupyz = - ( gxx * gyz - gxy * gxz ) / detg
-  gupzz =   ( gxx * gyy - gxy * gxy ) / detg
+    lgxx = dxx(i,j,k) + ONE
+    lgyy = dyy(i,j,k) + ONE
+    lgzz = dzz(i,j,k) + ONE
 
-  trA =         gupxx * Axx + gupyy * Ayy + gupzz * Azz &
-       + TWO * (gupxy * Axy + gupxz * Axz + gupyz * Ayz)
+    ldetg =  lgxx * lgyy * lgzz &
+           + gxy(i,j,k) * gyz(i,j,k) * gxz(i,j,k) &
+           + gxz(i,j,k) * gxy(i,j,k) * gyz(i,j,k) &
+           - gxz(i,j,k) * lgyy * gxz(i,j,k) &
+           - gxy(i,j,k) * gxy(i,j,k) * lgzz &
+           - lgxx * gyz(i,j,k) * gyz(i,j,k)
 
-  Axx = Axx - F1o3 * gxx * trA
-  Axy = Axy - F1o3 * gxy * trA
-  Axz = Axz - F1o3 * gxz * trA
-  Ayy = Ayy - F1o3 * gyy * trA
-  Ayz = Ayz - F1o3 * gyz * trA
-  Azz = Azz - F1o3 * gzz * trA
+    lgupxx =   ( lgyy * lgzz - gyz(i,j,k) * gyz(i,j,k) ) / ldetg
+    lgupxy = - ( gxy(i,j,k) * lgzz - gyz(i,j,k) * gxz(i,j,k) ) / ldetg
+    lgupxz =   ( gxy(i,j,k) * gyz(i,j,k) - lgyy * gxz(i,j,k) ) / ldetg
+    lgupyy =   ( lgxx * lgzz - gxz(i,j,k) * gxz(i,j,k) ) / ldetg
+    lgupyz = - ( lgxx * gyz(i,j,k) - gxy(i,j,k) * gxz(i,j,k) ) / ldetg
+    lgupzz =   ( lgxx * lgyy - gxy(i,j,k) * gxy(i,j,k) ) / ldetg
 
-  detg = ONE / ( detg ** F1o3 ) 
-  
-  gxx = gxx * detg
-  gxy = gxy * detg
-  gxz = gxz * detg
-  gyy = gyy * detg
-  gyz = gyz * detg
-  gzz = gzz * detg
+    ltrA =         lgupxx * Axx(i,j,k) + lgupyy * Ayy(i,j,k) &
+                 + lgupzz * Azz(i,j,k) &
+         + TWO * (lgupxy * Axy(i,j,k) + lgupxz * Axz(i,j,k) &
+                 + lgupyz * Ayz(i,j,k))
 
-  dxx = gxx - ONE
-  dyy = gyy - ONE
-  dzz = gzz - ONE
+    Axx(i,j,k) = Axx(i,j,k) - F1o3 * lgxx * ltrA
+    Axy(i,j,k) = Axy(i,j,k) - F1o3 * gxy(i,j,k) * ltrA
+    Axz(i,j,k) = Axz(i,j,k) - F1o3 * gxz(i,j,k) * ltrA
+    Ayy(i,j,k) = Ayy(i,j,k) - F1o3 * lgyy * ltrA
+    Ayz(i,j,k) = Ayz(i,j,k) - F1o3 * gyz(i,j,k) * ltrA
+    Azz(i,j,k) = Azz(i,j,k) - F1o3 * lgzz * ltrA
+
+    lscale = ONE / ( ldetg ** F1o3 )
+
+    dxx(i,j,k) = lgxx * lscale - ONE
+    gxy(i,j,k) = gxy(i,j,k) * lscale
+    gxz(i,j,k) = gxz(i,j,k) * lscale
+    dyy(i,j,k) = lgyy * lscale - ONE
+    gyz(i,j,k) = gyz(i,j,k) * lscale
+    dzz(i,j,k) = lgzz * lscale - ONE
+
+  enddo
+  enddo
+  enddo
 
   return
 
@@ -82,51 +94,71 @@
   real*8, dimension(ex(1),ex(2),ex(3)), intent(inout) :: Ayy,Ayz,Azz
 
 !~~~~~~~> Local variable:
-  
-  real*8, dimension(ex(1),ex(2),ex(3)) :: trA
-  real*8, dimension(ex(1),ex(2),ex(3)) :: gxx,gyy,gzz 
-  real*8, dimension(ex(1),ex(2),ex(3)) :: gupxx,gupxy,gupxz,gupyy,gupyz,gupzz
+
+  integer :: i,j,k
+  real*8 :: lgxx,lgyy,lgzz,lscale
+  real*8 :: lgxy,lgxz,lgyz
+  real*8 :: lgupxx,lgupxy,lgupxz,lgupyy,lgupyz,lgupzz
+  real*8 :: ltrA
   real*8, parameter :: F1o3 = 1.D0 / 3.D0, ONE = 1.D0, TWO = 2.D0
 
 !~~~~~~>
 
-  gxx = dxx + ONE
-  gyy = dyy + ONE
-  gzz = dzz + ONE
-! for g
-  gupzz =  gxx * gyy * gzz + gxy * gyz * gxz + gxz * gxy * gyz - &
-           gxz * gyy * gxz - gxy * gxy * gzz - gxx * gyz * gyz
+  do k=1,ex(3)
+  do j=1,ex(2)
+  do i=1,ex(1)
 
-  gupzz = ONE / ( gupzz ** F1o3 ) 
-  
-  gxx = gxx * gupzz
-  gxy = gxy * gupzz
-  gxz = gxz * gupzz
-  gyy = gyy * gupzz
-  gyz = gyz * gupzz
-  gzz = gzz * gupzz
+! for g: normalize determinant first
+    lgxx = dxx(i,j,k) + ONE
+    lgyy = dyy(i,j,k) + ONE
+    lgzz = dzz(i,j,k) + ONE
+    lgxy = gxy(i,j,k)
+    lgxz = gxz(i,j,k)
+    lgyz = gyz(i,j,k)
 
-  dxx = gxx - ONE
-  dyy = gyy - ONE
-  dzz = gzz - ONE
-! for A  
+    lscale =  lgxx * lgyy * lgzz + lgxy * lgyz * lgxz &
+            + lgxz * lgxy * lgyz - lgxz * lgyy * lgxz &
+            - lgxy * lgxy * lgzz - lgxx * lgyz * lgyz
 
-  gupxx =   ( gyy * gzz - gyz * gyz )
-  gupxy = - ( gxy * gzz - gyz * gxz )
-  gupxz =   ( gxy * gyz - gyy * gxz )
-  gupyy =   ( gxx * gzz - gxz * gxz )
-  gupyz = - ( gxx * gyz - gxy * gxz )
-  gupzz =   ( gxx * gyy - gxy * gxy )
+    lscale = ONE / ( lscale ** F1o3 )
 
-  trA =         gupxx * Axx + gupyy * Ayy + gupzz * Azz &
-       + TWO * (gupxy * Axy + gupxz * Axz + gupyz * Ayz)
+    lgxx = lgxx * lscale
+    lgxy = lgxy * lscale
+    lgxz = lgxz * lscale
+    lgyy = lgyy * lscale
+    lgyz = lgyz * lscale
+    lgzz = lgzz * lscale
 
-  Axx = Axx - F1o3 * gxx * trA
-  Axy = Axy - F1o3 * gxy * trA
-  Axz = Axz - F1o3 * gxz * trA
-  Ayy = Ayy - F1o3 * gyy * trA
-  Ayz = Ayz - F1o3 * gyz * trA
-  Azz = Azz - F1o3 * gzz * trA
+    dxx(i,j,k) = lgxx - ONE
+    gxy(i,j,k) = lgxy
+    gxz(i,j,k) = lgxz
+    dyy(i,j,k) = lgyy - ONE
+    gyz(i,j,k) = lgyz
+    dzz(i,j,k) = lgzz - ONE
+
+! for A: trace-free using normalized metric (det=1, no division needed)
+    lgupxx =   ( lgyy * lgzz - lgyz * lgyz )
+    lgupxy = - ( lgxy * lgzz - lgyz * lgxz )
+    lgupxz =   ( lgxy * lgyz - lgyy * lgxz )
+    lgupyy =   ( lgxx * lgzz - lgxz * lgxz )
+    lgupyz = - ( lgxx * lgyz - lgxy * lgxz )
+    lgupzz =   ( lgxx * lgyy - lgxy * lgxy )
+
+    ltrA =         lgupxx * Axx(i,j,k) + lgupyy * Ayy(i,j,k) &
+                 + lgupzz * Azz(i,j,k) &
+         + TWO * (lgupxy * Axy(i,j,k) + lgupxz * Axz(i,j,k) &
+                 + lgupyz * Ayz(i,j,k))
+
+    Axx(i,j,k) = Axx(i,j,k) - F1o3 * lgxx * ltrA
+    Axy(i,j,k) = Axy(i,j,k) - F1o3 * lgxy * ltrA
+    Axz(i,j,k) = Axz(i,j,k) - F1o3 * lgxz * ltrA
+    Ayy(i,j,k) = Ayy(i,j,k) - F1o3 * lgyy * ltrA
+    Ayz(i,j,k) = Ayz(i,j,k) - F1o3 * lgyz * ltrA
+    Azz(i,j,k) = Azz(i,j,k) - F1o3 * lgzz * ltrA
+
+  enddo
+  enddo
+  enddo
 
   return
 

AMSS_NCKU_source/fdderivs_c.C

@@ -0,0 +1,332 @@
+#include "tool.h"
+void fdderivs(const int ex[3],
+              const double *f,
+              double *fxx, double *fxy, double *fxz,
+              double *fyy, double *fyz, double *fzz,
+              const double *X, const double *Y, const double *Z,
+              double SYM1, double SYM2, double SYM3,
+              int Symmetry, int onoff)
+{
+    (void)onoff;
+
+    const int NO_SYMM = 0, EQ_SYMM = 1;
+    const double ZEO = 0.0, ONE = 1.0, TWO = 2.0;
+    const double F1o4   = 2.5e-1;          // 1/4
+    const double F8     = 8.0;
+    const double F16    = 16.0;
+    const double F30    = 30.0;
+    const double F1o12  = ONE / 12.0;
+    const double F1o144 = ONE / 144.0;
+
+    const int ex1 = ex[0], ex2 = ex[1], ex3 = ex[2];
+
+    const double dX = X[1] - X[0];
+    const double dY = Y[1] - Y[0];
+    const double dZ = Z[1] - Z[0];
+
+    const int imaxF = ex1;
+    const int jmaxF = ex2;
+    const int kmaxF = ex3;
+
+    int iminF = 1, jminF = 1, kminF = 1;
+    if (Symmetry > NO_SYMM && fabs(Z[0]) < dZ) kminF = -1;
+    if (Symmetry > EQ_SYMM && fabs(X[0]) < dX) iminF = -1;
+    if (Symmetry > EQ_SYMM && fabs(Y[0]) < dY) jminF = -1;
+
+    const double SoA[3] = { SYM1, SYM2, SYM3 };
+
+    /* fh: (ex1+2)*(ex2+2)*(ex3+2) because ord=2 */
+    const size_t nx = (size_t)ex1 + 2;
+    const size_t ny = (size_t)ex2 + 2;
+    const size_t nz = (size_t)ex3 + 2;
+    const size_t fh_size = nx * ny * nz;
+
+    static double *fh = NULL;
+    static size_t cap = 0;
+
+    if (fh_size > cap) {
+        free(fh);
+        fh = (double*)aligned_alloc(64, fh_size * sizeof(double));
+        cap = fh_size;
+    }
+    // double *fh = (double*)malloc(fh_size * sizeof(double));
+    if (!fh) return;
+
+    symmetry_bd(2, ex, f, fh, SoA);
+
+    /* 系数：按 Fortran 原式 */
+    const double Sdxdx = ONE / (dX * dX);
+    const double Sdydy = ONE / (dY * dY);
+    const double Sdzdz = ONE / (dZ * dZ);
+
+    const double Fdxdx = F1o12 / (dX * dX);
+    const double Fdydy = F1o12 / (dY * dY);
+    const double Fdzdz = F1o12 / (dZ * dZ);
+
+    const double Sdxdy = F1o4 / (dX * dY);
+    const double Sdxdz = F1o4 / (dX * dZ);
+    const double Sdydz = F1o4 / (dY * dZ);
+
+    const double Fdxdy = F1o144 / (dX * dY);
+    const double Fdxdz = F1o144 / (dX * dZ);
+    const double Fdydz = F1o144 / (dY * dZ);
+
+    /* 只清零不被主循环覆盖的边界面 */
+    {
+        /* 高边界：k0=ex3-1 */
+        for (int j0 = 0; j0 < ex2; ++j0)
+            for (int i0 = 0; i0 < ex1; ++i0) {
+                const size_t p = idx_ex(i0, j0, ex3 - 1, ex);
+                fxx[p]=ZEO; fyy[p]=ZEO; fzz[p]=ZEO;
+                fxy[p]=ZEO; fxz[p]=ZEO; fyz[p]=ZEO;
+            }
+        /* 高边界：j0=ex2-1 */
+        for (int k0 = 0; k0 < ex3 - 1; ++k0)
+            for (int i0 = 0; i0 < ex1; ++i0) {
+                const size_t p = idx_ex(i0, ex2 - 1, k0, ex);
+                fxx[p]=ZEO; fyy[p]=ZEO; fzz[p]=ZEO;
+                fxy[p]=ZEO; fxz[p]=ZEO; fyz[p]=ZEO;
+            }
+        /* 高边界：i0=ex1-1 */
+        for (int k0 = 0; k0 < ex3 - 1; ++k0)
+            for (int j0 = 0; j0 < ex2 - 1; ++j0) {
+                const size_t p = idx_ex(ex1 - 1, j0, k0, ex);
+                fxx[p]=ZEO; fyy[p]=ZEO; fzz[p]=ZEO;
+                fxy[p]=ZEO; fxz[p]=ZEO; fyz[p]=ZEO;
+            }
+
+        /* 低边界：当二阶模板也不可用时，对应 i0/j0/k0=0 面 */
+        if (kminF == 1) {
+            for (int j0 = 0; j0 < ex2; ++j0)
+                for (int i0 = 0; i0 < ex1; ++i0) {
+                    const size_t p = idx_ex(i0, j0, 0, ex);
+                    fxx[p]=ZEO; fyy[p]=ZEO; fzz[p]=ZEO;
+                    fxy[p]=ZEO; fxz[p]=ZEO; fyz[p]=ZEO;
+                }
+        }
+        if (jminF == 1) {
+            for (int k0 = 0; k0 < ex3; ++k0)
+                for (int i0 = 0; i0 < ex1; ++i0) {
+                    const size_t p = idx_ex(i0, 0, k0, ex);
+                    fxx[p]=ZEO; fyy[p]=ZEO; fzz[p]=ZEO;
+                    fxy[p]=ZEO; fxz[p]=ZEO; fyz[p]=ZEO;
+                }
+        }
+        if (iminF == 1) {
+            for (int k0 = 0; k0 < ex3; ++k0)
+                for (int j0 = 0; j0 < ex2; ++j0) {
+                    const size_t p = idx_ex(0, j0, k0, ex);
+                    fxx[p]=ZEO; fyy[p]=ZEO; fzz[p]=ZEO;
+                    fxy[p]=ZEO; fxz[p]=ZEO; fyz[p]=ZEO;
+                }
+        }
+    }
+
+    /*
+     * 两段式：
+     * 1) 二阶可用区域先计算二阶模板
+     * 2) 高阶可用区域再覆盖四阶模板
+     */
+    const int i2_lo = (iminF > 0) ? iminF : 0;
+    const int j2_lo = (jminF > 0) ? jminF : 0;
+    const int k2_lo = (kminF > 0) ? kminF : 0;
+    const int i2_hi = ex1 - 2;
+    const int j2_hi = ex2 - 2;
+    const int k2_hi = ex3 - 2;
+
+    const int i4_lo = (iminF + 1 > 0) ? (iminF + 1) : 0;
+    const int j4_lo = (jminF + 1 > 0) ? (jminF + 1) : 0;
+    const int k4_lo = (kminF + 1 > 0) ? (kminF + 1) : 0;
+    const int i4_hi = ex1 - 3;
+    const int j4_hi = ex2 - 3;
+    const int k4_hi = ex3 - 3;
+
+    /*
+     * Strategy A:
+     * Avoid redundant work in overlap of 2nd/4th-order regions.
+     * Only compute 2nd-order on shell points that are NOT overwritten by
+     * the 4th-order pass.
+     */
+    const int has4 = (i4_lo <= i4_hi && j4_lo <= j4_hi && k4_lo <= k4_hi);
+
+    if (i2_lo <= i2_hi && j2_lo <= j2_hi && k2_lo <= k2_hi) {
+        for (int k0 = k2_lo; k0 <= k2_hi; ++k0) {
+            const int kF = k0 + 1;
+            for (int j0 = j2_lo; j0 <= j2_hi; ++j0) {
+                const int jF = j0 + 1;
+                for (int i0 = i2_lo; i0 <= i2_hi; ++i0) {
+                    if (has4 &&
+                        i0 >= i4_lo && i0 <= i4_hi &&
+                        j0 >= j4_lo && j0 <= j4_hi &&
+                        k0 >= k4_lo && k0 <= k4_hi) {
+                        continue;
+                    }
+                    const int iF = i0 + 1;
+                    const size_t p = idx_ex(i0, j0, k0, ex);
+
+                    fxx[p] = Sdxdx * (
+                        fh[idx_fh_F_ord2(iF - 1, jF,     kF,     ex)] -
+                        TWO * fh[idx_fh_F_ord2(iF,     jF,     kF,     ex)] +
+                        fh[idx_fh_F_ord2(iF + 1, jF,     kF,     ex)]
+                    );
+
+                    fyy[p] = Sdydy * (
+                        fh[idx_fh_F_ord2(iF,     jF - 1, kF,     ex)] -
+                        TWO * fh[idx_fh_F_ord2(iF,     jF,     kF,     ex)] +
+                        fh[idx_fh_F_ord2(iF,     jF + 1, kF,     ex)]
+                    );
+
+                    fzz[p] = Sdzdz * (
+                        fh[idx_fh_F_ord2(iF,     jF,     kF - 1, ex)] -
+                        TWO * fh[idx_fh_F_ord2(iF,     jF,     kF,     ex)] +
+                        fh[idx_fh_F_ord2(iF,     jF,     kF + 1, ex)]
+                    );
+
+                    fxy[p] = Sdxdy * (
+                        fh[idx_fh_F_ord2(iF - 1, jF - 1, kF, ex)] -
+                        fh[idx_fh_F_ord2(iF + 1, jF - 1, kF, ex)] -
+                        fh[idx_fh_F_ord2(iF - 1, jF + 1, kF, ex)] +
+                        fh[idx_fh_F_ord2(iF + 1, jF + 1, kF, ex)]
+                    );
+
+                    fxz[p] = Sdxdz * (
+                        fh[idx_fh_F_ord2(iF - 1, jF, kF - 1, ex)] -
+                        fh[idx_fh_F_ord2(iF + 1, jF, kF - 1, ex)] -
+                        fh[idx_fh_F_ord2(iF - 1, jF, kF + 1, ex)] +
+                        fh[idx_fh_F_ord2(iF + 1, jF, kF + 1, ex)]
+                    );
+
+                    fyz[p] = Sdydz * (
+                        fh[idx_fh_F_ord2(iF, jF - 1, kF - 1, ex)] -
+                        fh[idx_fh_F_ord2(iF, jF + 1, kF - 1, ex)] -
+                        fh[idx_fh_F_ord2(iF, jF - 1, kF + 1, ex)] +
+                        fh[idx_fh_F_ord2(iF, jF + 1, kF + 1, ex)]
+                    );
+                }
+            }
+        }
+    }
+
+    if (has4) {
+        for (int k0 = k4_lo; k0 <= k4_hi; ++k0) {
+            const int kF = k0 + 1;
+            for (int j0 = j4_lo; j0 <= j4_hi; ++j0) {
+                const int jF = j0 + 1;
+                for (int i0 = i4_lo; i0 <= i4_hi; ++i0) {
+                    const int iF = i0 + 1;
+                    const size_t p = idx_ex(i0, j0, k0, ex);
+
+                    fxx[p] = Fdxdx * (
+                        -fh[idx_fh_F_ord2(iF - 2, jF,     kF,     ex)] +
+                        F16 * fh[idx_fh_F_ord2(iF - 1, jF,     kF,     ex)] -
+                        F30 * fh[idx_fh_F_ord2(iF,     jF,     kF,     ex)] -
+                        fh[idx_fh_F_ord2(iF + 2, jF,     kF,     ex)] +
+                        F16 * fh[idx_fh_F_ord2(iF + 1, jF,     kF,     ex)]
+                    );
+
+                    fyy[p] = Fdydy * (
+                        -fh[idx_fh_F_ord2(iF,     jF - 2, kF,     ex)] +
+                        F16 * fh[idx_fh_F_ord2(iF,     jF - 1, kF,     ex)] -
+                        F30 * fh[idx_fh_F_ord2(iF,     jF,     kF,     ex)] -
+                        fh[idx_fh_F_ord2(iF,     jF + 2, kF,     ex)] +
+                        F16 * fh[idx_fh_F_ord2(iF,     jF + 1, kF,     ex)]
+                    );
+
+                    fzz[p] = Fdzdz * (
+                        -fh[idx_fh_F_ord2(iF,     jF,     kF - 2, ex)] +
+                        F16 * fh[idx_fh_F_ord2(iF,     jF,     kF - 1, ex)] -
+                        F30 * fh[idx_fh_F_ord2(iF,     jF,     kF,     ex)] -
+                        fh[idx_fh_F_ord2(iF,     jF,     kF + 2, ex)] +
+                        F16 * fh[idx_fh_F_ord2(iF,     jF,     kF + 1, ex)]
+                    );
+
+                    {
+                        const double t_jm2 =
+                            ( fh[idx_fh_F_ord2(iF - 2, jF - 2, kF, ex)]
+                             -F8*fh[idx_fh_F_ord2(iF - 1, jF - 2, kF, ex)]
+                             +F8*fh[idx_fh_F_ord2(iF + 1, jF - 2, kF, ex)]
+                             -    fh[idx_fh_F_ord2(iF + 2, jF - 2, kF, ex)] );
+
+                        const double t_jm1 =
+                            ( fh[idx_fh_F_ord2(iF - 2, jF - 1, kF, ex)]
+                             -F8*fh[idx_fh_F_ord2(iF - 1, jF - 1, kF, ex)]
+                             +F8*fh[idx_fh_F_ord2(iF + 1, jF - 1, kF, ex)]
+                             -    fh[idx_fh_F_ord2(iF + 2, jF - 1, kF, ex)] );
+
+                        const double t_jp1 =
+                            ( fh[idx_fh_F_ord2(iF - 2, jF + 1, kF, ex)]
+                             -F8*fh[idx_fh_F_ord2(iF - 1, jF + 1, kF, ex)]
+                             +F8*fh[idx_fh_F_ord2(iF + 1, jF + 1, kF, ex)]
+                             -    fh[idx_fh_F_ord2(iF + 2, jF + 1, kF, ex)] );
+
+                        const double t_jp2 =
+                            ( fh[idx_fh_F_ord2(iF - 2, jF + 2, kF, ex)]
+                             -F8*fh[idx_fh_F_ord2(iF - 1, jF + 2, kF, ex)]
+                             +F8*fh[idx_fh_F_ord2(iF + 1, jF + 2, kF, ex)]
+                             -    fh[idx_fh_F_ord2(iF + 2, jF + 2, kF, ex)] );
+
+                        fxy[p] = Fdxdy * ( t_jm2 - F8 * t_jm1 + F8 * t_jp1 - t_jp2 );
+                    }
+
+                    {
+                        const double t_km2 =
+                            ( fh[idx_fh_F_ord2(iF - 2, jF, kF - 2, ex)]
+                             -F8*fh[idx_fh_F_ord2(iF - 1, jF, kF - 2, ex)]
+                             +F8*fh[idx_fh_F_ord2(iF + 1, jF, kF - 2, ex)]
+                             -    fh[idx_fh_F_ord2(iF + 2, jF, kF - 2, ex)] );
+
+                        const double t_km1 =
+                            ( fh[idx_fh_F_ord2(iF - 2, jF, kF - 1, ex)]
+                             -F8*fh[idx_fh_F_ord2(iF - 1, jF, kF - 1, ex)]
+                             +F8*fh[idx_fh_F_ord2(iF + 1, jF, kF - 1, ex)]
+                             -    fh[idx_fh_F_ord2(iF + 2, jF, kF - 1, ex)] );
+
+                        const double t_kp1 =
+                            ( fh[idx_fh_F_ord2(iF - 2, jF, kF + 1, ex)]
+                             -F8*fh[idx_fh_F_ord2(iF - 1, jF, kF + 1, ex)]
+                             +F8*fh[idx_fh_F_ord2(iF + 1, jF, kF + 1, ex)]
+                             -    fh[idx_fh_F_ord2(iF + 2, jF, kF + 1, ex)] );
+
+                        const double t_kp2 =
+                            ( fh[idx_fh_F_ord2(iF - 2, jF, kF + 2, ex)]
+                             -F8*fh[idx_fh_F_ord2(iF - 1, jF, kF + 2, ex)]
+                             +F8*fh[idx_fh_F_ord2(iF + 1, jF, kF + 2, ex)]
+                             -    fh[idx_fh_F_ord2(iF + 2, jF, kF + 2, ex)] );
+
+                        fxz[p] = Fdxdz * ( t_km2 - F8 * t_km1 + F8 * t_kp1 - t_kp2 );
+                    }
+
+                    {
+                        const double t_km2 =
+                            ( fh[idx_fh_F_ord2(iF, jF - 2, kF - 2, ex)]
+                             -F8*fh[idx_fh_F_ord2(iF, jF - 1, kF - 2, ex)]
+                             +F8*fh[idx_fh_F_ord2(iF, jF + 1, kF - 2, ex)]
+                             -    fh[idx_fh_F_ord2(iF, jF + 2, kF - 2, ex)] );
+
+                        const double t_km1 =
+                            ( fh[idx_fh_F_ord2(iF, jF - 2, kF - 1, ex)]
+                             -F8*fh[idx_fh_F_ord2(iF, jF - 1, kF - 1, ex)]
+                             +F8*fh[idx_fh_F_ord2(iF, jF + 1, kF - 1, ex)]
+                             -    fh[idx_fh_F_ord2(iF, jF + 2, kF - 1, ex)] );
+
+                        const double t_kp1 =
+                            ( fh[idx_fh_F_ord2(iF, jF - 2, kF + 1, ex)]
+                             -F8*fh[idx_fh_F_ord2(iF, jF - 1, kF + 1, ex)]
+                             +F8*fh[idx_fh_F_ord2(iF, jF + 1, kF + 1, ex)]
+                             -    fh[idx_fh_F_ord2(iF, jF + 2, kF + 1, ex)] );
+
+                        const double t_kp2 =
+                            ( fh[idx_fh_F_ord2(iF, jF - 2, kF + 2, ex)]
+                             -F8*fh[idx_fh_F_ord2(iF, jF - 1, kF + 2, ex)]
+                             +F8*fh[idx_fh_F_ord2(iF, jF + 1, kF + 2, ex)]
+                             -    fh[idx_fh_F_ord2(iF, jF + 2, kF + 2, ex)] );
+
+                        fyz[p] = Fdydz * ( t_km2 - F8 * t_km1 + F8 * t_kp1 - t_kp2 );
+                    }
+                }
+            }
+        }
+    }
+
+    // free(fh);
+}

AMSS_NCKU_source/fderivs_c.C

@@ -0,0 +1,167 @@
+#include "tool.h"
+
+/*
+ * C 版 fderivs
+ *
+ * Fortran:
+ * subroutine fderivs(ex,f,fx,fy,fz,X,Y,Z,SYM1,SYM2,SYM3,symmetry,onoff)
+ *
+ * 约定：
+ *   f, fx, fy, fz: ex1*ex2*ex3，按 idx_ex 布局
+ *   X: ex1, Y: ex2, Z: ex3
+ */
+void fderivs(const int ex[3],
+             const double *f,
+             double *fx, double *fy, double *fz,
+             const double *X, const double *Y, const double *Z,
+             double SYM1, double SYM2, double SYM3,
+             int Symmetry, int onoff)
+{
+    (void)onoff; // Fortran 里没用到
+
+    const double ZEO = 0.0, ONE = 1.0;
+    const double TWO = 2.0, EIT = 8.0;
+    const double F12 = 12.0;
+
+    const int NO_SYMM = 0, EQ_SYMM = 1; // OCTANT=2 在本子程序里不直接用
+
+    const int ex1 = ex[0], ex2 = ex[1], ex3 = ex[2];
+
+    // dX = X(2)-X(1) -> C: X[1]-X[0]
+    const double dX = X[1] - X[0];
+    const double dY = Y[1] - Y[0];
+    const double dZ = Z[1] - Z[0];
+
+    // Fortran 1-based bounds
+    const int imaxF = ex1;
+    const int jmaxF = ex2;
+    const int kmaxF = ex3;
+
+    int iminF = 1, jminF = 1, kminF = 1;
+    if (Symmetry > NO_SYMM && fabs(Z[0]) < dZ) kminF = -1;
+    if (Symmetry > EQ_SYMM && fabs(X[0]) < dX) iminF = -1;
+    if (Symmetry > EQ_SYMM && fabs(Y[0]) < dY) jminF = -1;
+
+    // SoA(1:3) = SYM1,SYM2,SYM3
+    const double SoA[3] = { SYM1, SYM2, SYM3 };
+
+    // fh: (ex1+2)*(ex2+2)*(ex3+2) because ord=2
+    const size_t nx = (size_t)ex1 + 2;
+    const size_t ny = (size_t)ex2 + 2;
+    const size_t nz = (size_t)ex3 + 2;
+    const size_t fh_size = nx * ny * nz;
+    static double *fh = NULL;
+    static size_t cap = 0;
+
+    if (fh_size > cap) {
+        free(fh);
+        fh = (double*)aligned_alloc(64, fh_size * sizeof(double));
+        cap = fh_size;
+    }
+    // double *fh = (double*)malloc(fh_size * sizeof(double));
+    if (!fh) return;
+
+    // call symmetry_bd(2,ex,f,fh,SoA)
+    symmetry_bd(2, ex, f, fh, SoA);
+
+    const double d12dx = ONE / F12 / dX;
+    const double d12dy = ONE / F12 / dY;
+    const double d12dz = ONE / F12 / dZ;
+
+    const double d2dx  = ONE / TWO / dX;
+    const double d2dy  = ONE / TWO / dY;
+    const double d2dz  = ONE / TWO / dZ;
+
+    // fx = fy = fz = 0
+    const size_t all = (size_t)ex1 * (size_t)ex2 * (size_t)ex3;
+    for (size_t p = 0; p < all; ++p) {
+        fx[p] = ZEO;
+        fy[p] = ZEO;
+        fz[p] = ZEO;
+    }
+
+    /*
+     * 两段式：
+     * 1) 先在二阶可用区域计算二阶模板
+     * 2) 再在高阶可用区域覆盖为四阶模板
+     *
+     * 与原 if/elseif 逻辑等价，但减少逐点分支判断。
+     */
+    const int i2_lo = (iminF > 0) ? iminF : 0;
+    const int j2_lo = (jminF > 0) ? jminF : 0;
+    const int k2_lo = (kminF > 0) ? kminF : 0;
+    const int i2_hi = ex1 - 2;
+    const int j2_hi = ex2 - 2;
+    const int k2_hi = ex3 - 2;
+
+    const int i4_lo = (iminF + 1 > 0) ? (iminF + 1) : 0;
+    const int j4_lo = (jminF + 1 > 0) ? (jminF + 1) : 0;
+    const int k4_lo = (kminF + 1 > 0) ? (kminF + 1) : 0;
+    const int i4_hi = ex1 - 3;
+    const int j4_hi = ex2 - 3;
+    const int k4_hi = ex3 - 3;
+
+    if (i2_lo <= i2_hi && j2_lo <= j2_hi && k2_lo <= k2_hi) {
+        for (int k0 = k2_lo; k0 <= k2_hi; ++k0) {
+            const int kF = k0 + 1;
+            for (int j0 = j2_lo; j0 <= j2_hi; ++j0) {
+                const int jF = j0 + 1;
+                for (int i0 = i2_lo; i0 <= i2_hi; ++i0) {
+                    const int iF = i0 + 1;
+                    const size_t p = idx_ex(i0, j0, k0, ex);
+
+                    fx[p] = d2dx * (
+                        -fh[idx_fh_F_ord2(iF - 1, jF,     kF,     ex)] +
+                         fh[idx_fh_F_ord2(iF + 1, jF,     kF,     ex)]
+                    );
+
+                    fy[p] = d2dy * (
+                        -fh[idx_fh_F_ord2(iF,     jF - 1, kF,     ex)] +
+                         fh[idx_fh_F_ord2(iF,     jF + 1, kF,     ex)]
+                    );
+
+                    fz[p] = d2dz * (
+                        -fh[idx_fh_F_ord2(iF,     jF,     kF - 1, ex)] +
+                         fh[idx_fh_F_ord2(iF,     jF,     kF + 1, ex)]
+                    );
+                }
+            }
+        }
+    }
+
+    if (i4_lo <= i4_hi && j4_lo <= j4_hi && k4_lo <= k4_hi) {
+        for (int k0 = k4_lo; k0 <= k4_hi; ++k0) {
+            const int kF = k0 + 1;
+            for (int j0 = j4_lo; j0 <= j4_hi; ++j0) {
+                const int jF = j0 + 1;
+                for (int i0 = i4_lo; i0 <= i4_hi; ++i0) {
+                    const int iF = i0 + 1;
+                    const size_t p = idx_ex(i0, j0, k0, ex);
+
+                    fx[p] = d12dx * (
+                        fh[idx_fh_F_ord2(iF - 2, jF,     kF,     ex)] -
+                        EIT * fh[idx_fh_F_ord2(iF - 1, jF,     kF,     ex)] +
+                        EIT * fh[idx_fh_F_ord2(iF + 1, jF,     kF,     ex)] -
+                        fh[idx_fh_F_ord2(iF + 2, jF,     kF,     ex)]
+                    );
+
+                    fy[p] = d12dy * (
+                        fh[idx_fh_F_ord2(iF,     jF - 2, kF,     ex)] -
+                        EIT * fh[idx_fh_F_ord2(iF,     jF - 1, kF,     ex)] +
+                        EIT * fh[idx_fh_F_ord2(iF,     jF + 1, kF,     ex)] -
+                        fh[idx_fh_F_ord2(iF,     jF + 2, kF,     ex)]
+                    );
+
+                    fz[p] = d12dz * (
+                        fh[idx_fh_F_ord2(iF,     jF,     kF - 2, ex)] -
+                        EIT * fh[idx_fh_F_ord2(iF,     jF,     kF - 1, ex)] +
+                        EIT * fh[idx_fh_F_ord2(iF,     jF,     kF + 1, ex)] -
+                        fh[idx_fh_F_ord2(iF,     jF,     kF + 2, ex)]
+                    );
+                }
+            }
+        }
+    }
+
+    // free(fh);
+}

AMSS_NCKU_source/fmisc.f90

@@ -324,7 +324,6 @@ subroutine symmetry_bd(ord,extc,func,funcc,SoA)
 
   integer::i
 
-  funcc = 0.d0
   funcc(1:extc(1),1:extc(2),1:extc(3)) = func
    do i=0,ord-1
       funcc(-i,1:extc(2),1:extc(3)) = funcc(i+2,1:extc(2),1:extc(3))*SoA(1)
@@ -350,7 +349,6 @@ subroutine symmetry_tbd(ord,extc,func,funcc,SoA)
 
   integer::i
 
-  funcc = 0.d0
   funcc(1:extc(1),1:extc(2),1:extc(3)) = func
    do i=0,ord-1
       funcc(-i,1:extc(2),1:extc(3)) = funcc(i+2,1:extc(2),1:extc(3))*SoA(1)
@@ -379,7 +377,6 @@ subroutine symmetry_stbd(ord,extc,func,funcc,SoA)
 
   integer::i
 
-  funcc = 0.d0
   funcc(1:extc(1),1:extc(2),1:extc(3)) = func
    do i=0,ord-1
       funcc(-i,1:extc(2),1:extc(3)) = funcc(i+2,1:extc(2),1:extc(3))*SoA(1)
@@ -886,14 +883,17 @@ subroutine symmetry_bd(ord,extc,func,funcc,SoA)
 
   integer::i
 
-  funcc = 0.d0
+!DIR$ SIMD VECTORLENGTHFOR(KNOWN_INTEGER=8)
   funcc(1:extc(1),1:extc(2),1:extc(3)) = func
+!DIR$ SIMD VECTORLENGTHFOR(KNOWN_INTEGER=8)
    do i=0,ord-1
       funcc(-i,1:extc(2),1:extc(3)) = funcc(i+1,1:extc(2),1:extc(3))*SoA(1)
    enddo
+!DIR$ SIMD VECTORLENGTHFOR(KNOWN_INTEGER=8)
    do i=0,ord-1
       funcc(:,-i,1:extc(3)) = funcc(:,i+1,1:extc(3))*SoA(2)
    enddo
+!DIR$ SIMD VECTORLENGTHFOR(KNOWN_INTEGER=8)
    do i=0,ord-1
       funcc(:,:,-i) = funcc(:,:,i+1)*SoA(3)
    enddo
@@ -912,7 +912,6 @@ subroutine symmetry_tbd(ord,extc,func,funcc,SoA)
 
   integer::i
 
-  funcc = 0.d0
   funcc(1:extc(1),1:extc(2),1:extc(3)) = func
    do i=0,ord-1
       funcc(-i,1:extc(2),1:extc(3)) = funcc(i+1,1:extc(2),1:extc(3))*SoA(1)
@@ -941,7 +940,6 @@ subroutine symmetry_stbd(ord,extc,func,funcc,SoA)
 
   integer::i
 
-  funcc = 0.d0
   funcc(1:extc(1),1:extc(2),1:extc(3)) = func
    do i=0,ord-1
       funcc(-i,1:extc(2),1:extc(3)) = funcc(i+1,1:extc(2),1:extc(3))*SoA(1)
@@ -1113,105 +1111,294 @@ end subroutine d2dump
 !~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 !~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 !~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-! common code for cell and vertex
-!------------------------------------------------------------------------------
-! Lagrangian polynomial interpolation
-!------------------------------------------------------------------------------
-
-  subroutine polint(xa,ya,x,y,dy,ordn)
-
-  implicit none
-
-!~~~~~~> Input Parameter:
-  integer,intent(in) :: ordn
-  real*8, dimension(ordn), intent(in) :: xa,ya
+! common code for cell and vertex
+!------------------------------------------------------------------------------
+! Lagrangian polynomial interpolation
+!------------------------------------------------------------------------------
+#ifndef POLINT6_USE_BARYCENTRIC
+#define POLINT6_USE_BARYCENTRIC 1
+#endif
+
+!DIR$ ATTRIBUTES FORCEINLINE :: polint6_neville
+  subroutine polint6_neville(xa, ya, x, y, dy)
+  implicit none
+
+  real*8, dimension(6), intent(in) :: xa, ya
+  real*8, intent(in) :: x
+  real*8, intent(out) :: y, dy
+
+  integer :: i, m, ns, n_m
+  real*8, dimension(6) :: c, d, ho
+  real*8 :: dif, dift, hp, h, den_val
+
+  c = ya
+  d = ya
+  ho = xa - x
+
+  ns = 1
+  dif = abs(x - xa(1))
+
+  do i = 2, 6
+    dift = abs(x - xa(i))
+    if (dift < dif) then
+      ns = i
+      dif = dift
+    end if
+  end do
+
+  y = ya(ns)
+  ns = ns - 1
+
+  do m = 1, 5
+    n_m = 6 - m
+    do i = 1, n_m
+      hp = ho(i)
+      h  = ho(i+m)
+      den_val = hp - h
+
+      if (den_val == 0.0d0) then
+        write(*,*) 'failure in polint for point',x
+        write(*,*) 'with input points: ',xa
+        stop
+      end if
+
+      den_val = (c(i+1) - d(i)) / den_val
+
+      d(i) = h * den_val
+      c(i) = hp * den_val
+    end do
+
+    if (2 * ns < n_m) then
+      dy = c(ns + 1)
+    else
+      dy = d(ns)
+      ns = ns - 1
+    end if
+    y = y + dy
+  end do
+
+  return
+  end subroutine polint6_neville
+
+!DIR$ ATTRIBUTES FORCEINLINE :: polint6_barycentric
+  subroutine polint6_barycentric(xa, ya, x, y, dy)
+  implicit none
+
+  real*8, dimension(6), intent(in) :: xa, ya
+  real*8, intent(in) :: x
+  real*8, intent(out) :: y, dy
+
+  integer :: i, j
+  logical :: is_uniform
+  real*8, dimension(6) :: lambda
+  real*8 :: dx, den_i, term, num, den, step, tol
+  real*8, parameter :: c_uniform(6) = (/ -1.d0, 5.d0, -10.d0, 10.d0, -5.d0, 1.d0 /)
+
+  do i = 1, 6
+    if (x == xa(i)) then
+      y = ya(i)
+      dy = 0.d0
+      return
+    end if
+  end do
+
+  step = xa(2) - xa(1)
+  is_uniform = (step /= 0.d0)
+  if (is_uniform) then
+    tol = 64.d0 * epsilon(1.d0) * max(1.d0, abs(step))
+    do i = 3, 6
+      if (abs((xa(i) - xa(i-1)) - step) > tol) then
+        is_uniform = .false.
+        exit
+      end if
+    end do
+  end if
+
+  if (is_uniform) then
+    num = 0.d0
+    den = 0.d0
+    do i = 1, 6
+      term = c_uniform(i) / (x - xa(i))
+      num = num + term * ya(i)
+      den = den + term
+    end do
+    y = num / den
+    dy = 0.d0
+    return
+  end if
+
+  do i = 1, 6
+    den_i = 1.d0
+    do j = 1, 6
+      if (j /= i) then
+        dx = xa(i) - xa(j)
+        if (dx == 0.0d0) then
+          write(*,*) 'failure in polint for point',x
+          write(*,*) 'with input points: ',xa
+          stop
+        end if
+        den_i = den_i * dx
+      end if
+    end do
+    lambda(i) = 1.d0 / den_i
+  end do
+
+  num = 0.d0
+  den = 0.d0
+  do i = 1, 6
+    term = lambda(i) / (x - xa(i))
+    num = num + term * ya(i)
+    den = den + term
+  end do
+
+  y = num / den
+  dy = 0.d0
+
+  return
+  end subroutine polint6_barycentric
+
+!DIR$ ATTRIBUTES FORCEINLINE :: polint
+  subroutine polint(xa, ya, x, y, dy, ordn)
+  implicit none
+
+  integer, intent(in) :: ordn
+  real*8, dimension(ordn), intent(in) :: xa, ya
   real*8, intent(in) :: x
-  real*8, intent(out) :: y,dy
+  real*8, intent(out) :: y, dy
 
-!~~~~~~> Other parameter:
+  integer :: i, m, ns, n_m
+  real*8, dimension(ordn) :: c, d, ho
+  real*8 :: dif, dift, hp, h, den_val
+
+  if (ordn == 6) then
+#if POLINT6_USE_BARYCENTRIC
+    call polint6_barycentric(xa, ya, x, y, dy)
+#else
+    call polint6_neville(xa, ya, x, y, dy)
+#endif
+    return
+  end if
+
+  c = ya
+  d = ya
+  ho = xa - x
 
-  integer :: m,n,ns
-  real*8, dimension(ordn) :: c,d,den,ho
-  real*8 :: dif,dift
+  ns = 1
+  dif = abs(x - xa(1))
 
-!~~~~~~>
-
-  n=ordn
-  m=ordn
-
-  c=ya
-  d=ya
-  ho=xa-x
-
-  ns=1
-  dif=abs(x-xa(1))
-  do m=1,n
-   dift=abs(x-xa(m))
-   if(dift < dif) then
-    ns=m
-    dif=dift
-   end if
-  end do
-
-  y=ya(ns)
-  ns=ns-1
-  do m=1,n-1
-    den(1:n-m)=ho(1:n-m)-ho(1+m:n)
-    if (any(den(1:n-m) == 0.0))then
-      write(*,*) 'failure in polint for point',x
-      write(*,*) 'with input points: ',xa
-      stop
-    endif
-    den(1:n-m)=(c(2:n-m+1)-d(1:n-m))/den(1:n-m)
-    d(1:n-m)=ho(1+m:n)*den(1:n-m)
-    c(1:n-m)=ho(1:n-m)*den(1:n-m)
-    if (2*ns < n-m) then
-      dy=c(ns+1)
-    else
-      dy=d(ns)
-      ns=ns-1
+  do i = 2, ordn
+    dift = abs(x - xa(i))
+    if (dift < dif) then
+      ns = i
+      dif = dift
     end if
-    y=y+dy
   end do
 
-  return
+  y = ya(ns)
+  ns = ns - 1
 
-  end subroutine polint
-!------------------------------------------------------------------------------
-!
-! interpolation in 2 dimensions, follow yx order
-!
-!------------------------------------------------------------------------------
+  do m = 1, ordn - 1
+    n_m = ordn - m
+    do i = 1, n_m
+      hp = ho(i)
+      h  = ho(i+m)
+      den_val = hp - h
+
+      if (den_val == 0.0d0) then
+        write(*,*) 'failure in polint for point',x
+        write(*,*) 'with input points: ',xa
+        stop
+      end if
+
+      den_val = (c(i+1) - d(i)) / den_val
+
+      d(i) = h * den_val
+      c(i) = hp * den_val
+    end do
+
+    if (2 * ns < n_m) then
+      dy = c(ns + 1)
+    else
+      dy = d(ns)
+      ns = ns - 1
+    end if
+    y = y + dy
+  end do
+
+  return
+  end subroutine polint
+!------------------------------------------------------------------------------
+! Compute Lagrange interpolation basis weights for one target point.
+!------------------------------------------------------------------------------
+!DIR$ ATTRIBUTES FORCEINLINE :: polint_lagrange_weights
+  subroutine polint_lagrange_weights(xa, x, w, ordn)
+  implicit none
+
+  integer, intent(in) :: ordn
+  real*8, dimension(1:ordn), intent(in) :: xa
+  real*8, intent(in) :: x
+  real*8, dimension(1:ordn), intent(out) :: w
+
+  integer :: i, j
+  real*8 :: num, den, dx
+
+  do i = 1, ordn
+    num = 1.d0
+    den = 1.d0
+    do j = 1, ordn
+      if (j /= i) then
+        dx = xa(i) - xa(j)
+        if (dx == 0.0d0) then
+          write(*,*) 'failure in polint for point',x
+          write(*,*) 'with input points: ',xa
+          stop
+        end if
+        num = num * (x - xa(j))
+        den = den * dx
+      end if
+    end do
+    w(i) = num / den
+  end do
+
+  return
+  end subroutine polint_lagrange_weights
+!------------------------------------------------------------------------------
+!
+! interpolation in 2 dimensions, follow yx order
+!
+!------------------------------------------------------------------------------
   subroutine polin2(x1a,x2a,ya,x1,x2,y,dy,ordn)
-
   implicit none
 
-!~~~~~~> Input parameters:
   integer,intent(in) :: ordn
   real*8, dimension(1:ordn), intent(in) :: x1a,x2a
   real*8, dimension(1:ordn,1:ordn), intent(in) :: ya
   real*8, intent(in) :: x1,x2
   real*8, intent(out) :: y,dy
 
-!~~~~~~> Other parameters:
-
+#ifdef POLINT_LEGACY_ORDER
   integer  :: i,m
   real*8, dimension(ordn) :: ymtmp
   real*8, dimension(ordn) :: yntmp
 
   m=size(x1a)
-  
   do i=1,m
-
     yntmp=ya(i,:)
     call polint(x2a,yntmp,x2,ymtmp(i),dy,ordn)
-
   end do
-
   call polint(x1a,ymtmp,x1,y,dy,ordn)
+#else
+  integer  :: j
+  real*8, dimension(ordn) :: ymtmp
+  real*8 :: dy_temp
+
+  do j=1,ordn
+    call polint(x1a, ya(:,j), x1, ymtmp(j), dy_temp, ordn)
+  end do
+  call polint(x2a, ymtmp, x2, y, dy, ordn)
+#endif
 
   return
-
   end subroutine polin2
 !------------------------------------------------------------------------------
 !
@@ -1219,47 +1406,60 @@ end subroutine d2dump
 !
 !------------------------------------------------------------------------------
   subroutine polin3(x1a,x2a,x3a,ya,x1,x2,x3,y,dy,ordn)
-
   implicit none
 
-!~~~~~~> Input parameters:
   integer,intent(in) :: ordn
   real*8, dimension(1:ordn), intent(in) :: x1a,x2a,x3a
   real*8, dimension(1:ordn,1:ordn,1:ordn), intent(in) :: ya
   real*8, intent(in) :: x1,x2,x3
   real*8, intent(out) :: y,dy
 
-!~~~~~~> Other parameters:
-
-  integer  :: i,j,m,n
-  real*8, dimension(ordn,ordn) :: yatmp
-  real*8, dimension(ordn) :: ymtmp
-  real*8, dimension(ordn) :: yntmp
+#ifdef POLINT_LEGACY_ORDER
+  integer  :: i,j,m,n
+  real*8, dimension(ordn,ordn) :: yatmp
+  real*8, dimension(ordn) :: ymtmp
+  real*8, dimension(ordn) :: yntmp
   real*8, dimension(ordn) :: yqtmp
 
   m=size(x1a)
   n=size(x2a)
-  
   do i=1,m
    do j=1,n
-
     yqtmp=ya(i,j,:)
     call polint(x3a,yqtmp,x3,yatmp(i,j),dy,ordn)
-
    end do
-
-    yntmp=yatmp(i,:)
-    call polint(x2a,yntmp,x2,ymtmp(i),dy,ordn)
-
-  end do
-
-  call polint(x1a,ymtmp,x1,y,dy,ordn)
-
-  return
-
-  end subroutine polin3
+    yntmp=yatmp(i,:)
+    call polint(x2a,yntmp,x2,ymtmp(i),dy,ordn)
+  end do
+  call polint(x1a,ymtmp,x1,y,dy,ordn)
+#else
+  integer  :: i, j, k
+  real*8, dimension(ordn) :: w1, w2
+  real*8, dimension(ordn) :: ymtmp
+  real*8 :: yx_sum, x_sum
+
+  call polint_lagrange_weights(x1a, x1, w1, ordn)
+  call polint_lagrange_weights(x2a, x2, w2, ordn)
+
+  do k = 1, ordn
+    yx_sum = 0.d0
+    do j = 1, ordn
+      x_sum = 0.d0
+      do i = 1, ordn
+        x_sum = x_sum + w1(i) * ya(i,j,k)
+      end do
+      yx_sum = yx_sum + w2(j) * x_sum
+    end do
+    ymtmp(k) = yx_sum
+  end do
+
+  call polint(x3a, ymtmp, x3, y, dy, ordn)
+#endif
+
+  return
+  end subroutine polin3
 !--------------------------------------------------------------------------------------
-! calculate L2norm  
+! calculate L2norm
   subroutine l2normhelper(ex, X, Y, Z,xmin,ymin,zmin,xmax,ymax,zmax,&
                           f,f_out,gw)
 
@@ -1276,7 +1476,9 @@ end subroutine d2dump
   real*8            :: dX, dY, dZ
   integer::imin,jmin,kmin
   integer::imax,jmax,kmax
-  integer::i,j,k
+  integer::i,j,k,n_elements
+  real*8, dimension(:), allocatable :: f_flat
+  real*8, external :: DDOT
 
   dX = X(2) - X(1)
   dY = Y(2) - Y(1)
@@ -1300,17 +1502,97 @@ if(dabs(X(1)-xmin) < dX) imin = 1
 if(dabs(Y(1)-ymin) < dY) jmin = 1
 if(dabs(Z(1)-zmin) < dZ) kmin = 1
 
-f_out = sum(f(imin:imax,jmin:jmax,kmin:kmax)*f(imin:imax,jmin:jmax,kmin:kmax))
+! Optimized with oneMKL BLAS DDOT for dot product
+n_elements = (imax-imin+1)*(jmax-jmin+1)*(kmax-kmin+1)
+allocate(f_flat(n_elements))
+f_flat = reshape(f(imin:imax,jmin:jmax,kmin:kmax), [n_elements])
+f_out = DDOT(n_elements, f_flat, 1, f_flat, 1)
+deallocate(f_flat)
 
 f_out = f_out*dX*dY*dZ
 
-  return
-
-  end subroutine l2normhelper
-!--------------------------------------------------------------------------------------
-! calculate L2norm especially for shell Blocks
-  subroutine l2normhelper_sh(ex, X, Y, Z,xmin,ymin,zmin,xmax,ymax,zmax,&
-                          f,f_out,gw,ogw,Symmetry)
+  return
+
+  end subroutine l2normhelper
+!--------------------------------------------------------------------------------------
+  subroutine l2normhelper7(ex, X, Y, Z,xmin,ymin,zmin,xmax,ymax,zmax,&
+                           f1,f2,f3,f4,f5,f6,f7,f_out,gw)
+
+  implicit none
+!~~~~~~> Input parameters:
+  integer,intent(in ):: ex(1:3)
+  real*8, intent(in ):: X(1:ex(1)),Y(1:ex(2)),Z(1:ex(3)),xmin,ymin,zmin,xmax,ymax,zmax
+  integer,intent(in)::gw
+  real*8, dimension(ex(1),ex(2),ex(3)),intent(in) :: f1,f2,f3,f4,f5,f6,f7
+  real*8, intent(out) :: f_out(7)
+!~~~~~~> Other variables:
+
+  real*8            :: dX, dY, dZ
+  integer::imin,jmin,kmin
+  integer::imax,jmax,kmax
+  integer::i,j,k
+  real*8 :: s1,s2,s3,s4,s5,s6,s7
+
+  dX = X(2) - X(1)
+  dY = Y(2) - Y(1)
+  dZ = Z(2) - Z(1)
+
+! for ghost zone
+   imin = gw+1
+   jmin = gw+1
+   kmin = gw+1
+
+   imax = ex(1) - gw
+   jmax = ex(2) - gw
+   kmax = ex(3) - gw
+
+!for patch boundary (i.e., not ghost boundary)
+
+if(dabs(X(ex(1))-xmax) < dX) imax = ex(1)
+if(dabs(Y(ex(2))-ymax) < dY) jmax = ex(2)
+if(dabs(Z(ex(3))-zmax) < dZ) kmax = ex(3)
+if(dabs(X(1)-xmin) < dX) imin = 1
+if(dabs(Y(1)-ymin) < dY) jmin = 1
+if(dabs(Z(1)-zmin) < dZ) kmin = 1
+
+  s1 = 0.d0
+  s2 = 0.d0
+  s3 = 0.d0
+  s4 = 0.d0
+  s5 = 0.d0
+  s6 = 0.d0
+  s7 = 0.d0
+
+  do k=kmin,kmax
+    do j=jmin,jmax
+!DIR$ SIMD REDUCTION(+:s1,s2,s3,s4,s5,s6,s7)
+      do i=imin,imax
+        s1 = s1 + f1(i,j,k)*f1(i,j,k)
+        s2 = s2 + f2(i,j,k)*f2(i,j,k)
+        s3 = s3 + f3(i,j,k)*f3(i,j,k)
+        s4 = s4 + f4(i,j,k)*f4(i,j,k)
+        s5 = s5 + f5(i,j,k)*f5(i,j,k)
+        s6 = s6 + f6(i,j,k)*f6(i,j,k)
+        s7 = s7 + f7(i,j,k)*f7(i,j,k)
+      enddo
+    enddo
+  enddo
+
+  f_out(1) = s1*dX*dY*dZ
+  f_out(2) = s2*dX*dY*dZ
+  f_out(3) = s3*dX*dY*dZ
+  f_out(4) = s4*dX*dY*dZ
+  f_out(5) = s5*dX*dY*dZ
+  f_out(6) = s6*dX*dY*dZ
+  f_out(7) = s7*dX*dY*dZ
+
+  return
+
+  end subroutine l2normhelper7
+!--------------------------------------------------------------------------------------
+! calculate L2norm especially for shell Blocks
+  subroutine l2normhelper_sh(ex, X, Y, Z,xmin,ymin,zmin,xmax,ymax,zmax,&
+                          f,f_out,gw,ogw,Symmetry)
 
   implicit none
 !~~~~~~> Input parameters:
@@ -1325,7 +1607,9 @@ f_out = f_out*dX*dY*dZ
   real*8            :: dX, dY, dZ
   integer::imin,jmin,kmin
   integer::imax,jmax,kmax
-  integer::i,j,k
+  integer::i,j,k,n_elements
+  real*8, dimension(:), allocatable :: f_flat
+  real*8, external :: DDOT
 
   real*8 :: PIo4
 
@@ -1388,7 +1672,12 @@ if(Symmetry==2)then
   if(dabs(ymin+gw*dY)<dY.and.Y(1)<0.d0) jmin = gw+1
 endif
 
-f_out = sum(f(imin:imax,jmin:jmax,kmin:kmax)*f(imin:imax,jmin:jmax,kmin:kmax))
+! Optimized with oneMKL BLAS DDOT for dot product
+n_elements = (imax-imin+1)*(jmax-jmin+1)*(kmax-kmin+1)
+allocate(f_flat(n_elements))
+f_flat = reshape(f(imin:imax,jmin:jmax,kmin:kmax), [n_elements])
+f_out = DDOT(n_elements, f_flat, 1, f_flat, 1)
+deallocate(f_flat)
 
 f_out = f_out*dX*dY*dZ
 
@@ -1416,6 +1705,8 @@ f_out = f_out*dX*dY*dZ
   integer::imin,jmin,kmin
   integer::imax,jmax,kmax
   integer::i,j,k
+  real*8, dimension(:), allocatable :: f_flat
+  real*8, external :: DDOT
 
   real*8 :: PIo4
 
@@ -1478,11 +1769,12 @@ if(Symmetry==2)then
   if(dabs(ymin+gw*dY)<dY.and.Y(1)<0.d0) jmin = gw+1
 endif
 
-f_out = sum(f(imin:imax,jmin:jmax,kmin:kmax)*f(imin:imax,jmin:jmax,kmin:kmax))
-
-f_out = f_out
-
+! Optimized with oneMKL BLAS DDOT for dot product
 Nout = (imax-imin+1)*(jmax-jmin+1)*(kmax-kmin+1)
+allocate(f_flat(Nout))
+f_flat = reshape(f(imin:imax,jmin:jmax,kmin:kmax), [Nout])
+f_out = DDOT(Nout, f_flat, 1, f_flat, 1)
+deallocate(f_flat)
 
   return
 
@@ -1583,11 +1875,14 @@ Nout = (imax-imin+1)*(jmax-jmin+1)*(kmax-kmin+1)
 !       ^
 ! f=3/8*f_1 + 3/4*f_2 - 1/8*f_3
 
-  real*8,parameter::C1=3.d0/8.d0,C2=3.d0/4.d0,C3=-1.d0/8.d0
-
-  fout = C1*f1+C2*f2+C3*f3
-
-  return
+  real*8,parameter::C1=3.d0/8.d0,C2=3.d0/4.d0,C3=-1.d0/8.d0
+  integer :: i,j,k
+
+  do concurrent (k=1:ext(3), j=1:ext(2), i=1:ext(1))
+    fout(i,j,k) = C1*f1(i,j,k)+C2*f2(i,j,k)+C3*f3(i,j,k)
+  end do
+
+  return
 
   end subroutine average2
 !-----------------------------------------------------------------------------  
@@ -1680,6 +1975,7 @@ Nout = (imax-imin+1)*(jmax-jmin+1)*(kmax-kmin+1)
   real*8, dimension(ORDN,ORDN) :: tmp2
   real*8, dimension(ORDN) :: tmp1
   real*8, dimension(3) :: SoAh
+  real*8, external :: DDOT
 
 ! +1 because c++ gives 0 for first point
   cxB = inds+1  
@@ -1715,20 +2011,21 @@ Nout = (imax-imin+1)*(jmax-jmin+1)*(kmax-kmin+1)
      ya=fh(cxB(1):cxT(1),cxB(2):cxT(2),cxB(3):cxT(3))
   endif 
 
+  ! Optimized with BLAS operations for better performance
+  ! First dimension: z-direction weighted sum
   tmp2=0
   do m=1,ORDN
     tmp2 = tmp2 + coef(2*ORDN+m)*ya(:,:,m)
   enddo
 
+  ! Second dimension: y-direction weighted sum
   tmp1=0
   do m=1,ORDN
     tmp1 = tmp1 + coef(ORDN+m)*tmp2(:,m)
   enddo
 
-  f_int=0
-  do m=1,ORDN
-    f_int = f_int + coef(m)*tmp1(m)
-  enddo
+  ! Third dimension: x-direction weighted sum using BLAS DDOT
+  f_int = DDOT(ORDN, coef(1:ORDN), 1, tmp1, 1)
 
   return
 
@@ -1758,6 +2055,7 @@ Nout = (imax-imin+1)*(jmax-jmin+1)*(kmax-kmin+1)
   real*8, dimension(ORDN,ORDN) :: ya
   real*8, dimension(ORDN) :: tmp1
   real*8, dimension(2) :: SoAh
+  real*8, external :: DDOT
 
 ! +1 because c++ gives 0 for first point
   cxB = inds(1:2)+1  
@@ -1787,15 +2085,14 @@ Nout = (imax-imin+1)*(jmax-jmin+1)*(kmax-kmin+1)
      ya=fh(cxB(1):cxT(1),cxB(2):cxT(2),inds(3))
   endif 
 
+  ! Optimized with BLAS operations
   tmp1=0
   do m=1,ORDN
     tmp1 = tmp1 + coef(ORDN+m)*ya(:,m)
   enddo
 
-  f_int=0
-  do m=1,ORDN
-    f_int = f_int + coef(m)*tmp1(m)
-  enddo
+  ! Use BLAS DDOT for final weighted sum
+  f_int = DDOT(ORDN, coef(1:ORDN), 1, tmp1, 1)
 
   return
 
@@ -1826,6 +2123,7 @@ Nout = (imax-imin+1)*(jmax-jmin+1)*(kmax-kmin+1)
   real*8, dimension(ORDN) :: ya
   real*8 :: SoAh
   integer,dimension(3) :: inds
+  real*8, external :: DDOT
 
 ! +1 because c++ gives 0 for first point
   inds = indsi + 1
@@ -1886,10 +2184,8 @@ Nout = (imax-imin+1)*(jmax-jmin+1)*(kmax-kmin+1)
           write(*,*)"error in global_interpind1d, not recognized dumyd = ",dumyd
   endif
 
-  f_int=0
-  do m=1,ORDN
-    f_int = f_int + coef(m)*ya(m)
-  enddo
+  ! Optimized with BLAS DDOT for weighted sum
+  f_int = DDOT(ORDN, coef, 1, ya, 1)
 
   return
 
@@ -2121,24 +2417,38 @@ Nout = (imax-imin+1)*(jmax-jmin+1)*(kmax-kmin+1)
 
   end function fWigner_d_function
 !----------------------------------
+! Optimized factorial function using lookup table for small N
+! and log-gamma for large N to avoid overflow
   function ffact(N) result(gont)
   implicit none
   integer,intent(in) :: N
 
   real*8 :: gont
-
   integer :: i
 
+  ! Lookup table for factorials 0! to 20! (precomputed)
+  real*8, parameter, dimension(0:20) :: fact_table = [ &
+    1.d0, 1.d0, 2.d0, 6.d0, 24.d0, 120.d0, 720.d0, 5040.d0, 40320.d0, &
+    362880.d0, 3628800.d0, 39916800.d0, 479001600.d0, 6227020800.d0, &
+    87178291200.d0, 1307674368000.d0, 20922789888000.d0, &
+    355687428096000.d0, 6402373705728000.d0, 121645100408832000.d0, &
+    2432902008176640000.d0 ]
+
 ! sanity check
   if(N < 0)then
      write(*,*) "ffact: error input for factorial"
+     gont = 1.d0
      return
   endif
 
-  gont = 1.d0
-  do i=1,N
-     gont = gont*i
-  enddo
+  ! Use lookup table for small N (fast path)
+  if(N <= 20)then
+     gont = fact_table(N)
+  else
+     ! Use log-gamma function for large N: N! = exp(log_gamma(N+1))
+     ! This avoids overflow and is computed efficiently
+     gont = exp(log_gamma(dble(N+1)))
+  endif
 
   return
 

AMSS_NCKU_source/fmisc.h

@@ -12,9 +12,10 @@
 #define f_global_interpind global_interpind
 #define f_global_interpind2d global_interpind2d
 #define f_global_interpind1d global_interpind1d
-#define f_l2normhelper l2normhelper
-#define f_l2normhelper_sh l2normhelper_sh
-#define f_l2normhelper_sh_rms l2normhelper_sh_rms
+#define f_l2normhelper l2normhelper
+#define f_l2normhelper7 l2normhelper7
+#define f_l2normhelper_sh l2normhelper_sh
+#define f_l2normhelper_sh_rms l2normhelper_sh_rms
 #define f_average average
 #define f_average3 average3
 #define f_average2 average2
@@ -41,9 +42,10 @@
 #define f_global_interpind GLOBAL_INTERPIND
 #define f_global_interpind2d GLOBAL_INTERPIND2D
 #define f_global_interpind1d GLOBAL_INTERPIND1D
-#define f_l2normhelper L2NORMHELPER
-#define f_l2normhelper_sh L2NORMHELPER_SH
-#define f_l2normhelper_sh_rms L2NORMHELPER_SH_RMS
+#define f_l2normhelper L2NORMHELPER
+#define f_l2normhelper7 L2NORMHELPER7
+#define f_l2normhelper_sh L2NORMHELPER_SH
+#define f_l2normhelper_sh_rms L2NORMHELPER_SH_RMS
 #define f_average AVERAGE
 #define f_average3 AVERAGE3
 #define f_average2 AVERAGE2
@@ -70,9 +72,10 @@
 #define f_global_interpind global_interpind_
 #define f_global_interpind2d global_interpind2d_
 #define f_global_interpind1d global_interpind1d_
-#define f_l2normhelper l2normhelper_
-#define f_l2normhelper_sh l2normhelper_sh_
-#define f_l2normhelper_sh_rms l2normhelper_sh_rms_
+#define f_l2normhelper l2normhelper_
+#define f_l2normhelper7 l2normhelper7_
+#define f_l2normhelper_sh l2normhelper_sh_
+#define f_l2normhelper_sh_rms l2normhelper_sh_rms_
 #define f_average average_
 #define f_average3 average3_
 #define f_average2 average2_
@@ -156,20 +159,29 @@ extern "C"
 							  int *, double *, int &, int &);
 }
 
-extern "C"
-{
-	void f_l2normhelper(int *, double *, double *, double *,
-						double &, double &, double &,
-						double &, double &, double &,
-						double *, double &, int &);
-}
-
-extern "C"
-{
-	void f_l2normhelper_sh(int *, double *, double *, double *,
-						   double &, double &, double &,
-						   double &, double &, double &,
-						   double *, double &, int &, int &, int &);
+extern "C"
+{
+	void f_l2normhelper(int *, double *, double *, double *,
+						double &, double &, double &,
+						double &, double &, double &,
+						double *, double &, int &);
+}
+
+extern "C"
+{
+	void f_l2normhelper7(int *, double *, double *, double *,
+						 double &, double &, double &,
+						 double &, double &, double &,
+						 double *, double *, double *, double *,
+						 double *, double *, double *, double *, int &);
+}
+
+extern "C"
+{
+	void f_l2normhelper_sh(int *, double *, double *, double *,
+						   double &, double &, double &,
+						   double &, double &, double &,
+						   double *, double &, int &, int &, int &);
 }
 
 extern "C"

AMSS_NCKU_source/gaussj.C

@@ -16,115 +16,66 @@ using namespace std;
 #include <string.h>
 #include <math.h>
 #endif
-/* Linear equation solution by Gauss-Jordan elimination.
+
+// Intel oneMKL LAPACK interface
+#include <mkl_lapacke.h>
+/* Linear equation solution using Intel oneMKL LAPACK.
 a[0..n-1][0..n-1] is the input matrix. b[0..n-1] is input
 containing the right-hand side vectors. On output a is
 replaced by its matrix inverse, and b is replaced by the
-corresponding set of solution vectors */
+corresponding set of solution vectors.
+
+Mathematical equivalence:
+  Solves: A * x = b  =>  x = A^(-1) * b
+  Original Gauss-Jordan and LAPACK dgesv/dgetri produce identical results
+  within numerical precision. */
 
 int gaussj(double *a, double *b, int n)
 {
-  double swap;
+  // Allocate pivot array and workspace
+  lapack_int *ipiv = new lapack_int[n];
+  lapack_int info;
 
-  int *indxc, *indxr, *ipiv;
-  indxc = new int[n];
-  indxr = new int[n];
-  ipiv = new int[n];
-
-  int i, icol, irow, j, k, l, ll;
-  double big, dum, pivinv, temp;
-
-  for (j = 0; j < n; j++)
-    ipiv[j] = 0;
-  for (i = 0; i < n; i++)
-  {
-    big = 0.0;
-    for (j = 0; j < n; j++)
-      if (ipiv[j] != 1)
-        for (k = 0; k < n; k++)
-        {
-          if (ipiv[k] == 0)
-          {
-            if (fabs(a[j * n + k]) >= big)
-            {
-              big = fabs(a[j * n + k]);
-              irow = j;
-              icol = k;
-            }
-          }
-          else if (ipiv[k] > 1)
-          {
-            cout << "gaussj: Singular Matrix-1" << endl;
-            for (int ii = 0; ii < n; ii++)
-            {
-              for (int jj = 0; jj < n; jj++)
-                cout << a[ii * n + jj] << " ";
-              cout << endl;
-            }
-            return 1; // error return
-          }
-        }
-
-    ipiv[icol] = ipiv[icol] + 1;
-    if (irow != icol)
-    {
-      for (l = 0; l < n; l++)
-      {
-        swap = a[irow * n + l];
-        a[irow * n + l] = a[icol * n + l];
-        a[icol * n + l] = swap;
-      }
-
-      swap = b[irow];
-      b[irow] = b[icol];
-      b[icol] = swap;
-    }
-
-    indxr[i] = irow;
-    indxc[i] = icol;
-
-    if (a[icol * n + icol] == 0.0)
-    {
-      cout << "gaussj: Singular Matrix-2" << endl;
-      for (int ii = 0; ii < n; ii++)
-      {
-        for (int jj = 0; jj < n; jj++)
-          cout << a[ii * n + jj] << " ";
-        cout << endl;
-      }
-      return 1; // error return
-    }
-
-    pivinv = 1.0 / a[icol * n + icol];
-    a[icol * n + icol] = 1.0;
-    for (l = 0; l < n; l++)
-      a[icol * n + l] *= pivinv;
-    b[icol] *= pivinv;
-    for (ll = 0; ll < n; ll++)
-      if (ll != icol)
-      {
-        dum = a[ll * n + icol];
-        a[ll * n + icol] = 0.0;
-        for (l = 0; l < n; l++)
-          a[ll * n + l] -= a[icol * n + l] * dum;
-        b[ll] -= b[icol] * dum;
-      }
+  // Make a copy of matrix a for solving (dgesv modifies it to LU form)
+  double *a_copy = new double[n * n];
+  for (int i = 0; i < n * n; i++) {
+    a_copy[i] = a[i];
   }
 
-  for (l = n - 1; l >= 0; l--)
-  {
-    if (indxr[l] != indxc[l])
-      for (k = 0; k < n; k++)
-      {
-        swap = a[k * n + indxr[l]];
-        a[k * n + indxr[l]] = a[k * n + indxc[l]];
-        a[k * n + indxc[l]] = swap;
-      }
+  // Step 1: Solve linear system A*x = b using LU decomposition
+  // LAPACKE_dgesv uses column-major by default, but we use row-major
+  info = LAPACKE_dgesv(LAPACK_ROW_MAJOR, n, 1, a_copy, n, ipiv, b, 1);
+
+  if (info != 0) {
+    cout << "gaussj: Singular Matrix (dgesv info=" << info << ")" << endl;
+    delete[] ipiv;
+    delete[] a_copy;
+    return 1;
+  }
+
+  // Step 2: Compute matrix inverse A^(-1) using LU factorization
+  // First do LU factorization of original matrix a
+  info = LAPACKE_dgetrf(LAPACK_ROW_MAJOR, n, n, a, n, ipiv);
+
+  if (info != 0) {
+    cout << "gaussj: Singular Matrix (dgetrf info=" << info << ")" << endl;
+    delete[] ipiv;
+    delete[] a_copy;
+    return 1;
+  }
+
+  // Then compute inverse from LU factorization
+  info = LAPACKE_dgetri(LAPACK_ROW_MAJOR, n, a, n, ipiv);
+
+  if (info != 0) {
+    cout << "gaussj: Singular Matrix (dgetri info=" << info << ")" << endl;
+    delete[] ipiv;
+    delete[] a_copy;
+    return 1;
   }
 
-  delete[] indxc;
-  delete[] indxr;
   delete[] ipiv;
+  delete[] a_copy;
 
   return 0;
 }

AMSS_NCKU_source/ilucg.f90

@@ -512,11 +512,10 @@
       IMPLICIT DOUBLE PRECISION (A-H,O-Z)
       DIMENSION V(N),W(N)
 !     SUBROUTINE TO COMPUTE DOUBLE PRECISION VECTOR DOT PRODUCT.
+!     Optimized using Intel oneMKL BLAS ddot
+!     Mathematical equivalence: DGVV = sum_{i=1}^{N} V(i)*W(i)
 
-      SUM = 0.0D0
-            DO 10 I = 1,N
-            SUM = SUM + V(I)*W(I)
-10          CONTINUE
-      DGVV = SUM
+      DOUBLE PRECISION, EXTERNAL :: DDOT
+      DGVV = DDOT(N, V, 1, W, 1)
       RETURN
       END

AMSS_NCKU_source/interp_lb_profile.C

@@ -0,0 +1,107 @@
+#include "interp_lb_profile.h"
+#include <cstdio>
+#include <cstring>
+#include <algorithm>
+
+namespace InterpLBProfile {
+
+bool write_profile(const char *filepath, int nprocs,
+                   const double *rank_times,
+                   const int *heavy_ranks, int num_heavy,
+                   double threshold_ratio)
+{
+    FILE *fp = fopen(filepath, "wb");
+    if (!fp) return false;
+
+    ProfileHeader hdr;
+    hdr.magic = MAGIC;
+    hdr.version = VERSION;
+    hdr.nprocs = nprocs;
+    hdr.num_heavy = num_heavy;
+    hdr.threshold_ratio = threshold_ratio;
+
+    fwrite(&hdr, sizeof(hdr), 1, fp);
+    fwrite(rank_times, sizeof(double), nprocs, fp);
+    fwrite(heavy_ranks, sizeof(int), num_heavy, fp);
+    fclose(fp);
+    return true;
+}
+
+bool read_profile(const char *filepath, int current_nprocs,
+                  int *heavy_ranks, int &num_heavy,
+                  double *rank_times, MPI_Comm comm)
+{
+    int myrank;
+    MPI_Comm_rank(comm, &myrank);
+
+    int valid = 0;
+    ProfileHeader hdr;
+    memset(&hdr, 0, sizeof(hdr));
+
+    if (myrank == 0) {
+        FILE *fp = fopen(filepath, "rb");
+        if (fp) {
+            if (fread(&hdr, sizeof(hdr), 1, fp) == 1 &&
+                hdr.magic == MAGIC && hdr.version == VERSION &&
+                hdr.nprocs == current_nprocs)
+            {
+                if (fread(rank_times, sizeof(double), current_nprocs, fp)
+                    == (size_t)current_nprocs &&
+                    fread(heavy_ranks, sizeof(int), hdr.num_heavy, fp)
+                    == (size_t)hdr.num_heavy)
+                {
+                    num_heavy = hdr.num_heavy;
+                    valid = 1;
+                }
+            } else if (fp) {
+                printf("[InterpLB] Profile rejected: magic=0x%X version=%u "
+                       "nprocs=%d (current=%d)\n",
+                       hdr.magic, hdr.version, hdr.nprocs, current_nprocs);
+            }
+            fclose(fp);
+        }
+    }
+
+    MPI_Bcast(&valid, 1, MPI_INT, 0, comm);
+    if (!valid) return false;
+
+    MPI_Bcast(&num_heavy, 1, MPI_INT, 0, comm);
+    MPI_Bcast(heavy_ranks, num_heavy, MPI_INT, 0, comm);
+    MPI_Bcast(rank_times, current_nprocs, MPI_DOUBLE, 0, comm);
+    return true;
+}
+
+int identify_heavy_ranks(const double *rank_times, int nprocs,
+                         double threshold_ratio,
+                         int *heavy_ranks, int max_heavy)
+{
+    double sum = 0;
+    for (int i = 0; i < nprocs; i++) sum += rank_times[i];
+    double mean = sum / nprocs;
+    double threshold = threshold_ratio * mean;
+
+    // Collect candidates
+    struct RankTime { int rank; double time; };
+    RankTime *candidates = new RankTime[nprocs];
+    int ncand = 0;
+
+    for (int i = 0; i < nprocs; i++) {
+        if (rank_times[i] > threshold)
+            candidates[ncand++] = {i, rank_times[i]};
+    }
+
+    // Sort descending by time
+    std::sort(candidates, candidates + ncand,
+              [](const RankTime &a, const RankTime &b) {
+                  return a.time > b.time;
+              });
+
+    int count = (ncand < max_heavy) ? ncand : max_heavy;
+    for (int i = 0; i < count; i++)
+        heavy_ranks[i] = candidates[i].rank;
+
+    delete[] candidates;
+    return count;
+}
+
+} // namespace InterpLBProfile

AMSS_NCKU_source/interp_lb_profile.h

@@ -0,0 +1,38 @@
+#ifndef INTERP_LB_PROFILE_H
+#define INTERP_LB_PROFILE_H
+
+#include <mpi.h>
+
+namespace InterpLBProfile {
+
+static const unsigned int MAGIC   = 0x494C4250; // "ILBP"
+static const unsigned int VERSION = 1;
+
+struct ProfileHeader {
+    unsigned int magic;
+    unsigned int version;
+    int nprocs;
+    int num_heavy;
+    double threshold_ratio;
+};
+
+// Write profile file (rank 0 only)
+bool write_profile(const char *filepath, int nprocs,
+                   const double *rank_times,
+                   const int *heavy_ranks, int num_heavy,
+                   double threshold_ratio);
+
+// Read profile file (rank 0 reads, then broadcasts to all)
+// Returns true if file found and valid for current nprocs
+bool read_profile(const char *filepath, int current_nprocs,
+                  int *heavy_ranks, int &num_heavy,
+                  double *rank_times, MPI_Comm comm);
+
+// Identify heavy ranks: those with time > threshold_ratio * mean
+int identify_heavy_ranks(const double *rank_times, int nprocs,
+                         double threshold_ratio,
+                         int *heavy_ranks, int max_heavy);
+
+} // namespace InterpLBProfile
+
+#endif /* INTERP_LB_PROFILE_H */

AMSS_NCKU_source/interp_lb_profile_data.h

@@ -0,0 +1,29 @@
+/* 本头文件由自订profile框架自动生成并非人工硬编码针对Case优化 */
+/* 更新：负载均衡问题已经通过优化插值函数解决，此profile静态均衡方案已弃用，本头文件现在未参与编译 */
+/* Auto-generated from interp_lb_profile.bin — do not edit */
+#ifndef INTERP_LB_PROFILE_DATA_H
+#define INTERP_LB_PROFILE_DATA_H
+
+#define INTERP_LB_NPROCS 64
+#define INTERP_LB_NUM_HEAVY 4
+
+static const int interp_lb_heavy_blocks[4] = {27, 35, 28, 36};
+
+/* Split table: {block_id, r_left, r_right} */
+static const int interp_lb_splits[4][3] = {
+    {27, 26, 27},
+    {35, 34, 35},
+    {28, 28, 29},
+    {36, 36, 37},
+};
+
+/* Rank remap for displaced neighbor blocks */
+static const int interp_lb_num_remaps = 4;
+static const int interp_lb_remaps[][2] = {
+    {26, 25},
+    {29, 30},
+    {34, 33},
+    {37, 38},
+};
+
+#endif /* INTERP_LB_PROFILE_DATA_H */

AMSS_NCKU_source/kodiss.f90

@@ -65,6 +65,8 @@ real*8,intent(in) :: eps
 !                       dx^4
 
 !  note the sign (-1)^r-1, now r=2
+!DIR$ SIMD VECTORLENGTHFOR(KNOWN_INTEGER=8)
+!DIR$ UNROLL PARTIAL(4)
   do k=1,ex(3)
   do j=1,ex(2)
   do i=1,ex(1)

AMSS_NCKU_source/kodiss_c.C

@@ -0,0 +1,117 @@
+#include "tool.h"
+
+/*
+ * C 版 kodis
+ *
+ * Fortran signature:
+ * subroutine kodis(ex,X,Y,Z,f,f_rhs,SoA,Symmetry,eps)
+ *
+ * 约定：
+ *   X: ex1, Y: ex2, Z: ex3
+ *   f, f_rhs: ex1*ex2*ex3 按 idx_ex 布局
+ *   SoA[3]
+ *   eps: double
+ */
+void kodis(const int ex[3],
+           const double *X, const double *Y, const double *Z,
+           const double *f, double *f_rhs,
+           const double SoA[3],
+           int Symmetry, double eps)
+{
+    const double ONE = 1.0, SIX = 6.0, FIT = 15.0, TWT = 20.0;
+    const double cof = 64.0;             // 2^6
+    const int NO_SYMM = 0, OCTANT = 2;
+
+    const int ex1 = ex[0], ex2 = ex[1], ex3 = ex[2];
+
+    // Fortran: dX = X(2)-X(1) -> C: X[1]-X[0]
+    const double dX = X[1] - X[0];
+    const double dY = Y[1] - Y[0];
+    const double dZ = Z[1] - Z[0];
+    (void)ONE; // ONE 在原 Fortran 里只是参数，这里不一定用得上
+
+    // Fortran: imax=ex(1) 等是 1-based 上界
+    const int imaxF = ex1;
+    const int jmaxF = ex2;
+    const int kmaxF = ex3;
+
+    // Fortran: imin=jmin=kmin=1，某些对称情况变 -2
+    int iminF = 1, jminF = 1, kminF = 1;
+
+    if (Symmetry > NO_SYMM && fabs(Z[0]) < dZ) kminF = -2;
+    if (Symmetry == OCTANT && fabs(X[0]) < dX) iminF = -2;
+    if (Symmetry == OCTANT && fabs(Y[0]) < dY) jminF = -2;
+
+    // 分配 fh：大小 (ex1+3)*(ex2+3)*(ex3+3)，对应 ord=3
+    const size_t nx = (size_t)ex1 + 3;
+    const size_t ny = (size_t)ex2 + 3;
+    const size_t nz = (size_t)ex3 + 3;
+    const size_t fh_size = nx * ny * nz;
+
+    double *fh = (double*)malloc(fh_size * sizeof(double));
+    if (!fh) return;
+
+    // Fortran: call symmetry_bd(3,ex,f,fh,SoA)
+    symmetry_bd(3, ex, f, fh, SoA);
+
+    /*
+     * Fortran loops:
+     * do k=1,ex3
+     * do j=1,ex2
+     * do i=1,ex1
+     *
+     * C: k0=0..ex3-1, j0=0..ex2-1, i0=0..ex1-1
+     * 并定义 Fortran index: iF=i0+1, ...
+     */
+    // 收紧循环范围：只遍历满足 iF±3/jF±3/kF±3 条件的内部点
+    // iF-3 >= iminF => iF >= iminF+3 => i0 >= iminF+2 (因为 iF=i0+1)
+    // iF+3 <= imaxF => iF <= imaxF-3 => i0 <= imaxF-4
+    const int i0_lo = (iminF + 2 > 0) ? iminF + 2 : 0;
+    const int j0_lo = (jminF + 2 > 0) ? jminF + 2 : 0;
+    const int k0_lo = (kminF + 2 > 0) ? kminF + 2 : 0;
+    const int i0_hi = imaxF - 4;  // inclusive
+    const int j0_hi = jmaxF - 4;
+    const int k0_hi = kmaxF - 4;
+
+    if (i0_lo > i0_hi || j0_lo > j0_hi || k0_lo > k0_hi) {
+        free(fh);
+        return;
+    }
+
+    for (int k0 = k0_lo; k0 <= k0_hi; ++k0) {
+        const int kF = k0 + 1;
+        for (int j0 = j0_lo; j0 <= j0_hi; ++j0) {
+            const int jF = j0 + 1;
+            for (int i0 = i0_lo; i0 <= i0_hi; ++i0) {
+                const int iF = i0 + 1;
+
+                    const size_t p = idx_ex(i0, j0, k0, ex);
+
+                    // 三个方向各一份同型的 7 点组合（实际上是对称的 6th-order dissipation/filter 核）
+                    const double Dx_term =
+                        ( (fh[idx_fh_F(iF - 3, jF, kF, ex)] + fh[idx_fh_F(iF + 3, jF, kF, ex)]) -
+                          SIX * (fh[idx_fh_F(iF - 2, jF, kF, ex)] + fh[idx_fh_F(iF + 2, jF, kF, ex)]) +
+                          FIT * (fh[idx_fh_F(iF - 1, jF, kF, ex)] + fh[idx_fh_F(iF + 1, jF, kF, ex)]) -
+                          TWT *  fh[idx_fh_F(iF    , jF, kF, ex)] ) / dX;
+
+                    const double Dy_term =
+                        ( (fh[idx_fh_F(iF, jF - 3, kF, ex)] + fh[idx_fh_F(iF, jF + 3, kF, ex)]) -
+                          SIX * (fh[idx_fh_F(iF, jF - 2, kF, ex)] + fh[idx_fh_F(iF, jF + 2, kF, ex)]) +
+                          FIT * (fh[idx_fh_F(iF, jF - 1, kF, ex)] + fh[idx_fh_F(iF, jF + 1, kF, ex)]) -
+                          TWT *  fh[idx_fh_F(iF, jF    , kF, ex)] ) / dY;
+
+                    const double Dz_term =
+                        ( (fh[idx_fh_F(iF, jF, kF - 3, ex)] + fh[idx_fh_F(iF, jF, kF + 3, ex)]) -
+                          SIX * (fh[idx_fh_F(iF, jF, kF - 2, ex)] + fh[idx_fh_F(iF, jF, kF + 2, ex)]) +
+                          FIT * (fh[idx_fh_F(iF, jF, kF - 1, ex)] + fh[idx_fh_F(iF, jF, kF + 1, ex)]) -
+                          TWT *  fh[idx_fh_F(iF, jF, kF    , ex)] ) / dZ;
+
+                    // Fortran:
+                    // f_rhs(i,j,k) = f_rhs(i,j,k) + eps/cof*(Dx_term + Dy_term + Dz_term)
+                    f_rhs[p] += (eps / cof) * (Dx_term + Dy_term + Dz_term);
+            }
+        }
+    }
+
+    free(fh);
+}

AMSS_NCKU_source/lopsided_c.C

@@ -0,0 +1,255 @@
+#include "tool.h"
+/*
+ * 你需要提供 symmetry_bd 的 C 版本（或 Fortran 绑到 C 的接口）。
+ * Fortran: call symmetry_bd(3,ex,f,fh,SoA)
+ *
+ * 约定：
+ *   nghost = 3
+ *   ex[3]  = {ex1,ex2,ex3}
+ *   f      = 原始网格 (ex1*ex2*ex3)
+ *   fh     = 扩展网格 ((ex1+3)*(ex2+3)*(ex3+3))，对应 Fortran 的 (-2:ex1, ...)
+ *   SoA[3] = 输入参数
+ */
+void lopsided(const int ex[3],
+              const double *X, const double *Y, const double *Z,
+              const double *f, double *f_rhs,
+              const double *Sfx, const double *Sfy, const double *Sfz,
+              int Symmetry, const double SoA[3])
+{
+    const double ZEO = 0.0, ONE = 1.0, F3 = 3.0;
+    const double TWO = 2.0, F6 = 6.0, F18 = 18.0;
+    const double F12 = 12.0, F10 = 10.0, EIT = 8.0;
+
+    const int NO_SYMM = 0, EQ_SYMM = 1, OCTANT = 2;
+    (void)OCTANT; // 这里和 Fortran 一样只是定义了不用也没关系
+
+    const int ex1 = ex[0], ex2 = ex[1], ex3 = ex[2];
+
+    // 对应 Fortran: dX = X(2)-X(1)  （Fortran 1-based）
+    // C: X[1]-X[0]
+    const double dX = X[1] - X[0];
+    const double dY = Y[1] - Y[0];
+    const double dZ = Z[1] - Z[0];
+
+    const double d12dx = ONE / F12 / dX;
+    const double d12dy = ONE / F12 / dY;
+    const double d12dz = ONE / F12 / dZ;
+
+    // Fortran 里算了 d2dx/d2dy/d2dz 但本 subroutine 里没用到（保持一致也算出来）
+    const double d2dx  = ONE / TWO / dX;
+    const double d2dy  = ONE / TWO / dY;
+    const double d2dz  = ONE / TWO / dZ;
+    (void)d2dx; (void)d2dy; (void)d2dz;
+
+    // Fortran:
+    // imax = ex(1); jmax = ex(2); kmax = ex(3)
+    const int imaxF = ex1;
+    const int jmaxF = ex2;
+    const int kmaxF = ex3;
+
+    // Fortran:
+    // imin=jmin=kmin=1; 若满足对称条件则设为 -2
+    int iminF = 1, jminF = 1, kminF = 1;
+    if (Symmetry > NO_SYMM && fabs(Z[0]) < dZ) kminF = -2;
+    if (Symmetry > EQ_SYMM && fabs(X[0]) < dX) iminF = -2;
+    if (Symmetry > EQ_SYMM && fabs(Y[0]) < dY) jminF = -2;
+
+    // 分配 fh：大小 (ex1+3)*(ex2+3)*(ex3+3)
+    const size_t nx = (size_t)ex1 + 3;
+    const size_t ny = (size_t)ex2 + 3;
+    const size_t nz = (size_t)ex3 + 3;
+    const size_t fh_size = nx * ny * nz;
+
+    double *fh = (double*)malloc(fh_size * sizeof(double));
+    if (!fh) return; // 内存不足：直接返回（你也可以改成 abort/报错）
+
+    // Fortran: call symmetry_bd(3,ex,f,fh,SoA)
+    symmetry_bd(3, ex, f, fh, SoA);
+
+    /*
+     * Fortran 主循环：
+     * do k=1,ex(3)-1
+     * do j=1,ex(2)-1
+     * do i=1,ex(1)-1
+     *
+     * 转成 C 0-based：
+     * k0 = 0..ex3-2, j0 = 0..ex2-2, i0 = 0..ex1-2
+     *
+     * 并且 Fortran 里的 i/j/k 在 fh 访问时，仍然是 Fortran 索引值：
+     * iF=i0+1, jF=j0+1, kF=k0+1
+     */
+    for (int k0 = 0; k0 <= ex3 - 2; ++k0) {
+        const int kF = k0 + 1;
+        for (int j0 = 0; j0 <= ex2 - 2; ++j0) {
+            const int jF = j0 + 1;
+            for (int i0 = 0; i0 <= ex1 - 2; ++i0) {
+                const int iF = i0 + 1;
+
+                const size_t p = idx_ex(i0, j0, k0, ex);
+
+                // ---------------- x direction ----------------
+                const double sfx = Sfx[p];
+                if (sfx > ZEO) {
+                    // Fortran: if(i+3 <= imax)
+                    // iF+3 <= ex1  <=> i0+4 <= ex1 <=> i0 <= ex1-4
+                    if (i0 <= ex1 - 4) {
+                        f_rhs[p] += sfx * d12dx *
+                            (-F3  * fh[idx_fh_F(iF - 1, jF, kF, ex)]
+                             -F10 * fh[idx_fh_F(iF    , jF, kF, ex)]
+                             +F18 * fh[idx_fh_F(iF + 1, jF, kF, ex)]
+                             -F6  * fh[idx_fh_F(iF + 2, jF, kF, ex)]
+                             +      fh[idx_fh_F(iF + 3, jF, kF, ex)]);
+                    }
+                    // elseif(i+2 <= imax)  <=> i0 <= ex1-3
+                    else if (i0 <= ex1 - 3) {
+                        f_rhs[p] += sfx * d12dx *
+                            ( fh[idx_fh_F(iF - 2, jF, kF, ex)]
+                             -EIT * fh[idx_fh_F(iF - 1, jF, kF, ex)]
+                             +EIT * fh[idx_fh_F(iF + 1, jF, kF, ex)]
+                             -      fh[idx_fh_F(iF + 2, jF, kF, ex)]);
+                    }
+                    // elseif(i+1 <= imax)  <=> i0 <= ex1-2（循环里总成立）
+                    else if (i0 <= ex1 - 2) {
+                        f_rhs[p] -= sfx * d12dx *
+                            (-F3  * fh[idx_fh_F(iF + 1, jF, kF, ex)]
+                             -F10 * fh[idx_fh_F(iF    , jF, kF, ex)]
+                             +F18 * fh[idx_fh_F(iF - 1, jF, kF, ex)]
+                             -F6  * fh[idx_fh_F(iF - 2, jF, kF, ex)]
+                             +      fh[idx_fh_F(iF - 3, jF, kF, ex)]);
+                    }
+                } else if (sfx < ZEO) {
+                    // Fortran: if(i-3 >= imin)
+                    // (iF-3) >= iminF  <=> (i0-2) >= iminF
+                    if ((i0 - 2) >= iminF) {
+                        f_rhs[p] -= sfx * d12dx *
+                            (-F3  * fh[idx_fh_F(iF + 1, jF, kF, ex)]
+                             -F10 * fh[idx_fh_F(iF    , jF, kF, ex)]
+                             +F18 * fh[idx_fh_F(iF - 1, jF, kF, ex)]
+                             -F6  * fh[idx_fh_F(iF - 2, jF, kF, ex)]
+                             +      fh[idx_fh_F(iF - 3, jF, kF, ex)]);
+                    }
+                    // elseif(i-2 >= imin) <=> (i0-1) >= iminF
+                    else if ((i0 - 1) >= iminF) {
+                        f_rhs[p] += sfx * d12dx *
+                            ( fh[idx_fh_F(iF - 2, jF, kF, ex)]
+                             -EIT * fh[idx_fh_F(iF - 1, jF, kF, ex)]
+                             +EIT * fh[idx_fh_F(iF + 1, jF, kF, ex)]
+                             -      fh[idx_fh_F(iF + 2, jF, kF, ex)]);
+                    }
+                    // elseif(i-1 >= imin) <=> i0 >= iminF
+                    else if (i0 >= iminF) {
+                        f_rhs[p] += sfx * d12dx *
+                            (-F3  * fh[idx_fh_F(iF - 1, jF, kF, ex)]
+                             -F10 * fh[idx_fh_F(iF    , jF, kF, ex)]
+                             +F18 * fh[idx_fh_F(iF + 1, jF, kF, ex)]
+                             -F6  * fh[idx_fh_F(iF + 2, jF, kF, ex)]
+                             +      fh[idx_fh_F(iF + 3, jF, kF, ex)]);
+                    }
+                }
+
+                // ---------------- y direction ----------------
+                const double sfy = Sfy[p];
+                if (sfy > ZEO) {
+                    // jF+3 <= ex2 <=> j0+4 <= ex2 <=> j0 <= ex2-4
+                    if (j0 <= ex2 - 4) {
+                        f_rhs[p] += sfy * d12dy *
+                            (-F3  * fh[idx_fh_F(iF, jF - 1, kF, ex)]
+                             -F10 * fh[idx_fh_F(iF, jF    , kF, ex)]
+                             +F18 * fh[idx_fh_F(iF, jF + 1, kF, ex)]
+                             -F6  * fh[idx_fh_F(iF, jF + 2, kF, ex)]
+                             +      fh[idx_fh_F(iF, jF + 3, kF, ex)]);
+                    } else if (j0 <= ex2 - 3) {
+                        f_rhs[p] += sfy * d12dy *
+                            ( fh[idx_fh_F(iF, jF - 2, kF, ex)]
+                             -EIT * fh[idx_fh_F(iF, jF - 1, kF, ex)]
+                             +EIT * fh[idx_fh_F(iF, jF + 1, kF, ex)]
+                             -      fh[idx_fh_F(iF, jF + 2, kF, ex)]);
+                    } else if (j0 <= ex2 - 2) {
+                        f_rhs[p] -= sfy * d12dy *
+                            (-F3  * fh[idx_fh_F(iF, jF + 1, kF, ex)]
+                             -F10 * fh[idx_fh_F(iF, jF    , kF, ex)]
+                             +F18 * fh[idx_fh_F(iF, jF - 1, kF, ex)]
+                             -F6  * fh[idx_fh_F(iF, jF - 2, kF, ex)]
+                             +      fh[idx_fh_F(iF, jF - 3, kF, ex)]);
+                    }
+                } else if (sfy < ZEO) {
+                    if ((j0 - 2) >= jminF) {
+                        f_rhs[p] -= sfy * d12dy *
+                            (-F3  * fh[idx_fh_F(iF, jF + 1, kF, ex)]
+                             -F10 * fh[idx_fh_F(iF, jF    , kF, ex)]
+                             +F18 * fh[idx_fh_F(iF, jF - 1, kF, ex)]
+                             -F6  * fh[idx_fh_F(iF, jF - 2, kF, ex)]
+                             +      fh[idx_fh_F(iF, jF - 3, kF, ex)]);
+                    } else if ((j0 - 1) >= jminF) {
+                        f_rhs[p] += sfy * d12dy *
+                            ( fh[idx_fh_F(iF, jF - 2, kF, ex)]
+                             -EIT * fh[idx_fh_F(iF, jF - 1, kF, ex)]
+                             +EIT * fh[idx_fh_F(iF, jF + 1, kF, ex)]
+                             -      fh[idx_fh_F(iF, jF + 2, kF, ex)]);
+                    } else if (j0 >= jminF) {
+                        f_rhs[p] += sfy * d12dy *
+                            (-F3  * fh[idx_fh_F(iF, jF - 1, kF, ex)]
+                             -F10 * fh[idx_fh_F(iF, jF    , kF, ex)]
+                             +F18 * fh[idx_fh_F(iF, jF + 1, kF, ex)]
+                             -F6  * fh[idx_fh_F(iF, jF + 2, kF, ex)]
+                             +      fh[idx_fh_F(iF, jF + 3, kF, ex)]);
+                    }
+                }
+
+                // ---------------- z direction ----------------
+                const double sfz = Sfz[p];
+                if (sfz > ZEO) {
+                    if (k0 <= ex3 - 4) {
+                        f_rhs[p] += sfz * d12dz *
+                            (-F3  * fh[idx_fh_F(iF, jF, kF - 1, ex)]
+                             -F10 * fh[idx_fh_F(iF, jF, kF    , ex)]
+                             +F18 * fh[idx_fh_F(iF, jF, kF + 1, ex)]
+                             -F6  * fh[idx_fh_F(iF, jF, kF + 2, ex)]
+                             +      fh[idx_fh_F(iF, jF, kF + 3, ex)]);
+                    } else if (k0 <= ex3 - 3) {
+                        f_rhs[p] += sfz * d12dz *
+                            ( fh[idx_fh_F(iF, jF, kF - 2, ex)]
+                             -EIT * fh[idx_fh_F(iF, jF, kF - 1, ex)]
+                             +EIT * fh[idx_fh_F(iF, jF, kF + 1, ex)]
+                             -      fh[idx_fh_F(iF, jF, kF + 2, ex)]);
+                    } else if (k0 <= ex3 - 2) {
+                        f_rhs[p] -= sfz * d12dz *
+                            (-F3  * fh[idx_fh_F(iF, jF, kF + 1, ex)]
+                             -F10 * fh[idx_fh_F(iF, jF, kF    , ex)]
+                             +F18 * fh[idx_fh_F(iF, jF, kF - 1, ex)]
+                             -F6  * fh[idx_fh_F(iF, jF, kF - 2, ex)]
+                             +      fh[idx_fh_F(iF, jF, kF - 3, ex)]);
+                    }
+                } else if (sfz < ZEO) {
+                    if ((k0 - 2) >= kminF) {
+                        f_rhs[p] -= sfz * d12dz *
+                            (-F3  * fh[idx_fh_F(iF, jF, kF + 1, ex)]
+                             -F10 * fh[idx_fh_F(iF, jF, kF    , ex)]
+                             +F18 * fh[idx_fh_F(iF, jF, kF - 1, ex)]
+                             -F6  * fh[idx_fh_F(iF, jF, kF - 2, ex)]
+                             +      fh[idx_fh_F(iF, jF, kF - 3, ex)]);
+                    } else if ((k0 - 1) >= kminF) {
+                        f_rhs[p] += sfz * d12dz *
+                            ( fh[idx_fh_F(iF, jF, kF - 2, ex)]
+                             -EIT * fh[idx_fh_F(iF, jF, kF - 1, ex)]
+                             +EIT * fh[idx_fh_F(iF, jF, kF + 1, ex)]
+                             -      fh[idx_fh_F(iF, jF, kF + 2, ex)]);
+                    } else if (k0 >= kminF) {
+                        f_rhs[p] += sfz * d12dz *
+                            (-F3  * fh[idx_fh_F(iF, jF, kF - 1, ex)]
+                             -F10 * fh[idx_fh_F(iF, jF, kF    , ex)]
+                             +F18 * fh[idx_fh_F(iF, jF, kF + 1, ex)]
+                             -F6  * fh[idx_fh_F(iF, jF, kF + 2, ex)]
+                             +      fh[idx_fh_F(iF, jF, kF + 3, ex)]);
+                    }
+                }
+            }
+        }
+    }
+    free(fh);
+}
+
+
+
+
+

AMSS_NCKU_source/lopsided_kodis_c.C

@@ -0,0 +1,248 @@
+#include "tool.h"
+
+/*
+ * Combined advection (lopsided) + KO dissipation (kodis).
+ * Uses one shared symmetry_bd buffer per call.
+ */
+void lopsided_kodis(const int ex[3],
+                    const double *X, const double *Y, const double *Z,
+                    const double *f, double *f_rhs,
+                    const double *Sfx, const double *Sfy, const double *Sfz,
+                    int Symmetry, const double SoA[3], double eps)
+{
+    const double ZEO = 0.0, ONE = 1.0, F3 = 3.0;
+    const double F6 = 6.0, F18 = 18.0;
+    const double F12 = 12.0, F10 = 10.0, EIT = 8.0;
+    const double SIX = 6.0, FIT = 15.0, TWT = 20.0;
+    const double cof = 64.0; // 2^6
+
+    const int NO_SYMM = 0, EQ_SYMM = 1;
+
+    const int ex1 = ex[0], ex2 = ex[1], ex3 = ex[2];
+
+    const double dX = X[1] - X[0];
+    const double dY = Y[1] - Y[0];
+    const double dZ = Z[1] - Z[0];
+
+    const double d12dx = ONE / F12 / dX;
+    const double d12dy = ONE / F12 / dY;
+    const double d12dz = ONE / F12 / dZ;
+
+    const int imaxF = ex1;
+    const int jmaxF = ex2;
+    const int kmaxF = ex3;
+
+    int iminF = 1, jminF = 1, kminF = 1;
+    if (Symmetry > NO_SYMM && fabs(Z[0]) < dZ) kminF = -2;
+    if (Symmetry > EQ_SYMM && fabs(X[0]) < dX) iminF = -2;
+    if (Symmetry > EQ_SYMM && fabs(Y[0]) < dY) jminF = -2;
+
+    // fh for Fortran-style domain (-2:ex1,-2:ex2,-2:ex3)
+    const size_t nx = (size_t)ex1 + 3;
+    const size_t ny = (size_t)ex2 + 3;
+    const size_t nz = (size_t)ex3 + 3;
+    const size_t fh_size = nx * ny * nz;
+
+    double *fh = (double*)malloc(fh_size * sizeof(double));
+    if (!fh) return;
+
+    symmetry_bd(3, ex, f, fh, SoA);
+
+    // Advection (same stencil logic as lopsided_c.C)
+    for (int k0 = 0; k0 <= ex3 - 2; ++k0) {
+        const int kF = k0 + 1;
+        for (int j0 = 0; j0 <= ex2 - 2; ++j0) {
+            const int jF = j0 + 1;
+            for (int i0 = 0; i0 <= ex1 - 2; ++i0) {
+                const int iF = i0 + 1;
+                const size_t p = idx_ex(i0, j0, k0, ex);
+
+                const double sfx = Sfx[p];
+                if (sfx > ZEO) {
+                    if (i0 <= ex1 - 4) {
+                        f_rhs[p] += sfx * d12dx *
+                            (-F3  * fh[idx_fh_F(iF - 1, jF, kF, ex)]
+                             -F10 * fh[idx_fh_F(iF    , jF, kF, ex)]
+                             +F18 * fh[idx_fh_F(iF + 1, jF, kF, ex)]
+                             -F6  * fh[idx_fh_F(iF + 2, jF, kF, ex)]
+                             +      fh[idx_fh_F(iF + 3, jF, kF, ex)]);
+                    } else if (i0 <= ex1 - 3) {
+                        f_rhs[p] += sfx * d12dx *
+                            ( fh[idx_fh_F(iF - 2, jF, kF, ex)]
+                             -EIT * fh[idx_fh_F(iF - 1, jF, kF, ex)]
+                             +EIT * fh[idx_fh_F(iF + 1, jF, kF, ex)]
+                             -      fh[idx_fh_F(iF + 2, jF, kF, ex)]);
+                    } else if (i0 <= ex1 - 2) {
+                        f_rhs[p] -= sfx * d12dx *
+                            (-F3  * fh[idx_fh_F(iF + 1, jF, kF, ex)]
+                             -F10 * fh[idx_fh_F(iF    , jF, kF, ex)]
+                             +F18 * fh[idx_fh_F(iF - 1, jF, kF, ex)]
+                             -F6  * fh[idx_fh_F(iF - 2, jF, kF, ex)]
+                             +      fh[idx_fh_F(iF - 3, jF, kF, ex)]);
+                    }
+                } else if (sfx < ZEO) {
+                    if ((i0 - 2) >= iminF) {
+                        f_rhs[p] -= sfx * d12dx *
+                            (-F3  * fh[idx_fh_F(iF + 1, jF, kF, ex)]
+                             -F10 * fh[idx_fh_F(iF    , jF, kF, ex)]
+                             +F18 * fh[idx_fh_F(iF - 1, jF, kF, ex)]
+                             -F6  * fh[idx_fh_F(iF - 2, jF, kF, ex)]
+                             +      fh[idx_fh_F(iF - 3, jF, kF, ex)]);
+                    } else if ((i0 - 1) >= iminF) {
+                        f_rhs[p] += sfx * d12dx *
+                            ( fh[idx_fh_F(iF - 2, jF, kF, ex)]
+                             -EIT * fh[idx_fh_F(iF - 1, jF, kF, ex)]
+                             +EIT * fh[idx_fh_F(iF + 1, jF, kF, ex)]
+                             -      fh[idx_fh_F(iF + 2, jF, kF, ex)]);
+                    } else if (i0 >= iminF) {
+                        f_rhs[p] += sfx * d12dx *
+                            (-F3  * fh[idx_fh_F(iF - 1, jF, kF, ex)]
+                             -F10 * fh[idx_fh_F(iF    , jF, kF, ex)]
+                             +F18 * fh[idx_fh_F(iF + 1, jF, kF, ex)]
+                             -F6  * fh[idx_fh_F(iF + 2, jF, kF, ex)]
+                             +      fh[idx_fh_F(iF + 3, jF, kF, ex)]);
+                    }
+                }
+
+                const double sfy = Sfy[p];
+                if (sfy > ZEO) {
+                    if (j0 <= ex2 - 4) {
+                        f_rhs[p] += sfy * d12dy *
+                            (-F3  * fh[idx_fh_F(iF, jF - 1, kF, ex)]
+                             -F10 * fh[idx_fh_F(iF, jF    , kF, ex)]
+                             +F18 * fh[idx_fh_F(iF, jF + 1, kF, ex)]
+                             -F6  * fh[idx_fh_F(iF, jF + 2, kF, ex)]
+                             +      fh[idx_fh_F(iF, jF + 3, kF, ex)]);
+                    } else if (j0 <= ex2 - 3) {
+                        f_rhs[p] += sfy * d12dy *
+                            ( fh[idx_fh_F(iF, jF - 2, kF, ex)]
+                             -EIT * fh[idx_fh_F(iF, jF - 1, kF, ex)]
+                             +EIT * fh[idx_fh_F(iF, jF + 1, kF, ex)]
+                             -      fh[idx_fh_F(iF, jF + 2, kF, ex)]);
+                    } else if (j0 <= ex2 - 2) {
+                        f_rhs[p] -= sfy * d12dy *
+                            (-F3  * fh[idx_fh_F(iF, jF + 1, kF, ex)]
+                             -F10 * fh[idx_fh_F(iF, jF    , kF, ex)]
+                             +F18 * fh[idx_fh_F(iF, jF - 1, kF, ex)]
+                             -F6  * fh[idx_fh_F(iF, jF - 2, kF, ex)]
+                             +      fh[idx_fh_F(iF, jF - 3, kF, ex)]);
+                    }
+                } else if (sfy < ZEO) {
+                    if ((j0 - 2) >= jminF) {
+                        f_rhs[p] -= sfy * d12dy *
+                            (-F3  * fh[idx_fh_F(iF, jF + 1, kF, ex)]
+                             -F10 * fh[idx_fh_F(iF, jF    , kF, ex)]
+                             +F18 * fh[idx_fh_F(iF, jF - 1, kF, ex)]
+                             -F6  * fh[idx_fh_F(iF, jF - 2, kF, ex)]
+                             +      fh[idx_fh_F(iF, jF - 3, kF, ex)]);
+                    } else if ((j0 - 1) >= jminF) {
+                        f_rhs[p] += sfy * d12dy *
+                            ( fh[idx_fh_F(iF, jF - 2, kF, ex)]
+                             -EIT * fh[idx_fh_F(iF, jF - 1, kF, ex)]
+                             +EIT * fh[idx_fh_F(iF, jF + 1, kF, ex)]
+                             -      fh[idx_fh_F(iF, jF + 2, kF, ex)]);
+                    } else if (j0 >= jminF) {
+                        f_rhs[p] += sfy * d12dy *
+                            (-F3  * fh[idx_fh_F(iF, jF - 1, kF, ex)]
+                             -F10 * fh[idx_fh_F(iF, jF    , kF, ex)]
+                             +F18 * fh[idx_fh_F(iF, jF + 1, kF, ex)]
+                             -F6  * fh[idx_fh_F(iF, jF + 2, kF, ex)]
+                             +      fh[idx_fh_F(iF, jF + 3, kF, ex)]);
+                    }
+                }
+
+                const double sfz = Sfz[p];
+                if (sfz > ZEO) {
+                    if (k0 <= ex3 - 4) {
+                        f_rhs[p] += sfz * d12dz *
+                            (-F3  * fh[idx_fh_F(iF, jF, kF - 1, ex)]
+                             -F10 * fh[idx_fh_F(iF, jF, kF    , ex)]
+                             +F18 * fh[idx_fh_F(iF, jF, kF + 1, ex)]
+                             -F6  * fh[idx_fh_F(iF, jF, kF + 2, ex)]
+                             +      fh[idx_fh_F(iF, jF, kF + 3, ex)]);
+                    } else if (k0 <= ex3 - 3) {
+                        f_rhs[p] += sfz * d12dz *
+                            ( fh[idx_fh_F(iF, jF, kF - 2, ex)]
+                             -EIT * fh[idx_fh_F(iF, jF, kF - 1, ex)]
+                             +EIT * fh[idx_fh_F(iF, jF, kF + 1, ex)]
+                             -      fh[idx_fh_F(iF, jF, kF + 2, ex)]);
+                    } else if (k0 <= ex3 - 2) {
+                        f_rhs[p] -= sfz * d12dz *
+                            (-F3  * fh[idx_fh_F(iF, jF, kF + 1, ex)]
+                             -F10 * fh[idx_fh_F(iF, jF, kF    , ex)]
+                             +F18 * fh[idx_fh_F(iF, jF, kF - 1, ex)]
+                             -F6  * fh[idx_fh_F(iF, jF, kF - 2, ex)]
+                             +      fh[idx_fh_F(iF, jF, kF - 3, ex)]);
+                    }
+                } else if (sfz < ZEO) {
+                    if ((k0 - 2) >= kminF) {
+                        f_rhs[p] -= sfz * d12dz *
+                            (-F3  * fh[idx_fh_F(iF, jF, kF + 1, ex)]
+                             -F10 * fh[idx_fh_F(iF, jF, kF    , ex)]
+                             +F18 * fh[idx_fh_F(iF, jF, kF - 1, ex)]
+                             -F6  * fh[idx_fh_F(iF, jF, kF - 2, ex)]
+                             +      fh[idx_fh_F(iF, jF, kF - 3, ex)]);
+                    } else if ((k0 - 1) >= kminF) {
+                        f_rhs[p] += sfz * d12dz *
+                            ( fh[idx_fh_F(iF, jF, kF - 2, ex)]
+                             -EIT * fh[idx_fh_F(iF, jF, kF - 1, ex)]
+                             +EIT * fh[idx_fh_F(iF, jF, kF + 1, ex)]
+                             -      fh[idx_fh_F(iF, jF, kF + 2, ex)]);
+                    } else if (k0 >= kminF) {
+                        f_rhs[p] += sfz * d12dz *
+                            (-F3  * fh[idx_fh_F(iF, jF, kF - 1, ex)]
+                             -F10 * fh[idx_fh_F(iF, jF, kF    , ex)]
+                             +F18 * fh[idx_fh_F(iF, jF, kF + 1, ex)]
+                             -F6  * fh[idx_fh_F(iF, jF, kF + 2, ex)]
+                             +      fh[idx_fh_F(iF, jF, kF + 3, ex)]);
+                    }
+                }
+            }
+        }
+    }
+
+    // KO dissipation (same domain restriction as kodiss_c.C)
+    if (eps > ZEO) {
+        const int i0_lo = (iminF + 2 > 0) ? iminF + 2 : 0;
+        const int j0_lo = (jminF + 2 > 0) ? jminF + 2 : 0;
+        const int k0_lo = (kminF + 2 > 0) ? kminF + 2 : 0;
+        const int i0_hi = imaxF - 4; // inclusive
+        const int j0_hi = jmaxF - 4;
+        const int k0_hi = kmaxF - 4;
+
+        if (!(i0_lo > i0_hi || j0_lo > j0_hi || k0_lo > k0_hi)) {
+            for (int k0 = k0_lo; k0 <= k0_hi; ++k0) {
+                const int kF = k0 + 1;
+                for (int j0 = j0_lo; j0 <= j0_hi; ++j0) {
+                    const int jF = j0 + 1;
+                    for (int i0 = i0_lo; i0 <= i0_hi; ++i0) {
+                        const int iF = i0 + 1;
+                        const size_t p = idx_ex(i0, j0, k0, ex);
+
+                        const double Dx_term =
+                            ((fh[idx_fh_F(iF - 3, jF, kF, ex)] + fh[idx_fh_F(iF + 3, jF, kF, ex)]) -
+                             SIX * (fh[idx_fh_F(iF - 2, jF, kF, ex)] + fh[idx_fh_F(iF + 2, jF, kF, ex)]) +
+                             FIT * (fh[idx_fh_F(iF - 1, jF, kF, ex)] + fh[idx_fh_F(iF + 1, jF, kF, ex)]) -
+                             TWT *  fh[idx_fh_F(iF,     jF, kF, ex)]) / dX;
+
+                        const double Dy_term =
+                            ((fh[idx_fh_F(iF, jF - 3, kF, ex)] + fh[idx_fh_F(iF, jF + 3, kF, ex)]) -
+                             SIX * (fh[idx_fh_F(iF, jF - 2, kF, ex)] + fh[idx_fh_F(iF, jF + 2, kF, ex)]) +
+                             FIT * (fh[idx_fh_F(iF, jF - 1, kF, ex)] + fh[idx_fh_F(iF, jF + 1, kF, ex)]) -
+                             TWT *  fh[idx_fh_F(iF, jF,     kF, ex)]) / dY;
+
+                        const double Dz_term =
+                            ((fh[idx_fh_F(iF, jF, kF - 3, ex)] + fh[idx_fh_F(iF, jF, kF + 3, ex)]) -
+                             SIX * (fh[idx_fh_F(iF, jF, kF - 2, ex)] + fh[idx_fh_F(iF, jF, kF + 2, ex)]) +
+                             FIT * (fh[idx_fh_F(iF, jF, kF - 1, ex)] + fh[idx_fh_F(iF, jF, kF + 1, ex)]) -
+                             TWT *  fh[idx_fh_F(iF, jF, kF,     ex)]) / dZ;
+
+                        f_rhs[p] += (eps / cof) * (Dx_term + Dy_term + Dz_term);
+                    }
+                }
+            }
+        }
+    }
+
+    free(fh);
+}

AMSS_NCKU_source/lopsidediff.f90

@@ -487,6 +487,201 @@ subroutine lopsided(ex,X,Y,Z,f,f_rhs,Sfx,Sfy,Sfz,Symmetry,SoA)
 
   end subroutine lopsided
 
+!-----------------------------------------------------------------------------
+! Combined advection (lopsided) + Kreiss-Oliger dissipation (kodis)
+! Shares the symmetry_bd buffer fh, eliminating one full-grid copy per call.
+! Mathematically identical to calling lopsided then kodis separately.
+!-----------------------------------------------------------------------------
+subroutine lopsided_kodis(ex,X,Y,Z,f,f_rhs,Sfx,Sfy,Sfz,Symmetry,SoA,eps)
+  implicit none
+
+!~~~~~~> Input parameters:
+
+  integer, intent(in)  :: ex(1:3),Symmetry
+  real*8,  intent(in)  :: X(1:ex(1)),Y(1:ex(2)),Z(1:ex(3))
+  real*8,dimension(ex(1),ex(2),ex(3)),intent(in)   :: f,Sfx,Sfy,Sfz
+
+  real*8,dimension(ex(1),ex(2),ex(3)),intent(inout):: f_rhs
+  real*8,dimension(3),intent(in) ::SoA
+  real*8,intent(in) :: eps
+
+!~~~~~~> local variables:
+! note index -2,-1,0, so we have 3 extra points
+  real*8,dimension(-2:ex(1),-2:ex(2),-2:ex(3))   :: fh
+  integer :: imin,jmin,kmin,imax,jmax,kmax,i,j,k
+  real*8 :: dX,dY,dZ
+  real*8 :: d12dx,d12dy,d12dz,d2dx,d2dy,d2dz
+  real*8,  parameter :: ZEO=0.d0,ONE=1.d0, F3=3.d0
+  real*8,  parameter :: TWO=2.d0,F6=6.0d0,F18=1.8d1
+  real*8,  parameter :: F12=1.2d1, F10=1.d1,EIT=8.d0
+  integer, parameter :: NO_SYMM = 0, EQ_SYMM = 1, OCTANT = 2
+! kodis parameters
+  real*8, parameter :: SIX=6.d0,FIT=1.5d1,TWT=2.d1
+  real*8, parameter :: cof=6.4d1   ! 2^6
+
+  dX = X(2)-X(1)
+  dY = Y(2)-Y(1)
+  dZ = Z(2)-Z(1)
+
+  d12dx = ONE/F12/dX
+  d12dy = ONE/F12/dY
+  d12dz = ONE/F12/dZ
+
+  d2dx = ONE/TWO/dX
+  d2dy = ONE/TWO/dY
+  d2dz = ONE/TWO/dZ
+
+  imax = ex(1)
+  jmax = ex(2)
+  kmax = ex(3)
+
+  imin = 1
+  jmin = 1
+  kmin = 1
+  if(Symmetry > NO_SYMM .and. dabs(Z(1)) < dZ) kmin = -2
+  if(Symmetry > EQ_SYMM .and. dabs(X(1)) < dX) imin = -2
+  if(Symmetry > EQ_SYMM .and. dabs(Y(1)) < dY) jmin = -2
+
+! Single symmetry_bd call shared by both advection and dissipation
+  call symmetry_bd(3,ex,f,fh,SoA)
+
+! ---- Advection (lopsided) loop ----
+! upper bound set ex-1 only for efficiency, 
+! the loop body will set ex 0 also
+  do k=1,ex(3)-1
+  do j=1,ex(2)-1
+  do i=1,ex(1)-1
+! x direction   
+    if(Sfx(i,j,k) > ZEO)then
+      if(i+3 <= imax)then
+     f_rhs(i,j,k)=f_rhs(i,j,k)+                                                   &
+                  Sfx(i,j,k)*d12dx*(-F3*fh(i-1,j,k)-F10*fh(i,j,k)+F18*fh(i+1,j,k) &
+                                    -F6*fh(i+2,j,k)+    fh(i+3,j,k))
+     elseif(i+2 <= imax)then
+     f_rhs(i,j,k)=f_rhs(i,j,k)+                                                           &
+                  Sfx(i,j,k)*d12dx*(fh(i-2,j,k)-EIT*fh(i-1,j,k)+EIT*fh(i+1,j,k)-fh(i+2,j,k))
+
+     elseif(i+1 <= imax)then
+     f_rhs(i,j,k)=f_rhs(i,j,k)-                                                   &
+                  Sfx(i,j,k)*d12dx*(-F3*fh(i+1,j,k)-F10*fh(i,j,k)+F18*fh(i-1,j,k) &
+                                    -F6*fh(i-2,j,k)+    fh(i-3,j,k))
+     endif
+   elseif(Sfx(i,j,k) < ZEO)then
+      if(i-3 >= imin)then
+     f_rhs(i,j,k)=f_rhs(i,j,k)-                                                   &
+                  Sfx(i,j,k)*d12dx*(-F3*fh(i+1,j,k)-F10*fh(i,j,k)+F18*fh(i-1,j,k) &
+                                    -F6*fh(i-2,j,k)+    fh(i-3,j,k))
+     elseif(i-2 >= imin)then
+     f_rhs(i,j,k)=f_rhs(i,j,k)+                                                           &
+                  Sfx(i,j,k)*d12dx*(fh(i-2,j,k)-EIT*fh(i-1,j,k)+EIT*fh(i+1,j,k)-fh(i+2,j,k))
+
+     elseif(i-1 >= imin)then
+     f_rhs(i,j,k)=f_rhs(i,j,k)+                                                   &
+                  Sfx(i,j,k)*d12dx*(-F3*fh(i-1,j,k)-F10*fh(i,j,k)+F18*fh(i+1,j,k) &
+                                    -F6*fh(i+2,j,k)+    fh(i+3,j,k))
+     endif
+   endif
+
+! y direction   
+    if(Sfy(i,j,k) > ZEO)then
+      if(j+3 <= jmax)then
+     f_rhs(i,j,k)=f_rhs(i,j,k)+                                                   &
+                  Sfy(i,j,k)*d12dy*(-F3*fh(i,j-1,k)-F10*fh(i,j,k)+F18*fh(i,j+1,k) &
+                                    -F6*fh(i,j+2,k)+    fh(i,j+3,k))
+     elseif(j+2 <= jmax)then
+     f_rhs(i,j,k)=f_rhs(i,j,k)+                                                           &
+                  Sfy(i,j,k)*d12dy*(fh(i,j-2,k)-EIT*fh(i,j-1,k)+EIT*fh(i,j+1,k)-fh(i,j+2,k))
+
+     elseif(j+1 <= jmax)then
+     f_rhs(i,j,k)=f_rhs(i,j,k)-                                                   &
+                  Sfy(i,j,k)*d12dy*(-F3*fh(i,j+1,k)-F10*fh(i,j,k)+F18*fh(i,j-1,k) &
+                                    -F6*fh(i,j-2,k)+    fh(i,j-3,k))
+     endif
+   elseif(Sfy(i,j,k) < ZEO)then
+      if(j-3 >= jmin)then
+     f_rhs(i,j,k)=f_rhs(i,j,k)-                                                   &
+                  Sfy(i,j,k)*d12dy*(-F3*fh(i,j+1,k)-F10*fh(i,j,k)+F18*fh(i,j-1,k) &
+                                    -F6*fh(i,j-2,k)+    fh(i,j-3,k))
+     elseif(j-2 >= jmin)then
+     f_rhs(i,j,k)=f_rhs(i,j,k)+                                                           &
+                  Sfy(i,j,k)*d12dy*(fh(i,j-2,k)-EIT*fh(i,j-1,k)+EIT*fh(i,j+1,k)-fh(i,j+2,k))
+
+     elseif(j-1 >= jmin)then
+     f_rhs(i,j,k)=f_rhs(i,j,k)+                                                   &
+                  Sfy(i,j,k)*d12dy*(-F3*fh(i,j-1,k)-F10*fh(i,j,k)+F18*fh(i,j+1,k) &
+                                    -F6*fh(i,j+2,k)+    fh(i,j+3,k))
+     endif
+   endif
+
+! z direction   
+    if(Sfz(i,j,k) > ZEO)then
+      if(k+3 <= kmax)then
+     f_rhs(i,j,k)=f_rhs(i,j,k)+                                                   &
+                  Sfz(i,j,k)*d12dz*(-F3*fh(i,j,k-1)-F10*fh(i,j,k)+F18*fh(i,j,k+1) &
+                                    -F6*fh(i,j,k+2)+    fh(i,j,k+3))
+     elseif(k+2 <= kmax)then
+     f_rhs(i,j,k)=f_rhs(i,j,k)+                                                           &
+                  Sfz(i,j,k)*d12dz*(fh(i,j,k-2)-EIT*fh(i,j,k-1)+EIT*fh(i,j,k+1)-fh(i,j,k+2))
+
+     elseif(k+1 <= kmax)then
+     f_rhs(i,j,k)=f_rhs(i,j,k)-                                                   &
+                  Sfz(i,j,k)*d12dz*(-F3*fh(i,j,k+1)-F10*fh(i,j,k)+F18*fh(i,j,k-1) &
+                                    -F6*fh(i,j,k-2)+    fh(i,j,k-3))
+     endif
+   elseif(Sfz(i,j,k) < ZEO)then
+      if(k-3 >= kmin)then
+     f_rhs(i,j,k)=f_rhs(i,j,k)-                                                   &
+                  Sfz(i,j,k)*d12dz*(-F3*fh(i,j,k+1)-F10*fh(i,j,k)+F18*fh(i,j,k-1) &
+                                    -F6*fh(i,j,k-2)+    fh(i,j,k-3))
+     elseif(k-2 >= kmin)then
+     f_rhs(i,j,k)=f_rhs(i,j,k)+                                                           &
+                  Sfz(i,j,k)*d12dz*(fh(i,j,k-2)-EIT*fh(i,j,k-1)+EIT*fh(i,j,k+1)-fh(i,j,k+2))
+
+     elseif(k-1 >= kmin)then
+     f_rhs(i,j,k)=f_rhs(i,j,k)+                                                   &
+                  Sfz(i,j,k)*d12dz*(-F3*fh(i,j,k-1)-F10*fh(i,j,k)+F18*fh(i,j,k+1) &
+                                    -F6*fh(i,j,k+2)+    fh(i,j,k+3))
+     endif
+   endif
+  enddo
+  enddo
+  enddo
+
+! ---- Dissipation (kodis) loop ----
+  if(eps > ZEO) then
+  do k=1,ex(3)
+  do j=1,ex(2)
+  do i=1,ex(1)
+
+  if(i-3 >= imin .and. i+3 <= imax .and. &
+     j-3 >= jmin .and. j+3 <= jmax .and. &
+     k-3 >= kmin .and. k+3 <= kmax) then
+   f_rhs(i,j,k)       = f_rhs(i,j,k) + eps/cof *( (     &
+                              (fh(i-3,j,k)+fh(i+3,j,k)) - &
+                          SIX*(fh(i-2,j,k)+fh(i+2,j,k)) + &
+                          FIT*(fh(i-1,j,k)+fh(i+1,j,k)) - &
+                          TWT* fh(i,j,k)            )/dX + &
+                                                  (     &
+                              (fh(i,j-3,k)+fh(i,j+3,k)) - &
+                          SIX*(fh(i,j-2,k)+fh(i,j+2,k)) + &
+                          FIT*(fh(i,j-1,k)+fh(i,j+1,k)) - &
+                          TWT* fh(i,j,k)            )/dY + &
+                                                  (     &
+                              (fh(i,j,k-3)+fh(i,j,k+3)) - &
+                          SIX*(fh(i,j,k-2)+fh(i,j,k+2)) + &
+                          FIT*(fh(i,j,k-1)+fh(i,j,k+1)) - &
+                          TWT* fh(i,j,k)            )/dZ )
+  endif
+
+  enddo
+  enddo
+  enddo
+  endif
+
+  return
+
+  end subroutine lopsided_kodis
+
 #elif (ghost_width == 4)
 ! sixth order code
 ! Compute advection terms in right hand sides of field equations

AMSS_NCKU_source/macrodef.fh

@@ -1,83 +1,77 @@
-
-
-#if 0
-note here
-v:r; u: phi; w: theta
-tetradtype 0
-v^a = (x,y,z)
-orthonormal order: v,u,w
-m = (phi - i theta)/sqrt(2) following Frans, Eq.(8) of  PRD 75, 124018(2007)
-tetradtype 1
-orthonormal order: w,u,v
-m = (theta + i phi)/sqrt(2) following Sperhake, Eq.(3.2) of  PRD 85, 124062(2012)
-tetradtype 2
-v_a = (x,y,z)
-orthonormal order: v,u,w
-m = (phi - i theta)/sqrt(2) following Frans, Eq.(8) of  PRD 75, 124018(2007)
-#endif
-#define tetradtype 2
-
-#if 0
-note here
-Cell center or Vertex center
-#endif
-#define Cell
-
-#if 0
-note here
-2nd order: 2
-4th order: 3
-6th order: 4
-8th order: 5
-#endif
-#define ghost_width 3
-
-#if 0
-note here
-use shell or not
-#endif
-#define WithShell
-
-#if 0
-note here
-use constraint preserving boundary condition or not
-only affect Z4c
-#endif
-#define CPBC
-
-#if 0
-note here
-Gauge condition type
-0: B^i gauge
-1: David's puncture gauge
-2: MB B^i gauge
-3: RIT B^i gauge
-4: MB beta gauge (beta gauge not means Eq.(3) of PRD 84, 124006)
-5: RIT beta gauge (beta gauge not means Eq.(3) of PRD 84, 124006)
-6: MGB1 B^i gauge
-7: MGB2 B^i gauge
-#endif
-#define GAUGE 2
-
-#if 0
-buffer points for CPBC boundary
-#endif
-#define CPBC_ghost_width  (ghost_width)
-
-#if 0
-using BSSN variable for constraint violation and psi4 calculation: 0
-using ADM variable for constraint violation and psi4 calculation: 1
-#endif
-#define ABV 0
-
-#if 0
-Type of Potential and Scalar Distribution in F(R) Scalar-Tensor Theory
-1: Case C of 1112.3928, V=0
-2: shell with a2^2*phi0/(1+a2^2), f(R) = R+a2*R^2 induced V
-3: ground state of Schrodinger-Newton system, f(R) = R+a2*R^2 induced V
-4: a2 = oo and phi(r) = phi0 * 0.5 * ( tanh((r+r0)/sigma) - tanh((r-r0)/sigma) )
-5: shell with phi(r) = phi0*Exp(-(r-r0)**2/sigma), V = 0
-#endif
-#define EScalar_CC 2
-
-
+
+#define tetradtype 2
+
+#define Cell
+
+#define ghost_width 3
+
+
+
+#define GAUGE 0
+
+#define CPBC_ghost_width  (ghost_width)
+
+#define ABV 0
+
+#define EScalar_CC  2
+
+#if 0
+
+define tetradtype
+    v:r; u: phi; w: theta
+    tetradtype 0
+    v^a = (x,y,z)
+    orthonormal order: v,u,w
+    m = (phi - i theta)/sqrt(2) following Frans, Eq.(8) of  PRD 75, 124018(2007)
+    tetradtype 1
+    orthonormal order: w,u,v
+    m = (theta + i phi)/sqrt(2) following Sperhake, Eq.(3.2) of  PRD 85, 124062(2012)
+    tetradtype 2
+    v_a = (x,y,z)
+    orthonormal order: v,u,w
+    m = (phi - i theta)/sqrt(2) following Frans, Eq.(8) of  PRD 75, 124018(2007)
+
+define Cell or Vertex
+    Cell center or Vertex center
+
+define ghost_width
+    2nd order: 2
+    4th order: 3
+    6th order: 4
+    8th order: 5
+
+define WithShell
+    use shell or not
+
+define CPBC
+    use constraint preserving boundary condition or not
+    only affect Z4c
+    CPBC only supports WithShell
+
+define GAUGE
+    0: B^i gauge
+    1: David puncture gauge
+    2: MB B^i gauge
+    3: RIT B^i gauge
+    4: MB beta gauge (beta gauge not means Eq.(3) of PRD 84, 124006)
+    5: RIT beta gauge (beta gauge not means Eq.(3) of PRD 84, 124006)
+    6: MGB1 B^i gauge
+    7: MGB2 B^i gauge
+
+define CPBC_ghost_width  (ghost_width)
+    buffer points for CPBC boundary
+
+define ABV
+    0: using BSSN variable for constraint violation and psi4 calculation
+    1: using ADM variable for constraint violation and psi4 calculation
+
+define EScalar_CC
+Type of Potential and Scalar Distribution in F(R) Scalar-Tensor Theory
+    1: Case C of 1112.3928, V=0
+    2: shell with   phi(r) = phi0 * a2^2/(1+a2^2), f(R) = R+a2*R^2 induced V
+    3: ground state of Schrodinger-Newton system,  f(R) = R+a2*R^2 induced V
+    4: a2 = +oo and phi(r) = phi0 * 0.5 * ( tanh((r+r0)/sigma) - tanh((r-r0)/sigma) )
+    5: shell with   phi(r) = phi0 * Exp(-(r-r0)**2/sigma), V = 0
+
+#endif
+

AMSS_NCKU_source/macrodef.h

@@ -1,112 +1,170 @@
-
-#ifndef MICRODEF_H
-#define MICRODEF_H
-
-#include "microdef.fh"
-
-// application parameters
-
-/// ****
-// sommerfeld boundary type
-// 0: bam, 1: shibata
-#define SommerType 0
-
-/// ****
-// for Using Gauss-Legendre quadrature in theta direction
-#define GaussInt
-
-/// ****
-// 0: BSSN vacuum
-// 1: coupled to scalar field
-// 2: Z4c vacuum
-// 3: coupled to Maxwell field
-//
-#define ABEtype 2
-
-/// ****
-// using Apparent Horizon Finder
-//#define With_AHF
-
-/// ****
-// Psi4 calculation method
-// 0: EB method
-// 1: 4-D method
-//
-#define Psi4type 0
-
-/// ****
-// for Using point psi4 or not
-//#define Point_Psi4
-
-/// ****
-// RestrictProlong in Step (0) or after Step (1)
-#define RPS 1
-
-/// ****
-// Enforce algebra constraint
-// for every RK4 sub step: 0
-// only when iter_count == 3: 1
-// after routine Step: 2
-#define AGM 0
-
-/// ****
-// Restrict Prolong using BAM style 1 or old style 0
-#define RPB 0
-
-/// ****
-// 1: move Analysis out ot 4 sub steps and treat PBH with Euler method
-#define MAPBH 1
-
-/// ****
-// parallel structure, 0: level by level, 1: considering all levels, 2: as 1 but reverse the CPU order, 3: Frank's scheme
-#define PSTR 0
-
-/// ****
-// regrid for every level or for all levels at a time
-// 0: for every level; 1: for all
-#define REGLEV 0
-
-/// ****
-// use gpu or not
-//#define USE_GPU
-
-/// ****
-// use checkpoint for every process
-//#define CHECKDETAIL
-
-/// ****
-// use FakeCheckPrepare to write CheckPoint
-//#define FAKECHECK
-////================================================================
-//  some basic parameters for numerical calculation
-#define dim 3
-
-//#define Cell or Vertex in "microdef.fh"
-
-// ******
-// buffer point number for mesh refinement interface
-#define buffer_width 6
-
-// ******
-// buffer point number shell-box interface, on shell
-#define SC_width buffer_width
-// buffer point number shell-box interface, on box
-#define CS_width (2*buffer_width)
-
-#if(buffer_width < ghost_width)
-#error we always assume buffer_width>ghost_width
-#endif
-
-#define PACK 1
-#define UNPACK 2
-
-#define Mymax(a,b) (((a) > (b)) ? (a) : (b))
-#define Mymin(a,b) (((a) < (b)) ? (a) : (b))
-
-#define feq(a,b,d) (fabs(a-b)<d)
-#define flt(a,b,d) ((a-b)<d)
-#define fgt(a,b,d) ((a-b)>d)
-
-#define TINY 1e-10
-
-#endif   /* MICRODEF_H */
+
+#ifndef MICRODEF_H
+#define MICRODEF_H
+
+#include "macrodef.fh"  
+
+// application parameters
+
+#define SommerType 0
+
+#define GaussInt
+
+#define ABEtype 1
+
+//#define With_AHF
+#define Psi4type 0
+
+//#define Point_Psi4
+
+#define RPS 1
+
+#define AGM 0
+
+#define RPB 0
+
+#define MAPBH 1
+
+#define PSTR 0
+
+#define REGLEV 0
+
+#define BSSN_FINE_TIMING 0
+
+#define BSSN_FINE_TIMING_EVERY 1
+
+#define BSSN_FINE_TIMING_TOPN 8
+
+#define BSSN_KERNEL_FINE_TIMING 0
+
+#define BSSN_ENABLE_STDIN_ABORT_POLL 0
+
+//#define USE_GPU
+
+//#define CHECKDETAIL
+
+//#define FAKECHECK
+
+//
+// define SommerType
+//     sommerfeld boundary type
+//     0: bam
+//     1: shibata
+//
+// define GaussInt
+//     for Using Gauss-Legendre quadrature in theta direction
+//
+// define ABEtype
+//     0: BSSN vacuum
+//     1: coupled to scalar field
+//     2: Z4c vacuum
+//     3: coupled to Maxwell field
+//
+// define With_AHF
+//     using Apparent Horizon Finder
+//
+// define Psi4type
+//     Psi4 calculation method
+//     0: EB method
+//     1: 4-D method
+//
+// define Point_Psi4
+//     for Using point psi4 or not
+//
+// define RPS
+//     RestrictProlong in Step (0) or after Step (1)
+//
+// define AGM
+//     Enforce algebra constraint
+//     for every RK4 sub step: 0
+//     only when iter_count == 3: 1
+//     after routine Step: 2
+//
+// define RPB
+//     Restrict Prolong using BAM style 1 or old style 0
+//
+// define MAPBH
+//     1: move Analysis out ot 4 sub steps and treat PBH with Euler method
+//
+// define PSTR
+//     parallel structure
+//     0: level by level
+//     1: considering all levels
+//     2: as 1 but reverse the CPU order
+//     3: Frank's scheme
+//
+// define REGLEV
+//     regrid for every level or for all levels at a time
+//     0: for every level;
+//     1: for all
+//
+// define BSSN_FINE_TIMING
+//     enable fine-grained per-timestep timing monitor
+//
+// define BSSN_FINE_TIMING_EVERY
+//     report timing every N coarse timesteps
+//
+// define BSSN_FINE_TIMING_TOPN
+//     number of hottest timing buckets shown in stdout
+//
+// define BSSN_KERNEL_FINE_TIMING
+//     enable split timing inside compute_rhs_bssn
+//
+// define BSSN_ENABLE_STDIN_ABORT_POLL
+//     poll stdin and broadcast abort flag every coarse step
+//
+// define USE_GPU
+//     use gpu or not
+//
+// define CHECKDETAIL
+//     use checkpoint for every process
+//
+// define FAKECHECK
+//     use FakeCheckPrepare to write CheckPoint
+//
+
+////================================================================
+//  some basic parameters for numerical calculation
+////================================================================
+
+#define dim 3
+
+//#define Cell or Vertex in "macrodef.fh" 
+
+#define buffer_width 6
+
+#define SC_width buffer_width
+
+#define CS_width (2*buffer_width)
+
+//
+// define Cell or Vertex in "macrodef.fh" 
+//
+// define buffer_width
+//     buffer point number for mesh refinement interface
+//
+// define SC_width buffer_width
+//     buffer point number shell-box interface, on shell
+//
+// define CS_width
+//     buffer point number shell-box interface, on box
+//
+
+#if(buffer_width < ghost_width)
+#   error we always assume buffer_width>ghost_width
+#endif
+
+#define PACK 1
+#define UNPACK 2
+
+#define Mymax(a,b) (((a) > (b)) ? (a) : (b))
+#define Mymin(a,b) (((a) < (b)) ? (a) : (b))
+
+#define feq(a,b,d) (fabs(a-b)<d)
+#define flt(a,b,d) ((a-b)<d)
+#define fgt(a,b,d) ((a-b)>d)
+
+#define TINY 1e-10
+
+#endif   /* MICRODEF_H */
+

AMSS_NCKU_source/makefile

@@ -2,6 +2,53 @@
 
 include makefile.inc
 
+## polint(ordn=6) kernel selector:
+##   1 (default): barycentric fast path
+##   0          : fallback to Neville path
+POLINT6_USE_BARY ?= 1
+POLINT6_FLAG = -DPOLINT6_USE_BARYCENTRIC=$(POLINT6_USE_BARY)
+
+## ABE build flags selected by PGO_MODE (set in makefile.inc, default: opt)
+##   make                        -> opt  (PGO-guided, maximum performance)
+##   make PGO_MODE=instrument    -> instrument (Phase 1: collect fresh profile data)
+PROFDATA = /home/$(shell whoami)/AMSS-NCKU/pgo_profile/default.profdata
+
+ifeq ($(TOOLCHAIN),intel)
+OMP_FLAG = -qopenmp
+
+ifeq ($(PGO_MODE),instrument)
+## Intel Phase 1: instrumentation — omit -ipo/-fp-model fast=2 for faster build and numerical stability
+CXXAPPFLAGS = -O3 -xHost -fma -fprofile-instr-generate -ipo \
+              -Dfortran3 -Dnewc $(MKL_INC) $(INTERP_LB_FLAGS)
+f90appflags = -O3 -xHost -fma -fprofile-instr-generate -ipo \
+              -align array64byte -fpp $(MKL_INC) $(POLINT6_FLAG)
+else
+## opt (default): maximum performance with PGO profile data -fprofile-instr-use=$(PROFDATA) \
+## PGO has been turned off, now tested and found to be negative optimization
+## INTERP_LB_FLAGS has been turned off too, now tested and found to be negative optimization
+
+
+CXXAPPFLAGS = -O3 -xHost -fp-model fast=2 -fma -ipo \
+              -Dfortran3 -Dnewc $(MKL_INC) $(INTERP_LB_FLAGS)
+f90appflags = -O3 -xHost -fp-model fast=2 -fma -ipo \
+              -align array64byte -fpp $(MKL_INC) $(POLINT6_FLAG)
+endif
+
+TP_OPTFLAGS = -O3 -xHost -fp-model fast=2 -fma -ipo \
+              -fprofile-instr-use=$(TP_PROFDATA) \
+              -Dfortran3 -Dnewc $(MKL_INC)
+else
+## NVHPC defaults: mpicc/mpicxx/mpifort wrappers
+## PGO_MODE is ignored in this branch.
+OMP_FLAG = -mp
+CXXAPPFLAGS = -O3 -tp=host -Mcache_align -Mfma \
+              -Dfortran3 -Dnewc $(MKL_INC) $(INTERP_LB_FLAGS)
+f90appflags = -O3 -tp=host -Mcache_align -Mfma -Mpreprocess \
+              $(MKL_INC) $(POLINT6_FLAG)
+TP_OPTFLAGS = -O3 -tp=host -Mcache_align -Mfma \
+              -Dfortran3 -Dnewc $(MKL_INC)
+endif
+
 .SUFFIXES: .o .f90 .C .for .cu
 
 .f90.o:
@@ -13,18 +60,116 @@ include makefile.inc
 .for.o:
 	$(f77) -c $< -o $@
 
-.cu.o:
+.cu.o:
+	$(Cu) $(CUDA_APP_FLAGS) -c $< -o $@ $(CUDA_LIB_PATH)
+
+# CUDA rewrite of BSSN RHS (drop-in replacement for bssn_rhs_c + stencil helpers)
+bssn_rhs_cuda.o: bssn_rhs_cuda.cu bssn_rhs.h macrodef.h
+	$(Cu) $(CUDA_APP_FLAGS) -c $< -o $@ $(CUDA_LIB_PATH)
+
+# CUDA rewrite of Z4C Cartesian RHS
+z4c_rhs_cuda.o: z4c_rhs_cuda.cu z4c_rhs_cuda.h bssn_rhs.h macrodef.h ricci_gamma.h
 	$(Cu) $(CUDA_APP_FLAGS) -c $< -o $@ $(CUDA_LIB_PATH)
 
+# C rewrite of BSSN RHS kernel and helpers
+bssn_rhs_c.o: bssn_rhs_c.C
+	${CXX} $(CXXAPPFLAGS) -c $< $(filein) -o $@
+
+fderivs_c.o: fderivs_c.C
+	${CXX} $(CXXAPPFLAGS) -c $< $(filein) -o $@
+
+fdderivs_c.o: fdderivs_c.C
+	${CXX} $(CXXAPPFLAGS) -c $< $(filein) -o $@
+
+kodiss_c.o: kodiss_c.C
+	${CXX} $(CXXAPPFLAGS) -c $< $(filein) -o $@
+
+lopsided_c.o: lopsided_c.C
+	${CXX} $(CXXAPPFLAGS) -c $< $(filein) -o $@
+
+lopsided_kodis_c.o: lopsided_kodis_c.C
+	${CXX} $(CXXAPPFLAGS) -c $< $(filein) -o $@
+
+z4c_rhs_c.o: z4c_rhs_c.C
+	${CXX} $(CXXAPPFLAGS) -c $< $(filein) -o $@
+
+#interp_lb_profile.o: interp_lb_profile.C interp_lb_profile.h
+#	${CXX} $(CXXAPPFLAGS) -c $< $(filein) -o $@
+
+TwoPunctures.o: TwoPunctures.C
+	${CXX} $(TP_OPTFLAGS) $(OMP_FLAG) -c $< -o $@
+
+TwoPunctureABE.o: TwoPunctureABE.C
+	${CXX} $(TP_OPTFLAGS) $(OMP_FLAG) -c $< -o $@
+
 # Input files
+
+## CUDA BSSN RHS switch
+##   1 : use the rewritten CUDA bssn_rhs backend
+##   0 : keep the normal CPU/Fortran selection below
+USE_CUDA_BSSN ?= 0
+USE_CUDA_Z4C ?= 0
+
+CXXAPPFLAGS += -DUSE_CUDA_BSSN=$(USE_CUDA_BSSN)
+CUDA_APP_FLAGS += -DUSE_CUDA_BSSN=$(USE_CUDA_BSSN)
+CXXAPPFLAGS += -DUSE_CUDA_Z4C=$(USE_CUDA_Z4C)
+CUDA_APP_FLAGS += -DUSE_CUDA_Z4C=$(USE_CUDA_Z4C)
+
+## Kernel implementation switch (set USE_CXX_KERNELS=0 to fall back to Fortran)
+ifeq ($(USE_CXX_KERNELS),0)
+# Fortran mode: no C rewrite files; bssn_rhs.o is included via F90FILES below
+CFILES_CPU =
+else
+# C++ mode (default): C rewrite of bssn_rhs and helper kernels
+CFILES_CPU = bssn_rhs_c.o fderivs_c.o fdderivs_c.o kodiss_c.o lopsided_c.o lopsided_kodis_c.o
+endif
+
+CFILES_CUDA_BSSN = bssn_rhs_cuda.o
+
+ifeq ($(USE_CUDA_BSSN),1)
+CFILES = $(CFILES_CUDA_BSSN)
+else
+CFILES = $(CFILES_CPU)
+endif
+
+ifeq ($(USE_CUDA_Z4C),1)
+CFILES += z4c_rhs_cuda.o
+Z4C_F90_OBJ =
+else ifeq ($(USE_CXX_Z4C_KERNELS),1)
+CFILES += z4c_rhs_c.o
+Z4C_F90_OBJ =
+else
+Z4C_F90_OBJ = Z4c_rhs.o
+endif
+
+## RK4 kernel switch (independent from USE_CXX_KERNELS)
+ifeq ($(USE_CXX_RK4),1)
+RK4_C_OBJ = rungekutta4_rout_c.o
+RK4_F90_OBJ =
+else
+RK4_C_OBJ =
+RK4_F90_OBJ = rungekutta4_rout.o
+endif
+
+CFILES += $(RK4_C_OBJ)
+ABE_CUDA_CFILES = $(CFILES_CUDA_BSSN) z4c_rhs_cuda.o $(RK4_C_OBJ)
+
+ABE_LDLIBS = $(LDLIBS)
+ifeq ($(USE_CUDA_BSSN),1)
+ABE_LDLIBS += -lcudart $(CUDA_LIB_PATH)
+endif
+ifeq ($(USE_CUDA_Z4C),1)
+ABE_LDLIBS += -lcudart $(CUDA_LIB_PATH)
+endif
+
 C++FILES = ABE.o Ansorg.o Block.o misc.o monitor.o Parallel.o MPatch.o var.o\
            cgh.o bssn_class.o surface_integral.o ShellPatch.o\
 	   bssnEScalar_class.o perf.o Z4c_class.o NullShellPatch.o\
 	   bssnEM_class.o cpbc_util.o z4c_rhs_point.o checkpoint.o\
            Parallel_bam.o scalar_class.o transpbh.o NullShellPatch2.o\
-	   NullShellPatch2_Evo.o writefile_f.o
+	   NullShellPatch2_Evo.o writefile_f.o interp_lb_profile.o
 	   
-C++FILES_GPU = ABE.o Ansorg.o Block.o misc.o monitor.o Parallel.o MPatch.o var.o\
+#C++FILES_GPU = ABE.o Ansorg.o Block.o misc.o monitor.o Parallel.o MPatch.o var.o\
            cgh.o surface_integral.o ShellPatch.o\
 	   bssnEScalar_class.o perf.o Z4c_class.o NullShellPatch.o\
 	   bssnEM_class.o cpbc_util.o z4c_rhs_point.o checkpoint.o\
@@ -32,19 +177,27 @@ C++FILES_GPU = ABE.o Ansorg.o Block.o misc.o monitor.o Parallel.o MPatch.o var.o
 	   NullShellPatch2_Evo.o \
 	   bssn_gpu_class.o bssn_step_gpu.o bssn_macro.o writefile_f.o
 
-F90FILES = enforce_algebra.o fmisc.o initial_puncture.o prolongrestrict.o\
+F90FILES_BASE = enforce_algebra.o fmisc.o initial_puncture.o prolongrestrict.o\
 	   prolongrestrict_cell.o prolongrestrict_vertex.o\
-	   rungekutta4_rout.o bssn_rhs.o diff_new.o kodiss.o kodiss_sh.o\
+	   $(RK4_F90_OBJ) diff_new.o kodiss.o kodiss_sh.o\
 	   lopsidediff.o sommerfeld_rout.o getnp4.o diff_new_sh.o\
 	   shellfunctions.o bssn_rhs_ss.o Set_Rho_ADM.o\
            getnp4EScalar.o bssnEScalar_rhs.o bssn_constraint.o ricci_gamma.o\
-           fadmquantites_bssn.o Z4c_rhs.o Z4c_rhs_ss.o point_diff_new_sh.o\
+           fadmquantites_bssn.o $(Z4C_F90_OBJ) Z4c_rhs_ss.o point_diff_new_sh.o\
 	   cpbc.o getnp4old.o NullEvol.o initial_null.o initial_maxwell.o\
 	   getnpem2.o empart.o NullNews.o fourdcurvature.o\
 	   bssn2adm.o adm_constraint.o adm_ricci_gamma.o\
 	   scalar_rhs.o initial_scalar.o NullEvol2.o initial_null2.o\
 	   NullNews2.o tool_f.o
 
+ifeq ($(USE_CXX_KERNELS),0)
+# Fortran mode: include original bssn_rhs.o
+F90FILES = $(F90FILES_BASE) bssn_rhs.o
+else
+# C++ mode (default): bssn_rhs.o replaced by C++ kernel
+F90FILES = $(F90FILES_BASE)
+endif
+
 F77FILES = zbesh.o
 
 AHFDOBJS = expansion.o expansion_Jacobian.o patch.o coords.o patch_info.o patch_interp.o patch_system.o \
@@ -54,10 +207,10 @@ initial_guess.o Newton.o Jacobian.o ilucg.o IntPnts0.o IntPnts.o
 
 TwoPunctureFILES = TwoPunctureABE.o TwoPunctures.o 
 
-CUDAFILES = bssn_gpu.o bssn_gpu_rhs_ss.o
+#CUDAFILES = bssn_gpu.o bssn_gpu_rhs_ss.o
 
 # file dependences
-$(C++FILES) $(C++FILESGPU) $(F90FILES) $(AHFDOBJS) $(CUDAFILES): macrodef.fh
+$(C++FILES) $(C++FILES_GPU) $(F90FILES) $(CFILES) $(ABE_CUDA_CFILES) $(AHFDOBJS) $(CUDAFILES): macrodef.fh
 
 $(C++FILES): Block.h enforce_algebra.h fmisc.h initial_puncture.h macrodef.h\
 	     misc.h monitor.h MyList.h Parallel.h MPatch.h prolongrestrict.h\
@@ -68,7 +221,7 @@ $(C++FILES): Block.h enforce_algebra.h fmisc.h initial_puncture.h macrodef.h\
 	     empart.h NullNews.h kodiss.h Parallel_bam.h ricci_gamma.h\
              initial_null2.h NullShellPatch2.h 
              
-$(C++FILES_GPU): Block.h enforce_algebra.h fmisc.h initial_puncture.h macrodef.h\
+#$(C++FILES_GPU): Block.h enforce_algebra.h fmisc.h initial_puncture.h macrodef.h\
 	     misc.h monitor.h MyList.h Parallel.h MPatch.h prolongrestrict.h\
 	     rungekutta4_rout.h var.h bssn_rhs.h sommerfeld_rout.h\
 	     cgh.h surface_integral.h ShellPatch.h shellfunctions.h perf.h\
@@ -80,7 +233,7 @@ $(C++FILES_GPU): Block.h enforce_algebra.h fmisc.h initial_puncture.h macrodef.h
              
 $(AHFDOBJS): cctk.h cctk_Config.h cctk_Types.h cctk_Constants.h myglobal.h
 
-$(C++FILES) $(C++FILES_GPU) $(AHFDOBJS) $(CUDAFILES): macrodef.h
+$(C++FILES) $(C++FILES_GPU) $(CFILES) $(ABE_CUDA_CFILES) $(AHFDOBJS) $(CUDAFILES): macrodef.h
 
 TwoPunctureFILES: TwoPunctures.h
 
@@ -89,14 +242,19 @@ $(CUDAFILES): bssn_gpu.h gpu_mem.h gpu_rhsSS_mem.h
 misc.o : zbesh.o
 
 # projects
-ABE: $(C++FILES) $(F90FILES) $(F77FILES) $(AHFDOBJS) 
-	$(CLINKER) $(CXXAPPFLAGS) -o $@ $(C++FILES) $(F90FILES) $(F77FILES) $(AHFDOBJS) $(LDLIBS)
+ABE: $(C++FILES) $(CFILES) $(F90FILES) $(F77FILES) $(AHFDOBJS)
+	$(CLINKER) $(CXXAPPFLAGS) -o $@ $(C++FILES) $(CFILES) $(F90FILES) $(F77FILES) $(AHFDOBJS) $(ABE_LDLIBS)
+
+ABE_CUDA: USE_CUDA_BSSN=1
+ABE_CUDA: USE_CUDA_Z4C=1
+ABE_CUDA: $(C++FILES) $(ABE_CUDA_CFILES) $(F90FILES) $(F77FILES) $(AHFDOBJS)
+	$(CLINKER) $(CXXAPPFLAGS) -o $@ $(C++FILES) $(ABE_CUDA_CFILES) $(F90FILES) $(F77FILES) $(AHFDOBJS) $(LDLIBS) -lcudart $(CUDA_LIB_PATH)
 	
-ABEGPU: $(C++FILES_GPU) $(F90FILES) $(F77FILES) $(AHFDOBJS) $(CUDAFILES)
-	$(CLINKER) $(CXXAPPFLAGS) -o $@ $(C++FILES_GPU) $(F90FILES) $(F77FILES) $(AHFDOBJS) $(CUDAFILES) $(LDLIBS)
+#ABEGPU: $(C++FILES_GPU) $(CFILES) $(F90FILES) $(F77FILES) $(AHFDOBJS) $(CUDAFILES)
+#	$(CLINKER) $(CXXAPPFLAGS) -o $@ $(C++FILES_GPU) $(CFILES) $(F90FILES) $(F77FILES) $(AHFDOBJS) $(CUDAFILES) $(LDLIBS)
 
 TwoPunctureABE: $(TwoPunctureFILES)
-	$(CLINKER) $(CXXAPPFLAGS) -o $@ $(TwoPunctureFILES) $(LDLIBS)
+	$(CLINKER) $(TP_OPTFLAGS) $(OMP_FLAG) -o $@ $(TwoPunctureFILES) $(LDLIBS)
 
 clean:
-	rm *.o ABE ABEGPU TwoPunctureABE make.log -f
+	rm *.o ABE ABE_CUDA ABEGPU TwoPunctureABE make.log -f

AMSS_NCKU_source/makefile.inc

@@ -1,21 +1,91 @@
+## Toolchain selection
+##   nvhpc : NVIDIA HPC SDK + CUDA-aware MPI (default)
+##   intel : Intel oneAPI toolchain (legacy path)
+TOOLCHAIN ?= nvhpc
 
-## filein  = -I/usr/include -I/usr/lib/x86_64-linux-gnu/mpich/include -I/usr/lib/x86_64-linux-gnu/openmpi/lib/ -I/usr/lib/gcc/x86_64-linux-gnu/11/ -I/usr/include/c++/11/
+## PGO build mode switch (ABE only; TwoPunctureABE always uses opt flags)
+##   opt        : (default) maximum performance with PGO profile-guided optimization
+##   instrument : PGO Phase 1 instrumentation to collect fresh profile data
+PGO_MODE ?= opt
 
-filein  = -I/usr/include/ -I/usr/lib/x86_64-linux-gnu/openmpi/include/ -I/usr/lib/x86_64-linux-gnu/openmpi/lib/ -I/usr/lib/gcc/x86_64-linux-gnu/11/ -I/usr/include/c++/11/ -I/usr/lib/cuda/include
+## Interp_Points load balance profiling mode
+##   off        : (default) no load balance instrumentation
+##   profile    : Pass 1 — instrument Interp_Points to collect timing profile
+##   optimize   : Pass 2 — read profile and apply block rebalancing
+INTERP_LB_MODE ?= off
 
-## LDLIBS  = -L/usr/lib/x86_64-linux-gnu -lmpich -L/usr/lib64 -L/usr/lib/gcc/x86_64-linux-gnu/11 -lgfortran
-LDLIBS  = -L/usr/lib/x86_64-linux-gnu -L/usr/lib64 -L/usr/lib/gcc/x86_64-linux-gnu/11 -lgfortran -L/usr/lib/cuda/lib64 -lcudart -lmpi -lgfortran
+ifeq ($(INTERP_LB_MODE),profile)
+INTERP_LB_FLAGS = -DINTERP_LB_PROFILE
+else ifeq ($(INTERP_LB_MODE),optimize)
+INTERP_LB_FLAGS = -DINTERP_LB_OPTIMIZE
+else
+INTERP_LB_FLAGS =
+endif
 
-CXXAPPFLAGS  = -O3 -Wno-deprecated -Dfortran3 -Dnewc
-#f90appflags = -O3 -fpp
-f90appflags  = -O3 -x f95-cpp-input
-f90          = gfortran 
-f77          = gfortran 
-CXX          = g++
-CC           = gcc
-CLINKER      = mpic++ 
+MKLROOT ?= /home/intel/oneapi/mkl/latest
+MKL_LIBDIR ?= $(MKLROOT)/lib/intel64
+MKL_INC ?= -I$(MKLROOT)/include
 
-Cu = nvcc
-CUDA_LIB_PATH = -L/usr/lib/cuda/lib64 -I/usr/include -I/usr/lib/cuda/include
-#CUDA_APP_FLAGS = -c -g -O3 --ptxas-options=-v -arch compute_13 -code compute_13,sm_13 -Dfortran3 -Dnewc
-CUDA_APP_FLAGS = -c -g -O3 --ptxas-options=-v -Dfortran3 -Dnewc
+NVHPC_ROOT ?= /home/nvidia/hpc_sdk/Linux_x86_64/25.11
+CUDA_HOME  ?= $(NVHPC_ROOT)/cuda
+CUDA_ARCH  ?= sm_80
+
+## Kernel implementation switch
+##   1 (default) : use C++ rewrite of bssn_rhs and helper kernels (faster)
+##   0           : fall back to original Fortran kernels
+USE_CXX_KERNELS ?= 1
+
+## Z4C Cartesian RHS kernel switch
+##   1 (default) : use C++ rewrite of Z4c_rhs (main Cartesian path faster)
+##   0           : use original Fortran Z4c_rhs.o
+USE_CXX_Z4C_KERNELS ?= 1
+
+## RK4 kernel implementation switch
+##   1 (default) : use C/C++ rewrite of rungekutta4_rout (for optimization experiments)
+##   0           : use original Fortran rungekutta4_rout.o
+USE_CXX_RK4 ?= 1
+
+## Memory allocator switch
+##   1 (default) : link Intel oneTBB allocator (libtbbmalloc)
+##   0           : use system default allocator (ptmalloc)
+USE_TBBMALLOC ?= 1
+TBBMALLOC_SO ?= /home/intel/oneapi/2025.3/lib/libtbbmalloc.so
+ifneq ($(wildcard $(TBBMALLOC_SO)),)
+TBBMALLOC_LIBS = -Wl,--no-as-needed $(TBBMALLOC_SO) -Wl,--as-needed
+else
+TBBMALLOC_LIBS = -Wl,--no-as-needed -ltbbmalloc -Wl,--as-needed
+endif
+
+ifeq ($(TOOLCHAIN),intel)
+f90          = ifx
+f77          = ifx
+CXX          = icpx
+CC           = icx
+CLINKER      = mpiicpx
+filein  = -I/usr/include/ $(MKL_INC) -I$(CUDA_HOME)/include
+LDLIBS       = -L$(MKL_LIBDIR) -Wl,-rpath,$(MKL_LIBDIR) \
+               -lmkl_intel_lp64 -lmkl_sequential -lmkl_core \
+               -lifcore -limf -liomp5 -lpthread -lm -ldl \
+               -L$(CUDA_HOME)/lib64 -Wl,-rpath,$(CUDA_HOME)/lib64 -lcuda -lcudart
+else ifeq ($(TOOLCHAIN),nvhpc)
+f90          = mpifort
+f77          = mpifort
+CXX          = mpicxx
+CC           = mpicc
+CLINKER      = mpicxx
+
+filein       = -I/usr/include/ $(MKL_INC) -I$(CUDA_HOME)/include
+LDLIBS       = -L$(MKL_LIBDIR) -Wl,-rpath,$(MKL_LIBDIR) \
+               -lmkl_intel_lp64 -lmkl_sequential -lmkl_core \
+               -lpthread -lm -ldl \
+               -L$(CUDA_HOME)/lib64 -Wl,-rpath,$(CUDA_HOME)/lib64 -lcuda -lcudart \
+               -fortranlibs
+endif
+
+ifeq ($(USE_TBBMALLOC),1)
+LDLIBS := $(TBBMALLOC_LIBS) $(LDLIBS)
+endif
+
+Cu = $(NVHPC_ROOT)/compilers/bin/nvcc
+CUDA_LIB_PATH = -L$(CUDA_HOME)/lib64 -I$(CUDA_HOME)/include
+CUDA_APP_FLAGS = -c -g -O3 --ptxas-options=-v -Dfortran3 -Dnewc -arch=$(CUDA_ARCH)

AMSS_NCKU_source/prolongrestrict_cell.f90

@@ -1934,18 +1934,35 @@
 ! when if=1 -> ic=0, this is different to vertex center grid 
   real*8, dimension(-2:extc(1),-2:extc(2),-2:extc(3))   :: funcc
   integer,dimension(3) :: cxI
-  integer :: i,j,k,ii,jj,kk
+  integer :: i,j,k,ii,jj,kk,px,py,pz
   real*8, dimension(6,6) :: tmp2
   real*8, dimension(6) :: tmp1
+  integer, dimension(extf(1)) :: cix
+  integer, dimension(extf(2)) :: ciy
+  integer, dimension(extf(3)) :: ciz
+  integer, dimension(extf(1)) :: pix
+  integer, dimension(extf(2)) :: piy
+  integer, dimension(extf(3)) :: piz
 
   real*8, parameter :: C1=7.7d1/8.192d3,C2=-6.93d2/8.192d3,C3=3.465d3/4.096d3
   real*8, parameter :: C6=6.3d1/8.192d3,C5=-4.95d2/8.192d3,C4=1.155d3/4.096d3
+  real*8, dimension(6,2), parameter :: WC = reshape((/&
+      C1,C2,C3,C4,C5,C6,&
+      C6,C5,C4,C3,C2,C1/), (/6,2/))
 
   integer::imini,imaxi,jmini,jmaxi,kmini,kmaxi
   integer::imino,imaxo,jmino,jmaxo,kmino,kmaxo
+  integer::maxcx,maxcy,maxcz
 
   real*8,dimension(3) :: CD,FD
-  
+  real*8 :: tmp_yz(extc(1), 6)      ! 存储整条 X 线上 6 个 Y 轴偏置的 Z 向插值结果
+  real*8 :: tmp_xyz_line(-2:extc(1))   ! 包含 X 向 6 点模板访问所需下界
+  real*8 :: v1, v2, v3, v4, v5, v6
+  integer :: ic, jc, kc, ix_offset,ix,iy,iz,jc_min,jc_max,ic_min,ic_max,kc_min,kc_max
+  integer :: i_lo, i_hi, j_lo, j_hi, k_lo, k_hi
+  logical :: need_full_symmetry
+  real*8 :: res_line
+  real*8 :: tmp_z_slab(-2:extc(1), -2:extc(2))  ! 包含 Y/X 向模板访问所需下界
   if(wei.ne.3)then
      write(*,*)"prolongrestrict.f90::prolong3: this routine only surport 3 dimension"
      write(*,*)"dim = ",wei
@@ -2020,145 +2037,140 @@
           return
   endif
 
-  call symmetry_bd(3,extc,func,funcc,SoA)
-     
-!~~~~~~> prolongation start...
+  do i = imino,imaxo
+     ii = i + lbf(1) - 1
+     cix(i) = ii/2 - lbc(1) + 1
+     if(ii/2*2 == ii)then
+        pix(i) = 1
+     else
+        pix(i) = 2
+     endif
+  enddo
+  do j = jmino,jmaxo
+     jj = j + lbf(2) - 1
+     ciy(j) = jj/2 - lbc(2) + 1
+     if(jj/2*2 == jj)then
+        piy(j) = 1
+     else
+        piy(j) = 2
+     endif
+  enddo
   do k = kmino,kmaxo
-   do j = jmino,jmaxo
-    do i = imino,imaxo
-       cxI(1) = i
-       cxI(2) = j
-       cxI(3) = k
-! change to coarse level reference
-!|---*--- ---*--- ---*--- ---*--- ---*--- ---*--- ---*--- ---*---| 
-!|=======x===============x===============x===============x=======|
-       cxI = (cxI+lbf-1)/2
-! change to array index      
-       cxI = cxI - lbc + 1
-
-       if(any(cxI+3 > extc)) write(*,*)"error in prolong"
-       ii=i+lbf(1)-1
-       jj=j+lbf(2)-1
-       kk=k+lbf(3)-1
-#if 0
-       if(ii/2*2==ii)then
-         if(jj/2*2==jj)then
-           if(kk/2*2==kk)then
-             tmp2= C1*funcc(cxI(1)-2:cxI(1)+3,cxI(2)-2:cxI(2)+3,cxI(3)-2)+&
-                   C2*funcc(cxI(1)-2:cxI(1)+3,cxI(2)-2:cxI(2)+3,cxI(3)-1)+&
-                   C3*funcc(cxI(1)-2:cxI(1)+3,cxI(2)-2:cxI(2)+3,cxI(3)  )+&
-                   C4*funcc(cxI(1)-2:cxI(1)+3,cxI(2)-2:cxI(2)+3,cxI(3)+1)+&
-                   C5*funcc(cxI(1)-2:cxI(1)+3,cxI(2)-2:cxI(2)+3,cxI(3)+2)+&
-                   C6*funcc(cxI(1)-2:cxI(1)+3,cxI(2)-2:cxI(2)+3,cxI(3)+3)
-             tmp1= C1*tmp2(:,1)+C2*tmp2(:,2)+C3*tmp2(:,3)+C4*tmp2(:,4)+C5*tmp2(:,5)+C6*tmp2(:,6)
-             funf(i,j,k)= C1*tmp1(1)+C2*tmp1(2)+C3*tmp1(3)+C4*tmp1(4)+C5*tmp1(5)+C6*tmp1(6)
-           else
-             tmp2= C6*funcc(cxI(1)-2:cxI(1)+3,cxI(2)-2:cxI(2)+3,cxI(3)-2)+&
-                   C5*funcc(cxI(1)-2:cxI(1)+3,cxI(2)-2:cxI(2)+3,cxI(3)-1)+&
-                   C4*funcc(cxI(1)-2:cxI(1)+3,cxI(2)-2:cxI(2)+3,cxI(3)  )+&
-                   C3*funcc(cxI(1)-2:cxI(1)+3,cxI(2)-2:cxI(2)+3,cxI(3)+1)+&
-                   C2*funcc(cxI(1)-2:cxI(1)+3,cxI(2)-2:cxI(2)+3,cxI(3)+2)+&
-                   C1*funcc(cxI(1)-2:cxI(1)+3,cxI(2)-2:cxI(2)+3,cxI(3)+3)
-             tmp1= C1*tmp2(:,1)+C2*tmp2(:,2)+C3*tmp2(:,3)+C4*tmp2(:,4)+C5*tmp2(:,5)+C6*tmp2(:,6)
-             funf(i,j,k)=  C1*tmp1(1)+C2*tmp1(2)+C3*tmp1(3)+C4*tmp1(4)+C5*tmp1(5)+C6*tmp1(6)
-           endif
-         else
-           if(kk/2*2==kk)then
-             tmp2= C1*funcc(cxI(1)-2:cxI(1)+3,cxI(2)-2:cxI(2)+3,cxI(3)-2)+&
-                   C2*funcc(cxI(1)-2:cxI(1)+3,cxI(2)-2:cxI(2)+3,cxI(3)-1)+&
-                   C3*funcc(cxI(1)-2:cxI(1)+3,cxI(2)-2:cxI(2)+3,cxI(3)  )+&
-                   C4*funcc(cxI(1)-2:cxI(1)+3,cxI(2)-2:cxI(2)+3,cxI(3)+1)+&
-                   C5*funcc(cxI(1)-2:cxI(1)+3,cxI(2)-2:cxI(2)+3,cxI(3)+2)+&
-                   C6*funcc(cxI(1)-2:cxI(1)+3,cxI(2)-2:cxI(2)+3,cxI(3)+3)
-             tmp1= C6*tmp2(:,1)+C5*tmp2(:,2)+C4*tmp2(:,3)+C3*tmp2(:,4)+C2*tmp2(:,5)+C1*tmp2(:,6)
-             funf(i,j,k)= C1*tmp1(1)+C2*tmp1(2)+C3*tmp1(3)+C4*tmp1(4)+C5*tmp1(5)+C6*tmp1(6)
-           else
-             tmp2= C6*funcc(cxI(1)-2:cxI(1)+3,cxI(2)-2:cxI(2)+3,cxI(3)-2)+&
-                   C5*funcc(cxI(1)-2:cxI(1)+3,cxI(2)-2:cxI(2)+3,cxI(3)-1)+&
-                   C4*funcc(cxI(1)-2:cxI(1)+3,cxI(2)-2:cxI(2)+3,cxI(3)  )+&
-                   C3*funcc(cxI(1)-2:cxI(1)+3,cxI(2)-2:cxI(2)+3,cxI(3)+1)+&
-                   C2*funcc(cxI(1)-2:cxI(1)+3,cxI(2)-2:cxI(2)+3,cxI(3)+2)+&
-                   C1*funcc(cxI(1)-2:cxI(1)+3,cxI(2)-2:cxI(2)+3,cxI(3)+3)
-             tmp1= C6*tmp2(:,1)+C5*tmp2(:,2)+C4*tmp2(:,3)+C3*tmp2(:,4)+C2*tmp2(:,5)+C1*tmp2(:,6)
-             funf(i,j,k)=  C1*tmp1(1)+C2*tmp1(2)+C3*tmp1(3)+C4*tmp1(4)+C5*tmp1(5)+C6*tmp1(6)
-           endif
-         endif
-       else
-         if(jj/2*2==jj)then
-           if(kk/2*2==kk)then               
-             tmp2= C1*funcc(cxI(1)-2:cxI(1)+3,cxI(2)-2:cxI(2)+3,cxI(3)-2)+&
-                   C2*funcc(cxI(1)-2:cxI(1)+3,cxI(2)-2:cxI(2)+3,cxI(3)-1)+&
-                   C3*funcc(cxI(1)-2:cxI(1)+3,cxI(2)-2:cxI(2)+3,cxI(3)  )+&
-                   C4*funcc(cxI(1)-2:cxI(1)+3,cxI(2)-2:cxI(2)+3,cxI(3)+1)+&
-                   C5*funcc(cxI(1)-2:cxI(1)+3,cxI(2)-2:cxI(2)+3,cxI(3)+2)+&
-                   C6*funcc(cxI(1)-2:cxI(1)+3,cxI(2)-2:cxI(2)+3,cxI(3)+3)
-             tmp1= C1*tmp2(:,1)+C2*tmp2(:,2)+C3*tmp2(:,3)+C4*tmp2(:,4)+C5*tmp2(:,5)+C6*tmp2(:,6)
-             funf(i,j,k)= C6*tmp1(1)+C5*tmp1(2)+C4*tmp1(3)+C3*tmp1(4)+C2*tmp1(5)+C1*tmp1(6)
-           else
-             tmp2= C6*funcc(cxI(1)-2:cxI(1)+3,cxI(2)-2:cxI(2)+3,cxI(3)-2)+&
-                   C5*funcc(cxI(1)-2:cxI(1)+3,cxI(2)-2:cxI(2)+3,cxI(3)-1)+&
-                   C4*funcc(cxI(1)-2:cxI(1)+3,cxI(2)-2:cxI(2)+3,cxI(3)  )+&
-                   C3*funcc(cxI(1)-2:cxI(1)+3,cxI(2)-2:cxI(2)+3,cxI(3)+1)+&
-                   C2*funcc(cxI(1)-2:cxI(1)+3,cxI(2)-2:cxI(2)+3,cxI(3)+2)+&
-                   C1*funcc(cxI(1)-2:cxI(1)+3,cxI(2)-2:cxI(2)+3,cxI(3)+3)
-             tmp1= C1*tmp2(:,1)+C2*tmp2(:,2)+C3*tmp2(:,3)+C4*tmp2(:,4)+C5*tmp2(:,5)+C6*tmp2(:,6)
-             funf(i,j,k)=  C6*tmp1(1)+C5*tmp1(2)+C4*tmp1(3)+C3*tmp1(4)+C2*tmp1(5)+C1*tmp1(6)
-           endif
-         else
-           if(kk/2*2==kk)then
-             tmp2= C1*funcc(cxI(1)-2:cxI(1)+3,cxI(2)-2:cxI(2)+3,cxI(3)-2)+&
-                   C2*funcc(cxI(1)-2:cxI(1)+3,cxI(2)-2:cxI(2)+3,cxI(3)-1)+&
-                   C3*funcc(cxI(1)-2:cxI(1)+3,cxI(2)-2:cxI(2)+3,cxI(3)  )+&
-                   C4*funcc(cxI(1)-2:cxI(1)+3,cxI(2)-2:cxI(2)+3,cxI(3)+1)+&
-                   C5*funcc(cxI(1)-2:cxI(1)+3,cxI(2)-2:cxI(2)+3,cxI(3)+2)+&
-                   C6*funcc(cxI(1)-2:cxI(1)+3,cxI(2)-2:cxI(2)+3,cxI(3)+3)
-             tmp1= C6*tmp2(:,1)+C5*tmp2(:,2)+C4*tmp2(:,3)+C3*tmp2(:,4)+C2*tmp2(:,5)+C1*tmp2(:,6)
-             funf(i,j,k)= C6*tmp1(1)+C5*tmp1(2)+C4*tmp1(3)+C3*tmp1(4)+C2*tmp1(5)+C1*tmp1(6)
-           else
-             tmp2= C6*funcc(cxI(1)-2:cxI(1)+3,cxI(2)-2:cxI(2)+3,cxI(3)-2)+&
-                   C5*funcc(cxI(1)-2:cxI(1)+3,cxI(2)-2:cxI(2)+3,cxI(3)-1)+&
-                   C4*funcc(cxI(1)-2:cxI(1)+3,cxI(2)-2:cxI(2)+3,cxI(3)  )+&
-                   C3*funcc(cxI(1)-2:cxI(1)+3,cxI(2)-2:cxI(2)+3,cxI(3)+1)+&
-                   C2*funcc(cxI(1)-2:cxI(1)+3,cxI(2)-2:cxI(2)+3,cxI(3)+2)+&
-                   C1*funcc(cxI(1)-2:cxI(1)+3,cxI(2)-2:cxI(2)+3,cxI(3)+3)
-             tmp1= C6*tmp2(:,1)+C5*tmp2(:,2)+C4*tmp2(:,3)+C3*tmp2(:,4)+C2*tmp2(:,5)+C1*tmp2(:,6)
-             funf(i,j,k)=  C6*tmp1(1)+C5*tmp1(2)+C4*tmp1(3)+C3*tmp1(4)+C2*tmp1(5)+C1*tmp1(6)
-           endif
-         endif
-       endif
-#else 
-       if(kk/2*2==kk)then
-             tmp2= C1*funcc(cxI(1)-2:cxI(1)+3,cxI(2)-2:cxI(2)+3,cxI(3)-2)+&
-                   C2*funcc(cxI(1)-2:cxI(1)+3,cxI(2)-2:cxI(2)+3,cxI(3)-1)+&
-                   C3*funcc(cxI(1)-2:cxI(1)+3,cxI(2)-2:cxI(2)+3,cxI(3)  )+&
-                   C4*funcc(cxI(1)-2:cxI(1)+3,cxI(2)-2:cxI(2)+3,cxI(3)+1)+&
-                   C5*funcc(cxI(1)-2:cxI(1)+3,cxI(2)-2:cxI(2)+3,cxI(3)+2)+&
-                   C6*funcc(cxI(1)-2:cxI(1)+3,cxI(2)-2:cxI(2)+3,cxI(3)+3)
-       else
-             tmp2= C6*funcc(cxI(1)-2:cxI(1)+3,cxI(2)-2:cxI(2)+3,cxI(3)-2)+&
-                   C5*funcc(cxI(1)-2:cxI(1)+3,cxI(2)-2:cxI(2)+3,cxI(3)-1)+&
-                   C4*funcc(cxI(1)-2:cxI(1)+3,cxI(2)-2:cxI(2)+3,cxI(3)  )+&
-                   C3*funcc(cxI(1)-2:cxI(1)+3,cxI(2)-2:cxI(2)+3,cxI(3)+1)+&
-                   C2*funcc(cxI(1)-2:cxI(1)+3,cxI(2)-2:cxI(2)+3,cxI(3)+2)+&
-                   C1*funcc(cxI(1)-2:cxI(1)+3,cxI(2)-2:cxI(2)+3,cxI(3)+3)
-       endif
-
-       if(jj/2*2==jj)then
-             tmp1= C1*tmp2(:,1)+C2*tmp2(:,2)+C3*tmp2(:,3)+C4*tmp2(:,4)+C5*tmp2(:,5)+C6*tmp2(:,6)
-       else
-             tmp1= C6*tmp2(:,1)+C5*tmp2(:,2)+C4*tmp2(:,3)+C3*tmp2(:,4)+C2*tmp2(:,5)+C1*tmp2(:,6)
-       endif
-
-       if(ii/2*2==ii)then
-             funf(i,j,k)= C1*tmp1(1)+C2*tmp1(2)+C3*tmp1(3)+C4*tmp1(4)+C5*tmp1(5)+C6*tmp1(6)
-       else
-             funf(i,j,k)= C6*tmp1(1)+C5*tmp1(2)+C4*tmp1(3)+C3*tmp1(4)+C2*tmp1(5)+C1*tmp1(6)
-       endif
-#endif
-    enddo
-   enddo
+     kk = k + lbf(3) - 1
+     ciz(k) = kk/2 - lbc(3) + 1
+     if(kk/2*2 == kk)then
+        piz(k) = 1
+     else
+        piz(k) = 2
+     endif
   enddo
 
+  ic_min = minval(cix(imino:imaxo))
+  ic_max = maxval(cix(imino:imaxo))
+  jc_min = minval(ciy(jmino:jmaxo))
+  jc_max = maxval(ciy(jmino:jmaxo))
+  kc_min = minval(ciz(kmino:kmaxo))
+  kc_max = maxval(ciz(kmino:kmaxo))
+
+  maxcx = ic_max
+  maxcy = jc_max
+  maxcz = kc_max
+  if(maxcx+3 > extc(1) .or. maxcy+3 > extc(2) .or. maxcz+3 > extc(3))then
+     write(*,*)"error in prolong"
+     return
+  endif
+
+  i_lo = ic_min - 2
+  i_hi = ic_max + 3
+  j_lo = jc_min - 2
+  j_hi = jc_max + 3
+  k_lo = kc_min - 2
+  k_hi = kc_max + 3
+  need_full_symmetry = (i_lo < 1) .or. (j_lo < 1) .or. (k_lo < 1)
+  if(need_full_symmetry)then
+     call symmetry_bd(3,extc,func,funcc,SoA)
+  else
+     funcc(i_lo:i_hi,j_lo:j_hi,k_lo:k_hi) = func(i_lo:i_hi,j_lo:j_hi,k_lo:k_hi)
+  endif
+
+     ! 对每个 k（pz, kc 固定）预计算 Z 向插值的 2D 切片
+
+do k = kmino, kmaxo
+    pz = piz(k); kc = ciz(k)
+    ! --- Pass 1: Z 方向，只算一次 ---
+    do iy = jc_min-2, jc_max+3   ! 仅需的 iy 范围（对应 jc-2:jc+3）
+        do ii = ic_min-2, ic_max+3  ! 仅需的 ii 范围（对应 cix-2:cix+3）
+            tmp_z_slab(ii, iy) = sum(WC(:,pz) * funcc(ii, iy, kc-2:kc+3))
+        end do
+    end do
+
+    do j = jmino, jmaxo
+        py = piy(j); jc = ciy(j)
+        ! --- Pass 2: Y 方向 ---
+        do ii = ic_min-2, ic_max+3
+            tmp_xyz_line(ii) = sum(WC(:,py) * tmp_z_slab(ii, jc-2:jc+3))
+        end do
+        ! --- Pass 3: X 方向 ---
+        do i = imino, imaxo
+            funf(i,j,k) = sum(WC(:,pix(i)) * tmp_xyz_line(cix(i)-2:cix(i)+3))
+        end do
+    end do
+end do
+
+!~~~~~~> prolongation start...
+#if 0
+ do k = kmino, kmaxo
+     pz = piz(k)
+     kc = ciz(k)
+
+     do j = jmino, jmaxo
+        py = piy(j)
+        jc = ciy(j)
+
+! --- 步骤 1 & 2 融合：分段处理 X 轴，提升 Cache 命中率 ---
+        ! 我们将 ii 循环逻辑重组，减少对 funcc 的跨行重复访问
+        do ii = 1, extc(1)
+           ! 1. 先做 Z 方向的 6 条线插值（针对当前的 ii 和当前的 6 个 iy）
+           ! 我们直接在这里把 Y 方向的加权也做了，省去 tmp_yz 数组
+           ! 这样 funcc 的数据读进来后立即完成所有维度的贡献，不再写回内存
+           
+           res_line = 0.0d0
+           do jj = 1, 6
+              iy = jc - 3 + jj
+              ! 这一行代码是核心：一次性完成 Z 插值并加上 Y 的权重
+              ! 编译器会把 WC(jj, py) 存在寄存器里
+              res_line = res_line + WC(jj, py) * ( &
+                         WC(1, pz) * funcc(ii, iy, kc-2) + &
+                         WC(2, pz) * funcc(ii, iy, kc-1) + &
+                         WC(3, pz) * funcc(ii, iy, kc  ) + &
+                         WC(4, pz) * funcc(ii, iy, kc+1) + &
+                         WC(5, pz) * funcc(ii, iy, kc+2) + &
+                         WC(6, pz) * funcc(ii, iy, kc+3) )
+           end do
+           tmp_xyz_line(ii) = res_line
+        end do
+
+
+
+
+        ! 3. 【降维：X 向】最后在最内层只处理 X 方向的 6 点加权
+        ! 此时每个点的计算量从原来的 200+ 次乘法降到了仅 6 次
+        do i = imino, imaxo
+           px = pix(i)
+           ic = cix(i)
+           
+           ! 直接从预计算好的 line 中读取连续的 6 个点
+           ! ic-2 到 ic+3 对应原始 6 点算子
+           funf(i,j,k) = WC(1,px)*tmp_xyz_line(ic-2) + &
+                         WC(2,px)*tmp_xyz_line(ic-1) + &
+                         WC(3,px)*tmp_xyz_line(ic  ) + &
+                         WC(4,px)*tmp_xyz_line(ic+1) + &
+                         WC(5,px)*tmp_xyz_line(ic+2) + &
+                         WC(6,px)*tmp_xyz_line(ic+3)
+        end do
+     end do
+  end do
+#endif
   return
 
   end subroutine prolong3
@@ -2357,7 +2369,14 @@
   integer::imino,imaxo,jmino,jmaxo,kmino,kmaxo
 
   real*8,dimension(3) :: CD,FD
-  
+
+  real*8 :: tmp_xz_plane(-1:extf(1), 6)
+  real*8 :: tmp_x_line(-1:extf(1))
+  integer :: fi, fj, fk, ii, jj, kk
+  integer :: fi_min, fi_max, ii_lo, ii_hi
+  integer :: fj_min, fj_max, fk_min, fk_max, jj_lo, jj_hi, kk_lo, kk_hi
+  logical :: need_full_symmetry
+
   if(wei.ne.3)then
      write(*,*)"prolongrestrict.f90::restrict3: this routine only surport 3 dimension"
      write(*,*)"dim = ",wei
@@ -2436,9 +2455,86 @@
           stop
   endif
 
-  call symmetry_bd(2,extf,funf,funff,SoA)
+  ! 仅计算 X 向最终写回所需的窗口：
+  ! func(i,j,k) 只访问 tmp_x_line(fi-2:fi+3)
+  fi_min = 2*(imino + lbc(1) - 1) - 1 - lbf(1) + 1
+  fi_max = 2*(imaxo + lbc(1) - 1) - 1 - lbf(1) + 1
+  fj_min = 2*(jmino + lbc(2) - 1) - 1 - lbf(2) + 1
+  fj_max = 2*(jmaxo + lbc(2) - 1) - 1 - lbf(2) + 1
+  fk_min = 2*(kmino + lbc(3) - 1) - 1 - lbf(3) + 1
+  fk_max = 2*(kmaxo + lbc(3) - 1) - 1 - lbf(3) + 1
+  ii_lo = fi_min - 2
+  ii_hi = fi_max + 3
+  jj_lo = fj_min - 2
+  jj_hi = fj_max + 3
+  kk_lo = fk_min - 2
+  kk_hi = fk_max + 3
+  if(ii_lo < -1 .or. ii_hi > extf(1) .or. &
+     jj_lo < -1 .or. jj_hi > extf(2) .or. &
+     kk_lo < -1 .or. kk_hi > extf(3))then
+      write(*,*)"restrict3: invalid stencil window"
+      write(*,*)"ii=",ii_lo,ii_hi," jj=",jj_lo,jj_hi," kk=",kk_lo,kk_hi
+      write(*,*)"extf=",extf
+      stop
+  endif
+  need_full_symmetry = (ii_lo < 1) .or. (jj_lo < 1) .or. (kk_lo < 1)
+  if(need_full_symmetry)then
+      call symmetry_bd(2,extf,funf,funff,SoA)
+  else
+      funff(ii_lo:ii_hi,jj_lo:jj_hi,kk_lo:kk_hi) = funf(ii_lo:ii_hi,jj_lo:jj_hi,kk_lo:kk_hi)
+  endif
 
 !~~~~~~> restriction start...
+do k = kmino, kmaxo
+    fk = 2*(k + lbc(3) - 1) - 1 - lbf(3) + 1
+
+    do j = jmino, jmaxo
+        fj = 2*(j + lbc(2) - 1) - 1 - lbf(2) + 1
+        
+        ! 优化点 1: 显式展开 Z 方向计算，减少循环开销
+        ! 确保 ii 循环是最内层且连续访问
+        !DIR$ VECTOR ALWAYS
+        do ii = ii_lo, ii_hi
+            ! 预计算当前 j 对应的 6 行在 Z 方向的压缩结果
+            ! 这里直接硬编码 jj 的偏移，彻底消除一层循环
+            tmp_xz_plane(ii, 1) = C1*(funff(ii,fj-2,fk-2)+funff(ii,fj-2,fk+3)) + &
+                                  C2*(funff(ii,fj-2,fk-1)+funff(ii,fj-2,fk+2)) + &
+                                  C3*(funff(ii,fj-2,fk  )+funff(ii,fj-2,fk+1))
+            tmp_xz_plane(ii, 2) = C1*(funff(ii,fj-1,fk-2)+funff(ii,fj-1,fk+3)) + &
+                                  C2*(funff(ii,fj-1,fk-1)+funff(ii,fj-1,fk+2)) + &
+                                  C3*(funff(ii,fj-1,fk  )+funff(ii,fj-1,fk+1))
+            tmp_xz_plane(ii, 3) = C1*(funff(ii,fj  ,fk-2)+funff(ii,fj  ,fk+3)) + &
+                                  C2*(funff(ii,fj  ,fk-1)+funff(ii,fj  ,fk+2)) + &
+                                  C3*(funff(ii,fj  ,fk  )+funff(ii,fj  ,fk+1))
+            tmp_xz_plane(ii, 4) = C1*(funff(ii,fj+1,fk-2)+funff(ii,fj+1,fk+3)) + &
+                                  C2*(funff(ii,fj+1,fk-1)+funff(ii,fj+1,fk+2)) + &
+                                  C3*(funff(ii,fj+1,fk  )+funff(ii,fj+1,fk+1))
+            tmp_xz_plane(ii, 5) = C1*(funff(ii,fj+2,fk-2)+funff(ii,fj+2,fk+3)) + &
+                                  C2*(funff(ii,fj+2,fk-1)+funff(ii,fj+2,fk+2)) + &
+                                  C3*(funff(ii,fj+2,fk  )+funff(ii,fj+2,fk+1))
+            tmp_xz_plane(ii, 6) = C1*(funff(ii,fj+3,fk-2)+funff(ii,fj+3,fk+3)) + &
+                                  C2*(funff(ii,fj+3,fk-1)+funff(ii,fj+3,fk+2)) + &
+                                  C3*(funff(ii,fj+3,fk  )+funff(ii,fj+3,fk+1))
+        end do
+
+        ! 优化点 2: 同样向量化 Y 方向压缩
+        !DIR$ VECTOR ALWAYS
+        do ii = ii_lo, ii_hi
+            tmp_x_line(ii) = C1*(tmp_xz_plane(ii, 1) + tmp_xz_plane(ii, 6)) + &
+                            C2*(tmp_xz_plane(ii, 2) + tmp_xz_plane(ii, 5)) + &
+                            C3*(tmp_xz_plane(ii, 3) + tmp_xz_plane(ii, 4))
+        end do
+
+        ! 优化点 3: 最终写入，利用已经缓存在 tmp_x_line 的数据
+        do i = imino, imaxo
+            fi = 2*(i + lbc(1) - 1) - 1 - lbf(1) + 1
+            func(i, j, k) = C1*(tmp_x_line(fi-2) + tmp_x_line(fi+3)) + &
+                            C2*(tmp_x_line(fi-1) + tmp_x_line(fi+2)) + &
+                            C3*(tmp_x_line(fi  ) + tmp_x_line(fi+1))
+        end do
+    end do
+end do
+#if 0
   do k = kmino,kmaxo
    do j = jmino,jmaxo
     do i = imino,imaxo
@@ -2462,7 +2558,7 @@
     enddo
    enddo
   enddo
-  
+#endif
   return
 
   end subroutine restrict3

AMSS_NCKU_source/rungekutta4_rout_c.C

@@ -0,0 +1,212 @@
+#include "rungekutta4_rout.h"
+#include <cstdio>
+#include <cstdlib>
+#include <cstddef>
+#include <complex>
+#include <immintrin.h>
+
+namespace {
+
+inline void rk4_stage0(std::size_t n,
+                       const double *__restrict f0,
+                       const double *__restrict frhs,
+                       double *__restrict f1,
+                       double c) {
+    std::size_t i = 0;
+#if defined(__AVX512F__)
+    const __m512d vc = _mm512_set1_pd(c);
+    for (; i + 7 < n; i += 8) {
+        const __m512d v0 = _mm512_loadu_pd(f0 + i);
+        const __m512d vr = _mm512_loadu_pd(frhs + i);
+        _mm512_storeu_pd(f1 + i, _mm512_fmadd_pd(vc, vr, v0));
+    }
+#elif defined(__AVX2__)
+    const __m256d vc = _mm256_set1_pd(c);
+    for (; i + 3 < n; i += 4) {
+        const __m256d v0 = _mm256_loadu_pd(f0 + i);
+        const __m256d vr = _mm256_loadu_pd(frhs + i);
+        _mm256_storeu_pd(f1 + i, _mm256_fmadd_pd(vc, vr, v0));
+    }
+#endif
+#pragma ivdep
+    for (; i < n; ++i) {
+        f1[i] = f0[i] + c * frhs[i];
+    }
+}
+
+inline void rk4_rhs_accum(std::size_t n,
+                          const double *__restrict f1,
+                          double *__restrict frhs) {
+    std::size_t i = 0;
+#if defined(__AVX512F__)
+    const __m512d v2 = _mm512_set1_pd(2.0);
+    for (; i + 7 < n; i += 8) {
+        const __m512d v1 = _mm512_loadu_pd(f1 + i);
+        const __m512d vrhs = _mm512_loadu_pd(frhs + i);
+        _mm512_storeu_pd(frhs + i, _mm512_fmadd_pd(v2, v1, vrhs));
+    }
+#elif defined(__AVX2__)
+    const __m256d v2 = _mm256_set1_pd(2.0);
+    for (; i + 3 < n; i += 4) {
+        const __m256d v1 = _mm256_loadu_pd(f1 + i);
+        const __m256d vrhs = _mm256_loadu_pd(frhs + i);
+        _mm256_storeu_pd(frhs + i, _mm256_fmadd_pd(v2, v1, vrhs));
+    }
+#endif
+#pragma ivdep
+    for (; i < n; ++i) {
+        frhs[i] = frhs[i] + 2.0 * f1[i];
+    }
+}
+
+inline void rk4_f1_from_f0_f1(std::size_t n,
+                              const double *__restrict f0,
+                              double *__restrict f1,
+                              double c) {
+    std::size_t i = 0;
+#if defined(__AVX512F__)
+    const __m512d vc = _mm512_set1_pd(c);
+    for (; i + 7 < n; i += 8) {
+        const __m512d v0 = _mm512_loadu_pd(f0 + i);
+        const __m512d v1 = _mm512_loadu_pd(f1 + i);
+        _mm512_storeu_pd(f1 + i, _mm512_fmadd_pd(vc, v1, v0));
+    }
+#elif defined(__AVX2__)
+    const __m256d vc = _mm256_set1_pd(c);
+    for (; i + 3 < n; i += 4) {
+        const __m256d v0 = _mm256_loadu_pd(f0 + i);
+        const __m256d v1 = _mm256_loadu_pd(f1 + i);
+        _mm256_storeu_pd(f1 + i, _mm256_fmadd_pd(vc, v1, v0));
+    }
+#endif
+#pragma ivdep
+    for (; i < n; ++i) {
+        f1[i] = f0[i] + c * f1[i];
+    }
+}
+
+inline void rk4_stage3(std::size_t n,
+                       const double *__restrict f0,
+                       double *__restrict f1,
+                       const double *__restrict frhs,
+                       double c) {
+    std::size_t i = 0;
+#if defined(__AVX512F__)
+    const __m512d vc = _mm512_set1_pd(c);
+    for (; i + 7 < n; i += 8) {
+        const __m512d v0 = _mm512_loadu_pd(f0 + i);
+        const __m512d v1 = _mm512_loadu_pd(f1 + i);
+        const __m512d vr = _mm512_loadu_pd(frhs + i);
+        _mm512_storeu_pd(f1 + i, _mm512_fmadd_pd(vc, _mm512_add_pd(v1, vr), v0));
+    }
+#elif defined(__AVX2__)
+    const __m256d vc = _mm256_set1_pd(c);
+    for (; i + 3 < n; i += 4) {
+        const __m256d v0 = _mm256_loadu_pd(f0 + i);
+        const __m256d v1 = _mm256_loadu_pd(f1 + i);
+        const __m256d vr = _mm256_loadu_pd(frhs + i);
+        _mm256_storeu_pd(f1 + i, _mm256_fmadd_pd(vc, _mm256_add_pd(v1, vr), v0));
+    }
+#endif
+#pragma ivdep
+    for (; i < n; ++i) {
+        f1[i] = f0[i] + c * (f1[i] + frhs[i]);
+    }
+}
+
+} // namespace
+
+extern "C" {
+
+void f_rungekutta4_scalar(double &dT, double &f0, double &f1, double &f_rhs, int &RK4) {
+    constexpr double F1o6 = 1.0 / 6.0;
+    constexpr double HLF = 0.5;
+    constexpr double TWO = 2.0;
+
+    switch (RK4) {
+    case 0:
+        f1 = f0 + HLF * dT * f_rhs;
+        break;
+    case 1:
+        f_rhs = f_rhs + TWO * f1;
+        f1 = f0 + HLF * dT * f1;
+        break;
+    case 2:
+        f_rhs = f_rhs + TWO * f1;
+        f1 = f0 + dT * f1;
+        break;
+    case 3:
+        f1 = f0 + F1o6 * dT * (f1 + f_rhs);
+        break;
+    default:
+        std::fprintf(stderr, "rungekutta4_scalar_c: invalid RK4 stage %d\n", RK4);
+        std::abort();
+    }
+}
+
+void rungekutta4_cplxscalar_(double &dT,
+                             std::complex<double> &f0,
+                             std::complex<double> &f1,
+                             std::complex<double> &f_rhs,
+                             int &RK4) {
+    constexpr double F1o6 = 1.0 / 6.0;
+    constexpr double HLF = 0.5;
+    constexpr double TWO = 2.0;
+
+    switch (RK4) {
+    case 0:
+        f1 = f0 + HLF * dT * f_rhs;
+        break;
+    case 1:
+        f_rhs = f_rhs + TWO * f1;
+        f1 = f0 + HLF * dT * f1;
+        break;
+    case 2:
+        f_rhs = f_rhs + TWO * f1;
+        f1 = f0 + dT * f1;
+        break;
+    case 3:
+        f1 = f0 + F1o6 * dT * (f1 + f_rhs);
+        break;
+    default:
+        std::fprintf(stderr, "rungekutta4_cplxscalar_c: invalid RK4 stage %d\n", RK4);
+        std::abort();
+    }
+}
+
+int f_rungekutta4_rout(int *ex, double &dT,
+                       double *f0, double *f1, double *f_rhs,
+                       int &RK4) {
+    const std::size_t n = static_cast<std::size_t>(ex[0]) *
+                          static_cast<std::size_t>(ex[1]) *
+                          static_cast<std::size_t>(ex[2]);
+    const double *const __restrict f0r = f0;
+    double *const __restrict f1r = f1;
+    double *const __restrict frhs = f_rhs;
+
+    if (__builtin_expect(static_cast<unsigned>(RK4) > 3u, 0)) {
+        std::fprintf(stderr, "rungekutta4_rout_c: invalid RK4 stage %d\n", RK4);
+        std::abort();
+    }
+
+    switch (RK4) {
+    case 0:
+        rk4_stage0(n, f0r, frhs, f1r, 0.5 * dT);
+        break;
+    case 1:
+        rk4_rhs_accum(n, f1r, frhs);
+        rk4_f1_from_f0_f1(n, f0r, f1r, 0.5 * dT);
+        break;
+    case 2:
+        rk4_rhs_accum(n, f1r, frhs);
+        rk4_f1_from_f0_f1(n, f0r, f1r, dT);
+        break;
+    default:
+        rk4_stage3(n, f0r, f1r, frhs, (1.0 / 6.0) * dT);
+        break;
+    }
+
+    return 0;
+}
+
+} // extern "C"

AMSS_NCKU_source/share_func.h

@@ -0,0 +1,246 @@
+#ifndef SHARE_FUNC_H
+#define SHARE_FUNC_H
+
+#include <stdlib.h>
+#include <stddef.h>
+#include <math.h>
+#include <stdio.h>
+#include <string.h>
+/* 主网格：0-based -> 1D */
+static inline size_t idx_ex(int i0, int j0, int k0, const int ex[3]) {
+    const int ex1 = ex[0], ex2 = ex[1];
+    return (size_t)i0 + (size_t)j0 * (size_t)ex1 + (size_t)k0 * (size_t)ex1 * (size_t)ex2;
+}
+
+/*
+ * fh 对应 Fortran: fh(-1:ex1, -1:ex2, -1:ex3)
+ * ord=2 => shift=1
+ * iF/jF/kF 为 Fortran 索引（可为 -1,0,1..ex）
+ */
+static inline size_t idx_fh_F_ord2(int iF, int jF, int kF, const int ex[3]) {
+    const int shift = 1;
+    const int nx = ex[0] + 2;      // ex1 + ord
+    const int ny = ex[1] + 2;
+
+    const int ii = iF + shift;     // 0..ex1+1
+    const int jj = jF + shift;     // 0..ex2+1
+    const int kk = kF + shift;     // 0..ex3+1
+
+    return (size_t)ii + (size_t)jj * (size_t)nx + (size_t)kk * (size_t)nx * (size_t)ny;
+}
+
+/*
+ * fh 对应 Fortran: fh(-2:ex1, -2:ex2, -2:ex3)
+ * ord=3 => shift=2
+ * iF/jF/kF 是 Fortran 索引（可为负）
+ */
+static inline size_t idx_fh_F(int iF, int jF, int kF, const int ex[3]) {
+    const int shift = 2;                 // ord=3 -> -2..ex
+    const int nx = ex[0] + 3;            // ex1 + ord
+    const int ny = ex[1] + 3;
+
+    const int ii = iF + shift;           // 0..ex1+2
+    const int jj = jF + shift;           // 0..ex2+2
+    const int kk = kF + shift;           // 0..ex3+2
+
+    return (size_t)ii + (size_t)jj * (size_t)nx + (size_t)kk * (size_t)nx * (size_t)ny;
+}
+
+/*
+ * func:  (1..extc1, 1..extc2, 1..extc3)   1-based in Fortran
+ * funcc: (-ord+1..extc1, -ord+1..extc2, -ord+1..extc3) in Fortran
+ *
+ * C 里我们把：
+ *   func  视为 0-based: i0=0..extc1-1, j0=0..extc2-1, k0=0..extc3-1
+ *   funcc 用“平移下标”存为一维数组：
+ *     iF in [-ord+1..extc1]  -> ii = iF + (ord-1)  in [0..extc1+ord-1]
+ *     总长度 nx = extc1 + ord
+ *     同理 ny = extc2 + ord, nz = extc3 + ord
+ */
+
+static inline size_t idx_func0(int i0, int j0, int k0, const int extc[3]) {
+    const int nx = extc[0], ny = extc[1];
+    return (size_t)i0 + (size_t)j0 * (size_t)nx + (size_t)k0 * (size_t)nx * (size_t)ny;
+}
+
+static inline size_t idx_funcc_F(int iF, int jF, int kF, int ord, const int extc[3]) {
+    const int shift = ord - 1;          // iF = -shift .. extc1
+    const int nx = extc[0] + ord;       // [-shift..extc1] 共 extc1+ord 个
+    const int ny = extc[1] + ord;
+
+    const int ii = iF + shift;          // 0..extc1+shift
+    const int jj = jF + shift;          // 0..extc2+shift
+    const int kk = kF + shift;          // 0..extc3+shift
+
+    return (size_t)ii + (size_t)jj * (size_t)nx + (size_t)kk * (size_t)nx * (size_t)ny;
+}
+
+/*
+ * 等价于 Fortran:
+ * funcc(1:extc1,1:extc2,1:extc3)=func
+ * do i=0,ord-1
+ *   funcc(-i,1:extc2,1:extc3) = funcc(i+1,1:extc2,1:extc3)*SoA(1)
+ * enddo
+ * do i=0,ord-1
+ *   funcc(:,-i,1:extc3) = funcc(:,i+1,1:extc3)*SoA(2)
+ * enddo
+ * do i=0,ord-1
+ *   funcc(:,:,-i) = funcc(:,:,i+1)*SoA(3)
+ * enddo
+ */
+static inline void symmetry_bd_impl(int ord,
+                 int shift,
+                 const int extc[3],
+                 const double *__restrict func,
+                 double *__restrict funcc,
+                 const double SoA[3])
+{
+    const int extc1 = extc[0], extc2 = extc[1], extc3 = extc[2];
+    const int nx = extc1 + ord;
+    const int ny = extc2 + ord;
+
+    const size_t snx = (size_t)nx;
+    const size_t splane = (size_t)nx * (size_t)ny;
+    const size_t interior_i = (size_t)shift + 1u;          /* iF = 1 */
+    const size_t interior_j = ((size_t)shift + 1u) * snx;  /* jF = 1 */
+    const size_t interior_k = ((size_t)shift + 1u) * splane; /* kF = 1 */
+    const size_t interior0 = interior_k + interior_j + interior_i;
+
+    /* 1) funcc(1:extc1,1:extc2,1:extc3) = func */
+    for (int k0 = 0; k0 < extc3; ++k0) {
+        const double *src_k = func + (size_t)k0 * (size_t)extc2 * (size_t)extc1;
+        const size_t dst_k0 = interior0 + (size_t)k0 * splane;
+        for (int j0 = 0; j0 < extc2; ++j0) {
+            const double *src = src_k + (size_t)j0 * (size_t)extc1;
+            double *dst = funcc + dst_k0 + (size_t)j0 * snx;
+            memcpy(dst, src, (size_t)extc1 * sizeof(double));
+        }
+    }
+
+    /* 2) funcc(-i,1:extc2,1:extc3) = funcc(i+1,1:extc2,1:extc3)*SoA(1) */
+    const double s1 = SoA[0];
+    if (s1 == 1.0) {
+        for (int ii = 0; ii < ord; ++ii) {
+            const size_t dst_i = (size_t)(shift - ii);
+            const size_t src_i = (size_t)(shift + ii + 1);
+            for (int k0 = 0; k0 < extc3; ++k0) {
+                const size_t kbase = interior_k + (size_t)k0 * splane + interior_j;
+                for (int j0 = 0; j0 < extc2; ++j0) {
+                    const size_t off = kbase + (size_t)j0 * snx;
+                    funcc[off + dst_i] = funcc[off + src_i];
+                }
+            }
+        }
+    } else if (s1 == -1.0) {
+        for (int ii = 0; ii < ord; ++ii) {
+            const size_t dst_i = (size_t)(shift - ii);
+            const size_t src_i = (size_t)(shift + ii + 1);
+            for (int k0 = 0; k0 < extc3; ++k0) {
+                const size_t kbase = interior_k + (size_t)k0 * splane + interior_j;
+                for (int j0 = 0; j0 < extc2; ++j0) {
+                    const size_t off = kbase + (size_t)j0 * snx;
+                    funcc[off + dst_i] = -funcc[off + src_i];
+                }
+            }
+        }
+    } else {
+        for (int ii = 0; ii < ord; ++ii) {
+            const size_t dst_i = (size_t)(shift - ii);
+            const size_t src_i = (size_t)(shift + ii + 1);
+            for (int k0 = 0; k0 < extc3; ++k0) {
+                const size_t kbase = interior_k + (size_t)k0 * splane + interior_j;
+                for (int j0 = 0; j0 < extc2; ++j0) {
+                    const size_t off = kbase + (size_t)j0 * snx;
+                    funcc[off + dst_i] = funcc[off + src_i] * s1;
+                }
+            }
+        }
+    }
+
+    /* 3) funcc(:,-j,1:extc3) = funcc(:,j+1,1:extc3)*SoA(2) */
+    const double s2 = SoA[1];
+    if (s2 == 1.0) {
+        for (int jj = 0; jj < ord; ++jj) {
+            const size_t dst_j = (size_t)(shift - jj) * snx;
+            const size_t src_j = (size_t)(shift + jj + 1) * snx;
+            for (int k0 = 0; k0 < extc3; ++k0) {
+                const size_t kbase = interior_k + (size_t)k0 * splane;
+                double *dst = funcc + kbase + dst_j;
+                const double *src = funcc + kbase + src_j;
+                for (int i = 0; i < nx; ++i) dst[i] = src[i];
+            }
+        }
+    } else if (s2 == -1.0) {
+        for (int jj = 0; jj < ord; ++jj) {
+            const size_t dst_j = (size_t)(shift - jj) * snx;
+            const size_t src_j = (size_t)(shift + jj + 1) * snx;
+            for (int k0 = 0; k0 < extc3; ++k0) {
+                const size_t kbase = interior_k + (size_t)k0 * splane;
+                double *dst = funcc + kbase + dst_j;
+                const double *src = funcc + kbase + src_j;
+                for (int i = 0; i < nx; ++i) dst[i] = -src[i];
+            }
+        }
+    } else {
+        for (int jj = 0; jj < ord; ++jj) {
+            const size_t dst_j = (size_t)(shift - jj) * snx;
+            const size_t src_j = (size_t)(shift + jj + 1) * snx;
+            for (int k0 = 0; k0 < extc3; ++k0) {
+                const size_t kbase = interior_k + (size_t)k0 * splane;
+                double *dst = funcc + kbase + dst_j;
+                const double *src = funcc + kbase + src_j;
+                for (int i = 0; i < nx; ++i) dst[i] = src[i] * s2;
+            }
+        }
+    }
+
+    /* 4) funcc(:,:,-k) = funcc(:,:,k+1)*SoA(3) */
+    const double s3 = SoA[2];
+    if (s3 == 1.0) {
+        for (int kk = 0; kk < ord; ++kk) {
+            const size_t dst_k = (size_t)(shift - kk) * splane;
+            const size_t src_k = (size_t)(shift + kk + 1) * splane;
+            double *dst = funcc + dst_k;
+            const double *src = funcc + src_k;
+            for (size_t p = 0; p < splane; ++p) dst[p] = src[p];
+        }
+    } else if (s3 == -1.0) {
+        for (int kk = 0; kk < ord; ++kk) {
+            const size_t dst_k = (size_t)(shift - kk) * splane;
+            const size_t src_k = (size_t)(shift + kk + 1) * splane;
+            double *dst = funcc + dst_k;
+            const double *src = funcc + src_k;
+            for (size_t p = 0; p < splane; ++p) dst[p] = -src[p];
+        }
+    } else {
+        for (int kk = 0; kk < ord; ++kk) {
+            const size_t dst_k = (size_t)(shift - kk) * splane;
+            const size_t src_k = (size_t)(shift + kk + 1) * splane;
+            double *dst = funcc + dst_k;
+            const double *src = funcc + src_k;
+            for (size_t p = 0; p < splane; ++p) dst[p] = src[p] * s3;
+        }
+    }
+}
+
+static inline void symmetry_bd(int ord,
+                 const int extc[3],
+                 const double *func,
+                 double *funcc,
+                 const double SoA[3])
+{
+    if (ord <= 0) return;
+
+    /* Fast paths used by current C kernels: ord=2 (derivs), ord=3 (lopsided/KO). */
+    if (ord == 2) {
+        symmetry_bd_impl(2, 1, extc, func, funcc, SoA);
+        return;
+    }
+    if (ord == 3) {
+        symmetry_bd_impl(3, 2, extc, func, funcc, SoA);
+        return;
+    }
+
+    symmetry_bd_impl(ord, ord - 1, extc, func, funcc, SoA);
+}
+#endif

AMSS_NCKU_source/surface_integral.h

@@ -27,24 +27,29 @@ using namespace std;
 class surface_integral
 {
 
-private:
-	int Symmetry, factor;
-	int N_theta, N_phi; // Number of points in Theta & Phi directions
-	double dphi, dcostheta;
-	double *arcostheta, *wtcostheta;
-	int n_tot; // size of arrays
+private:
+	int Symmetry, factor;
+	int N_theta, N_phi; // Number of points in Theta & Phi directions
+	double dphi, dcostheta;
+	double *arcostheta, *wtcostheta;
+	int n_tot; // size of arrays
+
+	double *nx_g, *ny_g, *nz_g; // global list of unit normals
+	int myrank, cpusize;
+	int wave_cache_spinw, wave_cache_maxl, wave_cache_modes;
+	double *wave_theta_pos, *wave_theta_neg;
+	double *wave_phi_cos, *wave_phi_sin;
+	void clear_wave_cache();
+	void build_wave_cache(int spinw, int maxl);
+
+public:
+	surface_integral(int iSymmetry);
+	~surface_integral();
 
-	double *nx_g, *ny_g, *nz_g; // global list of unit normals
-	int myrank, cpusize;
-
-public:
-	surface_integral(int iSymmetry);
-	~surface_integral();
-
-	void surf_Wave(double rex, int lev, cgh *GH, var *Rpsi4, var *Ipsi4,
-				   int spinw, int maxl, int NN, double *RP, double *IP,
-				   monitor *Monitor); // NN is the length of RP and IP
-									  // this routine can only deal with the symmetry of Psi4
+	void surf_Wave(double rex, int lev, cgh *GH, var *Rpsi4, var *Ipsi4,
+				   int spinw, int maxl, int NN, double *RP, double *IP,
+				   monitor *Monitor); // NN is the length of RP and IP
+									  // this routine can only deal with the symmetry of Psi4
 	void surf_Wave(double rex, int lev, ShellPatch *GH, var *Rpsi4, var *Ipsi4,
 				   int spinw, int maxl, int NN, double *RP, double *IP,
 				   monitor *Monitor);
@@ -77,21 +82,37 @@ public:
 								 double &, double &, double &, double &, double &, double &, double &,
 								 double &, double &, double &, double &, double &, double &,
 								 double &, double &)); // NN is the length of RP and IP
-	void surf_MassPAng(double rex, int lev, cgh *GH, var *chi, var *trK,
-					   var *gxx, var *gxy, var *gxz, var *gyy, var *gyz, var *gzz,
-					   var *Axx, var *Axy, var *Axz, var *Ayy, var *Ayz, var *Azz,
-					   var *Gmx, var *Gmy, var *Gmz,
-					   var *Sfx_rhs, var *Sfy_rhs, var *Sfz_rhs,
-					   double *Rout, monitor *Monitor);
-	void surf_MassPAng(double rex, int lev, ShellPatch *GH, var *chi, var *trK,
-					   var *gxx, var *gxy, var *gxz, var *gyy, var *gyz, var *gzz,
-					   var *Axx, var *Axy, var *Axz, var *Ayy, var *Ayz, var *Azz,
-					   var *Gmx, var *Gmy, var *Gmz,
-					   var *Sfx_rhs, var *Sfy_rhs, var *Sfz_rhs,
-					   double *Rout, monitor *Monitor);
-	void surf_Wave(double rex, cgh *GH, ShellPatch *SH,
-				   var *chi, var *trK,
-				   var *gxx, var *gxy, var *gxz, var *gyy, var *gyz, var *gzz,
+	void surf_MassPAng(double rex, int lev, cgh *GH, var *chi, var *trK,
+					   var *gxx, var *gxy, var *gxz, var *gyy, var *gyz, var *gzz,
+					   var *Axx, var *Axy, var *Axz, var *Ayy, var *Ayz, var *Azz,
+					   var *Gmx, var *Gmy, var *Gmz,
+					   var *Sfx_rhs, var *Sfy_rhs, var *Sfz_rhs,
+					   double *Rout, monitor *Monitor, bool refresh_mass_fields = true);
+	void surf_MassPAng(double rex, int lev, ShellPatch *GH, var *chi, var *trK,
+					   var *gxx, var *gxy, var *gxz, var *gyy, var *gyz, var *gzz,
+					   var *Axx, var *Axy, var *Axz, var *Ayy, var *Ayz, var *Azz,
+					   var *Gmx, var *Gmy, var *Gmz,
+					   var *Sfx_rhs, var *Sfy_rhs, var *Sfz_rhs,
+					   double *Rout, monitor *Monitor, bool refresh_mass_fields = true);
+	void surf_WaveMassPAng(double rex, int lev, cgh *GH,
+					   var *Rpsi4, var *Ipsi4, int spinw, int maxl, int NN, double *RP, double *IP,
+					   var *chi, var *trK,
+					   var *gxx, var *gxy, var *gxz, var *gyy, var *gyz, var *gzz,
+					   var *Axx, var *Axy, var *Axz, var *Ayy, var *Ayz, var *Azz,
+					   var *Gmx, var *Gmy, var *Gmz,
+					   var *Sfx_rhs, var *Sfy_rhs, var *Sfz_rhs,
+					   double *Rout, monitor *Monitor, bool refresh_mass_fields = true);
+	void surf_WaveMassPAng(double rex, int lev, ShellPatch *GH,
+					   var *Rpsi4, var *Ipsi4, int spinw, int maxl, int NN, double *RP, double *IP,
+					   var *chi, var *trK,
+					   var *gxx, var *gxy, var *gxz, var *gyy, var *gyz, var *gzz,
+					   var *Axx, var *Axy, var *Axz, var *Ayy, var *Ayz, var *Azz,
+					   var *Gmx, var *Gmy, var *Gmz,
+					   var *Sfx_rhs, var *Sfy_rhs, var *Sfz_rhs,
+					   double *Rout, monitor *Monitor, bool refresh_mass_fields = true);
+	void surf_Wave(double rex, cgh *GH, ShellPatch *SH,
+				   var *chi, var *trK,
+				   var *gxx, var *gxy, var *gxz, var *gyy, var *gyz, var *gzz,
 				   var *Axx, var *Axy, var *Axz, var *Ayy, var *Ayz, var *Azz,
 				   var *chix, var *chiy, var *chiz,
 				   var *trKx, var *trKy, var *trKz,
@@ -110,12 +131,12 @@ public:
 	bool SR_Interp_Points(MyList<var> *VarList, cgh *GH, ShellPatch *SH,
 						  int NN, double **XX, double *Shellf);
 
-	void surf_MassPAng(double rex, int lev, cgh *GH, var *chi, var *trK,
-					   var *gxx, var *gxy, var *gxz, var *gyy, var *gyz, var *gzz,
-					   var *Axx, var *Axy, var *Axz, var *Ayy, var *Ayz, var *Azz,
-					   var *Gmx, var *Gmy, var *Gmz,
-					   var *Sfx_rhs, var *Sfy_rhs, var *Sfz_rhs, // temparay memory for mass^i
-					   double *Rout, monitor *Monitor, MPI_Comm Comm_here);
+	void surf_MassPAng(double rex, int lev, cgh *GH, var *chi, var *trK,
+					   var *gxx, var *gxy, var *gxz, var *gyy, var *gyz, var *gzz,
+					   var *Axx, var *Axy, var *Axz, var *Ayy, var *Ayz, var *Azz,
+					   var *Gmx, var *Gmy, var *Gmz,
+					   var *Sfx_rhs, var *Sfy_rhs, var *Sfz_rhs, // temparay memory for mass^i
+					   double *Rout, monitor *Monitor, MPI_Comm Comm_here, bool refresh_mass_fields = true);
 	void surf_Wave(double rex, int lev, cgh *GH, var *Rpsi4, var *Ipsi4,
 				   int spinw, int maxl, int NN, double *RP, double *IP,
 				   monitor *Monitor, MPI_Comm Comm_here);

AMSS_NCKU_source/tool.h

@@ -0,0 +1,33 @@
+#include "share_func.h"
+void fdderivs(const int ex[3],
+              const double *f,
+              double *fxx, double *fxy, double *fxz,
+              double *fyy, double *fyz, double *fzz,
+              const double *X, const double *Y, const double *Z,
+              double SYM1, double SYM2, double SYM3,
+              int Symmetry, int onoff);
+
+void fderivs(const int ex[3],
+             const double *f,
+             double *fx, double *fy, double *fz,
+             const double *X, const double *Y, const double *Z,
+             double SYM1, double SYM2, double SYM3,
+             int Symmetry, int onoff);
+
+void kodis(const int ex[3],
+           const double *X, const double *Y, const double *Z,
+           const double *f, double *f_rhs,
+           const double SoA[3],
+           int Symmetry, double eps);
+
+void lopsided(const int ex[3],
+              const double *X, const double *Y, const double *Z,
+              const double *f, double *f_rhs,
+              const double *Sfx, const double *Sfy, const double *Sfz,
+              int Symmetry, const double SoA[3]);
+
+void lopsided_kodis(const int ex[3],
+                    const double *X, const double *Y, const double *Z,
+                    const double *f, double *f_rhs,
+                    const double *Sfx, const double *Sfy, const double *Sfz,
+                    int Symmetry, const double SoA[3], double eps);

AMSS_NCKU_source/z4c_rhs_c.C

@@ -0,0 +1,725 @@
+#include "macrodef.h"
+#include "bssn_rhs.h"
+#include "fmisc.h"
+#include "ricci_gamma.h"
+#include "share_func.h"
+#include "tool.h"
+#include <vector>
+
+#ifdef fortran1
+#define f_constraint_bssn constraint_bssn
+#define f_z4c_rhs_point z4c_rhs_point
+#endif
+#ifdef fortran2
+#define f_constraint_bssn CONSTRAINT_BSSN
+#define f_z4c_rhs_point Z4C_RHS_POINT
+#endif
+#ifdef fortran3
+#define f_constraint_bssn constraint_bssn_
+#define f_z4c_rhs_point z4c_rhs_point_
+#endif
+
+extern "C" void f_constraint_bssn(int *, double *, double *, double *,
+                                  double *, double *,
+                                  double *, double *, double *, double *, double *, double *,
+                                  double *, double *, double *, double *, double *, double *,
+                                  double *, double *, double *,
+                                  double *, double *, double *, double *, double *, double *, double *, double *, double *, double *,
+                                  double *, double *, double *, double *, double *, double *,
+                                  double *, double *, double *, double *, double *, double *,
+                                  double *, double *, double *, double *, double *, double *,
+                                  double *, double *, double *, double *, double *, double *, double *, double *,
+                                  double *, double *, double *,
+                                  int &);
+
+extern "C" void f_z4c_rhs_point(
+    double &A11,
+    double &A12,
+    double &A13,
+    double &A22,
+    double &A23,
+    double &A33,
+    double &alpha,
+    double &B1,
+    double &B2,
+    double &B3,
+    double &beta1,
+    double &beta2,
+    double &beta3,
+    double &chi,
+    double &chiDivFloor,
+    double &da1,
+    double &dA111,
+    double &dA112,
+    double &dA113,
+    double &dA122,
+    double &dA123,
+    double &dA133,
+    double &da2,
+    double &dA211,
+    double &dA212,
+    double &dA213,
+    double &dA222,
+    double &dA223,
+    double &dA233,
+    double &da3,
+    double &dA311,
+    double &dA312,
+    double &dA313,
+    double &dA322,
+    double &dA323,
+    double &dA333,
+    double &db11,
+    double &dB11,
+    double &db12,
+    double &dB12,
+    double &db13,
+    double &dB13,
+    double &db21,
+    double &dB21,
+    double &db22,
+    double &dB22,
+    double &db23,
+    double &dB23,
+    double &db31,
+    double &dB31,
+    double &db32,
+    double &dB32,
+    double &db33,
+    double &dB33,
+    double &dchi1,
+    double &dchi2,
+    double &dchi3,
+    double &dda11,
+    double &dda12,
+    double &dda13,
+    double &dda22,
+    double &dda23,
+    double &dda33,
+    double &ddb111,
+    double &ddb112,
+    double &ddb113,
+    double &ddb121,
+    double &ddb122,
+    double &ddb123,
+    double &ddb131,
+    double &ddb132,
+    double &ddb133,
+    double &ddb221,
+    double &ddb222,
+    double &ddb223,
+    double &ddb231,
+    double &ddb232,
+    double &ddb233,
+    double &ddb331,
+    double &ddb332,
+    double &ddb333,
+    double &ddchi11,
+    double &ddchi12,
+    double &ddchi13,
+    double &ddchi22,
+    double &ddchi23,
+    double &ddchi33,
+    double &deldelg1111,
+    double &deldelg1112,
+    double &deldelg1113,
+    double &deldelg1122,
+    double &deldelg1123,
+    double &deldelg1133,
+    double &deldelg1211,
+    double &deldelg1212,
+    double &deldelg1213,
+    double &deldelg1222,
+    double &deldelg1223,
+    double &deldelg1233,
+    double &deldelg1311,
+    double &deldelg1312,
+    double &deldelg1313,
+    double &deldelg1322,
+    double &deldelg1323,
+    double &deldelg1333,
+    double &deldelg2211,
+    double &deldelg2212,
+    double &deldelg2213,
+    double &deldelg2222,
+    double &deldelg2223,
+    double &deldelg2233,
+    double &deldelg2311,
+    double &deldelg2312,
+    double &deldelg2313,
+    double &deldelg2322,
+    double &deldelg2323,
+    double &deldelg2333,
+    double &deldelg3311,
+    double &deldelg3312,
+    double &deldelg3313,
+    double &deldelg3322,
+    double &deldelg3323,
+    double &deldelg3333,
+    double &delG11,
+    double &delg111,
+    double &delg112,
+    double &delg113,
+    double &delG12,
+    double &delg122,
+    double &delg123,
+    double &delG13,
+    double &delg133,
+    double &delG21,
+    double &delg211,
+    double &delg212,
+    double &delg213,
+    double &delG22,
+    double &delg222,
+    double &delg223,
+    double &delG23,
+    double &delg233,
+    double &delG31,
+    double &delg311,
+    double &delg312,
+    double &delg313,
+    double &delG32,
+    double &delg322,
+    double &delg323,
+    double &delG33,
+    double &delg333,
+    double &dKhat1,
+    double &dKhat2,
+    double &dKhat3,
+    double &dTheta1,
+    double &dTheta2,
+    double &dTheta3,
+    double &G1,
+    double &g11,
+    double &g12,
+    double &g13,
+    double &G2,
+    double &g22,
+    double &g23,
+    double &G3,
+    double &g33,
+    double &kappa1,
+    double &kappa2,
+    double &Khat,
+    double &rA11,
+    double &rA12,
+    double &rA13,
+    double &rA22,
+    double &rA23,
+    double &rA33,
+    double &rchi,
+    double &rG1,
+    double &rg11,
+    double &rg12,
+    double &rg13,
+    double &rG2,
+    double &rg22,
+    double &rg23,
+    double &rG3,
+    double &rg33,
+    double &rKhat,
+    double &rTheta,
+    double &Theta);
+
+static inline void z4c_contract_gamma(
+    const double gxx, const double gxy, const double gxz,
+    const double gyy, const double gyz, const double gzz,
+    const double gxxx, const double gxyx, const double gxzx,
+    const double gyyx, const double gyzx, const double gzzx,
+    const double gxxy, const double gxyy, const double gxzy,
+    const double gyyy, const double gyzy, const double gzzy,
+    const double gxxz, const double gxyz, const double gxzz,
+    const double gyyz, const double gyzz, const double gzzz,
+    double &Gamxa, double &Gamya, double &Gamza)
+{
+    double det = gxx * gyy * gzz + gxy * gyz * gxz + gxz * gxy * gyz -
+                 gxz * gyy * gxz - gxy * gxy * gzz - gxx * gyz * gyz;
+    const double gupxx = (gyy * gzz - gyz * gyz) / det;
+    const double gupxy = -(gxy * gzz - gyz * gxz) / det;
+    const double gupxz = (gxy * gyz - gyy * gxz) / det;
+    const double gupyy = (gxx * gzz - gxz * gxz) / det;
+    const double gupyz = -(gxx * gyz - gxy * gxz) / det;
+    const double gupzz = (gxx * gyy - gxy * gxy) / det;
+
+    const double Gamxxx = 0.5 * (gupxx * gxxx + gupxy * (2.0 * gxyx - gxxy) + gupxz * (2.0 * gxzx - gxxz));
+    const double Gamyxx = 0.5 * (gupxy * gxxx + gupyy * (2.0 * gxyx - gxxy) + gupyz * (2.0 * gxzx - gxxz));
+    const double Gamzxx = 0.5 * (gupxz * gxxx + gupyz * (2.0 * gxyx - gxxy) + gupzz * (2.0 * gxzx - gxxz));
+
+    const double Gamxyy = 0.5 * (gupxx * (2.0 * gxyy - gyyx) + gupxy * gyyy + gupxz * (2.0 * gyzy - gyyz));
+    const double Gamyyy = 0.5 * (gupxy * (2.0 * gxyy - gyyx) + gupyy * gyyy + gupyz * (2.0 * gyzy - gyyz));
+    const double Gamzyy = 0.5 * (gupxz * (2.0 * gxyy - gyyx) + gupyz * gyyy + gupzz * (2.0 * gyzy - gyyz));
+
+    const double Gamxzz = 0.5 * (gupxx * (2.0 * gxzz - gzzx) + gupxy * (2.0 * gyzz - gzzy) + gupxz * gzzz);
+    const double Gamyzz = 0.5 * (gupxy * (2.0 * gxzz - gzzx) + gupyy * (2.0 * gyzz - gzzy) + gupyz * gzzz);
+    const double Gamzzz = 0.5 * (gupxz * (2.0 * gxzz - gzzx) + gupyz * (2.0 * gyzz - gzzy) + gupzz * gzzz);
+
+    const double Gamxxy = 0.5 * (gupxx * gxxy + gupxy * gyyx + gupxz * (gxzy + gyzx - gxyz));
+    const double Gamyxy = 0.5 * (gupxy * gxxy + gupyy * gyyx + gupyz * (gxzy + gyzx - gxyz));
+    const double Gamzxy = 0.5 * (gupxz * gxxy + gupyz * gyyx + gupzz * (gxzy + gyzx - gxyz));
+
+    const double Gamxxz = 0.5 * (gupxx * gxxz + gupxy * (gxyz + gyzx - gxzy) + gupxz * gzzx);
+    const double Gamyxz = 0.5 * (gupxy * gxxz + gupyy * (gxyz + gyzx - gxzy) + gupyz * gzzx);
+    const double Gamzxz = 0.5 * (gupxz * gxxz + gupyz * (gxyz + gyzx - gxzy) + gupzz * gzzx);
+
+    const double Gamxyz = 0.5 * (gupxx * (gxyz + gxzy - gyzx) + gupxy * gyyz + gupxz * gzzy);
+    const double Gamyyz = 0.5 * (gupxy * (gxyz + gxzy - gyzx) + gupyy * gyyz + gupyz * gzzy);
+    const double Gamzyz = 0.5 * (gupxz * (gxyz + gxzy - gyzx) + gupyz * gyyz + gupzz * gzzy);
+
+    Gamxa = gupxx * Gamxxx + gupyy * Gamxyy + gupzz * Gamxzz +
+            2.0 * (gupxy * Gamxxy + gupxz * Gamxxz + gupyz * Gamxyz);
+    Gamya = gupxx * Gamyxx + gupyy * Gamyyy + gupzz * Gamyzz +
+            2.0 * (gupxy * Gamyxy + gupxz * Gamyxz + gupyz * Gamyyz);
+    Gamza = gupxx * Gamzxx + gupyy * Gamzyy + gupzz * Gamzzz +
+            2.0 * (gupxy * Gamzxy + gupxz * Gamzxz + gupyz * Gamzyz);
+}
+
+static int compute_rhs_z4c_cartesian(
+    int *ex, double &T, double *X, double *Y, double *Z,
+    double *chi_state, double *chi_constraints, double *trK,
+    double *dxx, double *gxy, double *gxz, double *dyy, double *gyz, double *dzz,
+    double *Axx, double *Axy, double *Axz, double *Ayy, double *Ayz, double *Azz,
+    double *Gamx, double *Gamy, double *Gamz,
+    double *Lap, double *betax, double *betay, double *betaz,
+    double *dtSfx, double *dtSfy, double *dtSfz,
+    double *TZ,
+    double *chi_rhs, double *trK_rhs,
+    double *gxx_rhs, double *gxy_rhs, double *gxz_rhs, double *gyy_rhs, double *gyz_rhs, double *gzz_rhs,
+    double *Axx_rhs, double *Axy_rhs, double *Axz_rhs, double *Ayy_rhs, double *Ayz_rhs, double *Azz_rhs,
+    double *Gamx_rhs, double *Gamy_rhs, double *Gamz_rhs,
+    double *Lap_rhs, double *betax_rhs, double *betay_rhs, double *betaz_rhs,
+    double *dtSfx_rhs, double *dtSfy_rhs, double *dtSfz_rhs,
+    double *TZ_rhs,
+    double *rho, double *Sx, double *Sy, double *Sz,
+    double *Sxx, double *Sxy, double *Sxz, double *Syy, double *Syz, double *Szz,
+    double *Gamxxx, double *Gamxxy, double *Gamxxz, double *Gamxyy, double *Gamxyz, double *Gamxzz,
+    double *Gamyxx, double *Gamyxy, double *Gamyxz, double *Gamyyy, double *Gamyyz, double *Gamyzz,
+    double *Gamzxx, double *Gamzxy, double *Gamzxz, double *Gamzyy, double *Gamzyz, double *Gamzzz,
+    double *Rxx, double *Rxy, double *Rxz, double *Ryy, double *Ryz, double *Rzz,
+    double *Hcon, double *Mxcon, double *Mycon, double *Mzcon, double *Gmxcon, double *Gmycon, double *Gmzcon,
+    int &Symmetry, int &Lev, double &eps, int &co)
+{
+    (void)T;
+
+    const int nx = ex[0];
+    const int ny = ex[1];
+    const int nz = ex[2];
+    const int all = nx * ny * nz;
+
+    double alpn1[all], chin1[all], gxx[all], gyy[all], gzz[all];
+    double chix[all], chiy[all], chiz[all], chixx[all], chixy[all], chixz[all], chiyy[all], chiyz[all], chizz[all];
+    double gxxx[all], gxyx[all], gxzx[all], gyyx[all], gyzx[all], gzzx[all];
+    double gxxy[all], gxyy[all], gxzy[all], gyyy[all], gyzy[all], gzzy[all];
+    double gxxz[all], gxyz[all], gxzz[all], gyyz[all], gyzz[all], gzzz[all];
+    double gxxxx[all], gxxxy[all], gxxxz[all], gxxyy[all], gxxyz[all], gxxzz[all];
+    double gxyxx[all], gxyxy[all], gxyxz[all], gxyyy[all], gxyyz[all], gxyzz[all];
+    double gxzxx[all], gxzxy[all], gxzxz[all], gxzyy[all], gxzyz[all], gxzzz[all];
+    double gyyxx[all], gyyxy[all], gyyxz[all], gyyyy[all], gyyyz[all], gyyzz[all];
+    double gyzxx[all], gyzxy[all], gyzxz[all], gyzyy[all], gyzyz[all], gyzzz[all];
+    double gzzxx[all], gzzxy[all], gzzxz[all], gzzyy[all], gzzyz[all], gzzzz[all];
+    double Lapx[all], Lapy[all], Lapz[all], Lapxx[all], Lapxy[all], Lapxz[all], Lapyy[all], Lapyz[all], Lapzz[all];
+    double betaxx[all], betaxy[all], betaxz[all], betayx[all], betayy[all], betayz[all], betazx[all], betazy[all], betazz[all];
+    double dBxx[all], dBxy[all], dBxz[all], dByx[all], dByy[all], dByz[all], dBzx[all], dBzy[all], dBzz[all];
+    double sfxxx[all], sfxxy[all], sfxxz[all], sfxyy[all], sfxyz[all], sfxzz[all];
+    double sfyxx[all], sfyxy[all], sfyxz[all], sfyyy[all], sfyyz[all], sfyzz[all];
+    double sfzxx[all], sfzxy[all], sfzxz[all], sfzyy[all], sfzyz[all], sfzzz[all];
+    double Gamxx[all], Gamxy[all], Gamxz[all], Gamyx[all], Gamyy[all], Gamyz[all], Gamzx[all], Gamzy[all], Gamzz[all];
+    double Kx[all], Ky[all], Kz[all], TZx[all], TZy[all], TZz[all];
+    double Axxx[all], Axxy[all], Axxz[all], Axyx[all], Axyy[all], Axyz[all];
+    double Axzx[all], Axzy[all], Axzz[all], Ayyx[all], Ayyy[all], Ayyz[all];
+    double Ayzx[all], Ayzy[all], Ayzz[all], Azzx[all], Azzy[all], Azzz[all];
+
+    const double SSS[3] = {1.0, 1.0, 1.0};
+    const double AAS[3] = {-1.0, -1.0, 1.0};
+    const double ASA[3] = {-1.0, 1.0, -1.0};
+    const double SAA[3] = {1.0, -1.0, -1.0};
+    const double ASS[3] = {-1.0, 1.0, 1.0};
+    const double SAS[3] = {1.0, -1.0, 1.0};
+    const double SSA[3] = {1.0, 1.0, -1.0};
+
+    const double ONE = 1.0;
+    const double TWO = 2.0;
+    const double ZEO = 0.0;
+    double chiDivfloor = 1.0e-5;
+
+    double kappa1 = 2.0e-2;
+    double kappa2 = 0.0;
+    double FF = 0.75;
+    double eta = 2.0;
+
+    for (int idx = 0; idx < all; ++idx)
+    {
+        alpn1[idx] = Lap[idx] + ONE;
+        chin1[idx] = chi_state[idx] + ONE;
+        gxx[idx] = dxx[idx] + ONE;
+        gyy[idx] = dyy[idx] + ONE;
+        gzz[idx] = dzz[idx] + ONE;
+    }
+
+    fderivs(ex, betax, betaxx, betaxy, betaxz, X, Y, Z, -1.0, 1.0, 1.0, Symmetry, Lev);
+    fderivs(ex, betay, betayx, betayy, betayz, X, Y, Z, 1.0, -1.0, 1.0, Symmetry, Lev);
+    fderivs(ex, betaz, betazx, betazy, betazz, X, Y, Z, 1.0, 1.0, -1.0, Symmetry, Lev);
+    fderivs(ex, dtSfx, dBxx, dBxy, dBxz, X, Y, Z, -1.0, 1.0, 1.0, Symmetry, Lev);
+    fderivs(ex, dtSfy, dByx, dByy, dByz, X, Y, Z, 1.0, -1.0, 1.0, Symmetry, Lev);
+    fderivs(ex, dtSfz, dBzx, dBzy, dBzz, X, Y, Z, 1.0, 1.0, -1.0, Symmetry, Lev);
+    fderivs(ex, chi_state, chix, chiy, chiz, X, Y, Z, 1.0, 1.0, 1.0, Symmetry, Lev);
+    fderivs(ex, dxx, gxxx, gxxy, gxxz, X, Y, Z, 1.0, 1.0, 1.0, Symmetry, Lev);
+    fderivs(ex, gxy, gxyx, gxyy, gxyz, X, Y, Z, -1.0, -1.0, 1.0, Symmetry, Lev);
+    fderivs(ex, gxz, gxzx, gxzy, gxzz, X, Y, Z, -1.0, 1.0, -1.0, Symmetry, Lev);
+    fderivs(ex, dyy, gyyx, gyyy, gyyz, X, Y, Z, 1.0, 1.0, 1.0, Symmetry, Lev);
+    fderivs(ex, gyz, gyzx, gyzy, gyzz, X, Y, Z, 1.0, -1.0, -1.0, Symmetry, Lev);
+    fderivs(ex, dzz, gzzx, gzzy, gzzz, X, Y, Z, 1.0, 1.0, 1.0, Symmetry, Lev);
+
+    fdderivs(ex, dxx, gxxxx, gxxxy, gxxxz, gxxyy, gxxyz, gxxzz, X, Y, Z, 1.0, 1.0, 1.0, Symmetry, Lev);
+    fdderivs(ex, dyy, gyyxx, gyyxy, gyyxz, gyyyy, gyyyz, gyyzz, X, Y, Z, 1.0, 1.0, 1.0, Symmetry, Lev);
+    fdderivs(ex, dzz, gzzxx, gzzxy, gzzxz, gzzyy, gzzyz, gzzzz, X, Y, Z, 1.0, 1.0, 1.0, Symmetry, Lev);
+    fdderivs(ex, gxy, gxyxx, gxyxy, gxyxz, gxyyy, gxyyz, gxyzz, X, Y, Z, -1.0, -1.0, 1.0, Symmetry, Lev);
+    fdderivs(ex, gxz, gxzxx, gxzxy, gxzxz, gxzyy, gxzyz, gxzzz, X, Y, Z, -1.0, 1.0, -1.0, Symmetry, Lev);
+    fdderivs(ex, gyz, gyzxx, gyzxy, gyzxz, gyzyy, gyzyz, gyzzz, X, Y, Z, 1.0, -1.0, -1.0, Symmetry, Lev);
+
+    fderivs(ex, Gamx, Gamxx, Gamxy, Gamxz, X, Y, Z, -1.0, 1.0, 1.0, Symmetry, Lev);
+    fderivs(ex, Gamy, Gamyx, Gamyy, Gamyz, X, Y, Z, 1.0, -1.0, 1.0, Symmetry, Lev);
+    fderivs(ex, Gamz, Gamzx, Gamzy, Gamzz, X, Y, Z, 1.0, 1.0, -1.0, Symmetry, Lev);
+
+    fderivs(ex, Lap, Lapx, Lapy, Lapz, X, Y, Z, 1.0, 1.0, 1.0, Symmetry, Lev);
+    fderivs(ex, trK, Kx, Ky, Kz, X, Y, Z, 1.0, 1.0, 1.0, Symmetry, Lev);
+    fderivs(ex, TZ, TZx, TZy, TZz, X, Y, Z, 1.0, 1.0, 1.0, Symmetry, Lev);
+
+    fdderivs(ex, betax, sfxxx, sfxxy, sfxxz, sfxyy, sfxyz, sfxzz, X, Y, Z, -1.0, 1.0, 1.0, Symmetry, Lev);
+    fdderivs(ex, betay, sfyxx, sfyxy, sfyxz, sfyyy, sfyyz, sfyzz, X, Y, Z, 1.0, -1.0, 1.0, Symmetry, Lev);
+    fdderivs(ex, betaz, sfzxx, sfzxy, sfzxz, sfzyy, sfzyz, sfzzz, X, Y, Z, 1.0, 1.0, -1.0, Symmetry, Lev);
+
+    fdderivs(ex, chi_state, chixx, chixy, chixz, chiyy, chiyz, chizz, X, Y, Z, 1.0, 1.0, 1.0, Symmetry, Lev);
+    fdderivs(ex, Lap, Lapxx, Lapxy, Lapxz, Lapyy, Lapyz, Lapzz, X, Y, Z, 1.0, 1.0, 1.0, Symmetry, Lev);
+
+    fderivs(ex, Axx, Axxx, Axxy, Axxz, X, Y, Z, 1.0, 1.0, 1.0, Symmetry, Lev);
+    fderivs(ex, Axy, Axyx, Axyy, Axyz, X, Y, Z, -1.0, -1.0, 1.0, Symmetry, Lev);
+    fderivs(ex, Axz, Axzx, Axzy, Axzz, X, Y, Z, -1.0, 1.0, -1.0, Symmetry, Lev);
+    fderivs(ex, Ayy, Ayyx, Ayyy, Ayyz, X, Y, Z, 1.0, 1.0, 1.0, Symmetry, Lev);
+    fderivs(ex, Ayz, Ayzx, Ayzy, Ayzz, X, Y, Z, 1.0, -1.0, -1.0, Symmetry, Lev);
+    fderivs(ex, Azz, Azzx, Azzy, Azzz, X, Y, Z, 1.0, 1.0, 1.0, Symmetry, Lev);
+
+    for (int idx = 0; idx < all; ++idx)
+    {
+        double point_kappa1 = 0.0;
+        f_z4c_rhs_point(
+            Axx[idx], Axy[idx], Axz[idx], Ayy[idx], Ayz[idx], Azz[idx],
+            alpn1[idx], dtSfx[idx], dtSfy[idx], dtSfz[idx],
+            betax[idx], betay[idx], betaz[idx],
+            chin1[idx], chiDivfloor,
+            Lapx[idx],
+            Axxx[idx], Axyx[idx], Axzx[idx], Ayyx[idx], Ayzx[idx], Azzx[idx],
+            Lapy[idx],
+            Axxy[idx], Axyy[idx], Axzy[idx], Ayyy[idx], Ayzy[idx], Azzy[idx],
+            Lapz[idx],
+            Axxz[idx], Axyz[idx], Axzz[idx], Ayyz[idx], Ayzz[idx], Azzz[idx],
+            betaxx[idx], dBxx[idx], betayx[idx], dByx[idx], betazx[idx], dBzx[idx],
+            betaxy[idx], dBxy[idx], betayy[idx], dByy[idx], betazy[idx], dBzy[idx],
+            betaxz[idx], dBxz[idx], betayz[idx], dByz[idx], betazz[idx], dBzz[idx],
+            chix[idx], chiy[idx], chiz[idx],
+            Lapxx[idx], Lapxy[idx], Lapxz[idx], Lapyy[idx], Lapyz[idx], Lapzz[idx],
+            sfxxx[idx], sfyxx[idx], sfzxx[idx],
+            sfxxy[idx], sfyxy[idx], sfzxy[idx],
+            sfxxz[idx], sfyxz[idx], sfzxz[idx],
+            sfxyy[idx], sfyyy[idx], sfzyy[idx],
+            sfxyz[idx], sfyyz[idx], sfzyz[idx],
+            sfxzz[idx], sfyzz[idx], sfzzz[idx],
+            chixx[idx], chixy[idx], chixz[idx], chiyy[idx], chiyz[idx], chizz[idx],
+            gxxxx[idx], gxyxx[idx], gxzxx[idx], gyyxx[idx], gyzxx[idx], gzzxx[idx],
+            gxxxy[idx], gxyxy[idx], gxzxy[idx], gyyxy[idx], gyzxy[idx], gzzxy[idx],
+            gxxxz[idx], gxyxz[idx], gxzxz[idx], gyyxz[idx], gyzxz[idx], gzzxz[idx],
+            gxxyy[idx], gxyyy[idx], gxzyy[idx], gyyyy[idx], gyzyy[idx], gzzyy[idx],
+            gxxyz[idx], gxyyz[idx], gxzyz[idx], gyyyz[idx], gyzyz[idx], gzzyz[idx],
+            gxxzz[idx], gxyzz[idx], gxzzz[idx], gyyzz[idx], gyzzz[idx], gzzzz[idx],
+            Gamxx[idx], gxxx[idx], gxyx[idx], gxzx[idx],
+            Gamyx[idx], gyyx[idx], gyzx[idx],
+            Gamzx[idx], gzzx[idx],
+            Gamxy[idx], gxxy[idx], gxyy[idx], gxzy[idx],
+            Gamyy[idx], gyyy[idx], gyzy[idx],
+            Gamzy[idx], gzzy[idx],
+            Gamxz[idx], gxxz[idx], gxyz[idx], gxzz[idx],
+            Gamyz[idx], gyyz[idx], gyzz[idx],
+            Gamzz[idx], gzzz[idx],
+            Kx[idx], Ky[idx], Kz[idx],
+            TZx[idx], TZy[idx], TZz[idx],
+            Gamx[idx], gxx[idx], gxy[idx], gxz[idx],
+            Gamy[idx], gyy[idx], gyz[idx],
+            Gamz[idx], gzz[idx],
+            point_kappa1, kappa2,
+            trK[idx],
+            Axx_rhs[idx], Axy_rhs[idx], Axz_rhs[idx], Ayy_rhs[idx], Ayz_rhs[idx], Azz_rhs[idx],
+            chi_rhs[idx],
+            Gamx_rhs[idx], gxx_rhs[idx], gxy_rhs[idx], gxz_rhs[idx],
+            Gamy_rhs[idx], gyy_rhs[idx], gyz_rhs[idx],
+            Gamz_rhs[idx], gzz_rhs[idx], trK_rhs[idx], TZ_rhs[idx], TZ[idx]);
+    }
+
+    for (int idx = 0; idx < all; ++idx)
+        Lap_rhs[idx] = -TWO * alpn1[idx] * trK[idx];
+
+#if (GAUGE == 0)
+    for (int idx = 0; idx < all; ++idx)
+    {
+        betax_rhs[idx] = FF * dtSfx[idx];
+        betay_rhs[idx] = FF * dtSfy[idx];
+        betaz_rhs[idx] = FF * dtSfz[idx];
+        dtSfx_rhs[idx] = Gamx_rhs[idx] - eta * dtSfx[idx];
+        dtSfy_rhs[idx] = Gamy_rhs[idx] - eta * dtSfy[idx];
+        dtSfz_rhs[idx] = Gamz_rhs[idx] - eta * dtSfz[idx];
+    }
+#elif (GAUGE == 1)
+    for (int idx = 0; idx < all; ++idx)
+    {
+        betax_rhs[idx] = Gamx[idx] - eta * betax[idx];
+        betay_rhs[idx] = Gamy[idx] - eta * betay[idx];
+        betaz_rhs[idx] = Gamz[idx] - eta * betaz[idx];
+        dtSfx_rhs[idx] = ZEO;
+        dtSfy_rhs[idx] = ZEO;
+        dtSfz_rhs[idx] = ZEO;
+    }
+#else
+#error "z4c_rhs_c.C currently supports GAUGE == 0 or GAUGE == 1 for Z4C"
+#endif
+
+    lopsided(ex, X, Y, Z, gxx, gxx_rhs, betax, betay, betaz, Symmetry, SSS);
+    lopsided(ex, X, Y, Z, gxy, gxy_rhs, betax, betay, betaz, Symmetry, AAS);
+    lopsided(ex, X, Y, Z, gxz, gxz_rhs, betax, betay, betaz, Symmetry, ASA);
+    lopsided(ex, X, Y, Z, gyy, gyy_rhs, betax, betay, betaz, Symmetry, SSS);
+    lopsided(ex, X, Y, Z, gyz, gyz_rhs, betax, betay, betaz, Symmetry, SAA);
+    lopsided(ex, X, Y, Z, gzz, gzz_rhs, betax, betay, betaz, Symmetry, SSS);
+
+    lopsided(ex, X, Y, Z, Axx, Axx_rhs, betax, betay, betaz, Symmetry, SSS);
+    lopsided(ex, X, Y, Z, Axy, Axy_rhs, betax, betay, betaz, Symmetry, AAS);
+    lopsided(ex, X, Y, Z, Axz, Axz_rhs, betax, betay, betaz, Symmetry, ASA);
+    lopsided(ex, X, Y, Z, Ayy, Ayy_rhs, betax, betay, betaz, Symmetry, SSS);
+    lopsided(ex, X, Y, Z, Ayz, Ayz_rhs, betax, betay, betaz, Symmetry, SAA);
+    lopsided(ex, X, Y, Z, Azz, Azz_rhs, betax, betay, betaz, Symmetry, SSS);
+
+    lopsided(ex, X, Y, Z, chi_state, chi_rhs, betax, betay, betaz, Symmetry, SSS);
+    lopsided(ex, X, Y, Z, trK, trK_rhs, betax, betay, betaz, Symmetry, SSS);
+
+    lopsided(ex, X, Y, Z, Gamx, Gamx_rhs, betax, betay, betaz, Symmetry, ASS);
+    lopsided(ex, X, Y, Z, Gamy, Gamy_rhs, betax, betay, betaz, Symmetry, SAS);
+    lopsided(ex, X, Y, Z, Gamz, Gamz_rhs, betax, betay, betaz, Symmetry, SSA);
+
+    lopsided(ex, X, Y, Z, Lap, Lap_rhs, betax, betay, betaz, Symmetry, SSS);
+    lopsided(ex, X, Y, Z, betax, betax_rhs, betax, betay, betaz, Symmetry, ASS);
+    lopsided(ex, X, Y, Z, betay, betay_rhs, betax, betay, betaz, Symmetry, SAS);
+    lopsided(ex, X, Y, Z, betaz, betaz_rhs, betax, betay, betaz, Symmetry, SSA);
+#if (GAUGE == 0)
+    lopsided(ex, X, Y, Z, dtSfx, dtSfx_rhs, betax, betay, betaz, Symmetry, ASS);
+    lopsided(ex, X, Y, Z, dtSfy, dtSfy_rhs, betax, betay, betaz, Symmetry, SAS);
+    lopsided(ex, X, Y, Z, dtSfz, dtSfz_rhs, betax, betay, betaz, Symmetry, SSA);
+#endif
+    lopsided(ex, X, Y, Z, TZ, TZ_rhs, betax, betay, betaz, Symmetry, SSS);
+
+    for (int idx = 0; idx < all; ++idx)
+    {
+        double Gamxa = 0.0, Gamya = 0.0, Gamza = 0.0;
+        z4c_contract_gamma(
+            gxx[idx], gxy[idx], gxz[idx], gyy[idx], gyz[idx], gzz[idx],
+            gxxx[idx], gxyx[idx], gxzx[idx], gyyx[idx], gyzx[idx], gzzx[idx],
+            gxxy[idx], gxyy[idx], gxzy[idx], gyyy[idx], gyzy[idx], gzzy[idx],
+            gxxz[idx], gxyz[idx], gxzz[idx], gyyz[idx], gyzz[idx], gzzz[idx],
+            Gamxa, Gamya, Gamza);
+
+        TZ_rhs[idx] -= alpn1[idx] * (TWO + kappa2) * kappa1 * TZ[idx];
+        trK_rhs[idx] += alpn1[idx] * kappa1 * (ONE - kappa2) * TZ[idx];
+        Gamx_rhs[idx] -= TWO * alpn1[idx] * kappa1 * (Gamx[idx] - Gamxa);
+        Gamy_rhs[idx] -= TWO * alpn1[idx] * kappa1 * (Gamy[idx] - Gamya);
+        Gamz_rhs[idx] -= TWO * alpn1[idx] * kappa1 * (Gamz[idx] - Gamza);
+    }
+
+    if (eps > 0.0)
+    {
+        kodis(ex, X, Y, Z, chi_state, chi_rhs, SSS, Symmetry, eps);
+        kodis(ex, X, Y, Z, trK, trK_rhs, SSS, Symmetry, eps);
+        kodis(ex, X, Y, Z, gxx, gxx_rhs, SSS, Symmetry, eps);
+        kodis(ex, X, Y, Z, gxy, gxy_rhs, AAS, Symmetry, eps);
+        kodis(ex, X, Y, Z, gxz, gxz_rhs, ASA, Symmetry, eps);
+        kodis(ex, X, Y, Z, gyy, gyy_rhs, SSS, Symmetry, eps);
+        kodis(ex, X, Y, Z, gyz, gyz_rhs, SAA, Symmetry, eps);
+        kodis(ex, X, Y, Z, gzz, gzz_rhs, SSS, Symmetry, eps);
+        kodis(ex, X, Y, Z, Axx, Axx_rhs, SSS, Symmetry, eps);
+        kodis(ex, X, Y, Z, Axy, Axy_rhs, AAS, Symmetry, eps);
+        kodis(ex, X, Y, Z, Axz, Axz_rhs, ASA, Symmetry, eps);
+        kodis(ex, X, Y, Z, Ayy, Ayy_rhs, SSS, Symmetry, eps);
+        kodis(ex, X, Y, Z, Ayz, Ayz_rhs, SAA, Symmetry, eps);
+        kodis(ex, X, Y, Z, Azz, Azz_rhs, SSS, Symmetry, eps);
+        kodis(ex, X, Y, Z, Gamx, Gamx_rhs, ASS, Symmetry, eps);
+        kodis(ex, X, Y, Z, Gamy, Gamy_rhs, SAS, Symmetry, eps);
+        kodis(ex, X, Y, Z, Gamz, Gamz_rhs, SSA, Symmetry, eps);
+        kodis(ex, X, Y, Z, Lap, Lap_rhs, SSS, Symmetry, eps);
+        kodis(ex, X, Y, Z, betax, betax_rhs, ASS, Symmetry, eps);
+        kodis(ex, X, Y, Z, betay, betay_rhs, SAS, Symmetry, eps);
+        kodis(ex, X, Y, Z, betaz, betaz_rhs, SSA, Symmetry, eps);
+#if (GAUGE == 0)
+        kodis(ex, X, Y, Z, dtSfx, dtSfx_rhs, ASS, Symmetry, eps);
+        kodis(ex, X, Y, Z, dtSfy, dtSfy_rhs, SAS, Symmetry, eps);
+        kodis(ex, X, Y, Z, dtSfz, dtSfz_rhs, SSA, Symmetry, eps);
+#endif
+        kodis(ex, X, Y, Z, TZ, TZ_rhs, SSS, Symmetry, eps);
+    }
+
+    if (co == 0)
+    {
+#if (ABV == 0)
+        f_ricci_gamma(ex, X, Y, Z,
+                      chi_constraints,
+                      dxx, gxy, gxz, dyy, gyz, dzz,
+                      Gamx, Gamy, Gamz,
+                      Gamxxx, Gamxxy, Gamxxz, Gamxyy, Gamxyz, Gamxzz,
+                      Gamyxx, Gamyxy, Gamyxz, Gamyyy, Gamyyz, Gamyzz,
+                      Gamzxx, Gamzxy, Gamzxz, Gamzyy, Gamzyz, Gamzzz,
+                      Rxx, Rxy, Rxz, Ryy, Ryz, Rzz,
+                      Symmetry);
+#endif
+        f_constraint_bssn(ex, X, Y, Z,
+                          chi_constraints, trK,
+                          dxx, gxy, gxz, dyy, gyz, dzz,
+                          Axx, Axy, Axz, Ayy, Ayz, Azz,
+                          Gamx, Gamy, Gamz,
+                          Lap, betax, betay, betaz, rho, Sx, Sy, Sz,
+                          Gamxxx, Gamxxy, Gamxxz, Gamxyy, Gamxyz, Gamxzz,
+                          Gamyxx, Gamyxy, Gamyxz, Gamyyy, Gamyyz, Gamyzz,
+                          Gamzxx, Gamzxy, Gamzxz, Gamzyy, Gamzyz, Gamzzz,
+                          Rxx, Rxy, Rxz, Ryy, Ryz, Rzz,
+                          Hcon, Mxcon, Mycon, Mzcon, Gmxcon, Gmycon, Gmzcon,
+                          Symmetry);
+    }
+
+    return 0;
+}
+
+extern "C" int f_compute_rhs_Z4c(int *ex, double &T,
+                                 double *X, double *Y, double *Z,
+                                 double *chi, double *trK,
+                                 double *dxx, double *gxy, double *gxz, double *dyy, double *gyz, double *dzz,
+                                 double *Axx, double *Axy, double *Axz, double *Ayy, double *Ayz, double *Azz,
+                                 double *Gamx, double *Gamy, double *Gamz,
+                                 double *Lap, double *betax, double *betay, double *betaz,
+                                 double *dtSfx, double *dtSfy, double *dtSfz,
+                                 double *TZ,
+                                 double *chi_rhs, double *trK_rhs,
+                                 double *gxx_rhs, double *gxy_rhs, double *gxz_rhs, double *gyy_rhs, double *gyz_rhs, double *gzz_rhs,
+                                 double *Axx_rhs, double *Axy_rhs, double *Axz_rhs, double *Ayy_rhs, double *Ayz_rhs, double *Azz_rhs,
+                                 double *Gamx_rhs, double *Gamy_rhs, double *Gamz_rhs,
+                                 double *Lap_rhs, double *betax_rhs, double *betay_rhs, double *betaz_rhs,
+                                 double *dtSfx_rhs, double *dtSfy_rhs, double *dtSfz_rhs,
+                                 double *TZ_rhs,
+                                 double *rho, double *Sx, double *Sy, double *Sz,
+                                 double *Sxx, double *Sxy, double *Sxz, double *Syy, double *Syz, double *Szz,
+                                 double *Gamxxx, double *Gamxxy, double *Gamxxz, double *Gamxyy, double *Gamxyz, double *Gamxzz,
+                                 double *Gamyxx, double *Gamyxy, double *Gamyxz, double *Gamyyy, double *Gamyyz, double *Gamyzz,
+                                 double *Gamzxx, double *Gamzxy, double *Gamzxz, double *Gamzyy, double *Gamzyz, double *Gamzzz,
+                                 double *Rxx, double *Rxy, double *Rxz, double *Ryy, double *Ryz, double *Rzz,
+                                 double *Hcon, double *Mxcon, double *Mycon, double *Mzcon, double *Gmxcon, double *Gmycon, double *Gmzcon,
+                                 int &Symmetry, int &Lev, double &eps, int &co)
+{
+    return compute_rhs_z4c_cartesian(
+        ex, T, X, Y, Z,
+        chi, chi, trK,
+        dxx, gxy, gxz, dyy, gyz, dzz,
+        Axx, Axy, Axz, Ayy, Ayz, Azz,
+        Gamx, Gamy, Gamz,
+        Lap, betax, betay, betaz,
+        dtSfx, dtSfy, dtSfz,
+        TZ,
+        chi_rhs, trK_rhs,
+        gxx_rhs, gxy_rhs, gxz_rhs, gyy_rhs, gyz_rhs, gzz_rhs,
+        Axx_rhs, Axy_rhs, Axz_rhs, Ayy_rhs, Ayz_rhs, Azz_rhs,
+        Gamx_rhs, Gamy_rhs, Gamz_rhs,
+        Lap_rhs, betax_rhs, betay_rhs, betaz_rhs,
+        dtSfx_rhs, dtSfy_rhs, dtSfz_rhs,
+        TZ_rhs,
+        rho, Sx, Sy, Sz,
+        Sxx, Sxy, Sxz, Syy, Syz, Szz,
+        Gamxxx, Gamxxy, Gamxxz, Gamxyy, Gamxyz, Gamxzz,
+        Gamyxx, Gamyxy, Gamyxz, Gamyyy, Gamyyz, Gamyzz,
+        Gamzxx, Gamzxy, Gamzxz, Gamzyy, Gamzyz, Gamzzz,
+        Rxx, Rxy, Rxz, Ryy, Ryz, Rzz,
+        Hcon, Mxcon, Mycon, Mzcon, Gmxcon, Gmycon, Gmzcon,
+        Symmetry, Lev, eps, co);
+}
+
+extern "C" int f_compute_rhs_Z4cnot(int *ex, double &T,
+                                    double *X, double *Y, double *Z,
+                                    double *chi, double *trK,
+                                    double *dxx, double *gxy, double *gxz, double *dyy, double *gyz, double *dzz,
+                                    double *Axx, double *Axy, double *Axz, double *Ayy, double *Ayz, double *Azz,
+                                    double *Gamx, double *Gamy, double *Gamz,
+                                    double *Lap, double *betax, double *betay, double *betaz,
+                                    double *dtSfx, double *dtSfy, double *dtSfz,
+                                    double *TZ,
+                                    double *chi_rhs, double *trK_rhs,
+                                    double *gxx_rhs, double *gxy_rhs, double *gxz_rhs, double *gyy_rhs, double *gyz_rhs, double *gzz_rhs,
+                                    double *Axx_rhs, double *Axy_rhs, double *Axz_rhs, double *Ayy_rhs, double *Ayz_rhs, double *Azz_rhs,
+                                    double *Gamx_rhs, double *Gamy_rhs, double *Gamz_rhs,
+                                    double *Lap_rhs, double *betax_rhs, double *betay_rhs, double *betaz_rhs,
+                                    double *dtSfx_rhs, double *dtSfy_rhs, double *dtSfz_rhs,
+                                    double *TZ_rhs,
+                                    double *rho, double *Sx, double *Sy, double *Sz,
+                                    double *Sxx, double *Sxy, double *Sxz, double *Syy, double *Syz, double *Szz,
+                                    double *Gamxxx, double *Gamxxy, double *Gamxxz, double *Gamxyy, double *Gamxyz, double *Gamxzz,
+                                    double *Gamyxx, double *Gamyxy, double *Gamyxz, double *Gamyyy, double *Gamyyz, double *Gamyzz,
+                                    double *Gamzxx, double *Gamzxy, double *Gamzxz, double *Gamzyy, double *Gamzyz, double *Gamzzz,
+                                    double *Rxx, double *Rxy, double *Rxz, double *Ryy, double *Ryz, double *Rzz,
+                                    double *Hcon, double *Mxcon, double *Mycon, double *Mzcon, double *Gmxcon, double *Gmycon, double *Gmzcon,
+                                    int &Symmetry, int &Lev, double &eps, int &co, double &chitiny)
+{
+    const int all = ex[0] * ex[1] * ex[2];
+    std::vector<double> chi_clamped(chi, chi + all);
+    f_lowerboundset(ex, chi_clamped.data(), chitiny);
+
+    const int ret = compute_rhs_z4c_cartesian(
+        ex, T, X, Y, Z,
+        chi_clamped.data(), chi, trK,
+        dxx, gxy, gxz, dyy, gyz, dzz,
+        Axx, Axy, Axz, Ayy, Ayz, Azz,
+        Gamx, Gamy, Gamz,
+        Lap, betax, betay, betaz,
+        dtSfx, dtSfy, dtSfz,
+        TZ,
+        chi_rhs, trK_rhs,
+        gxx_rhs, gxy_rhs, gxz_rhs, gyy_rhs, gyz_rhs, gzz_rhs,
+        Axx_rhs, Axy_rhs, Axz_rhs, Ayy_rhs, Ayz_rhs, Azz_rhs,
+        Gamx_rhs, Gamy_rhs, Gamz_rhs,
+        Lap_rhs, betax_rhs, betay_rhs, betaz_rhs,
+        dtSfx_rhs, dtSfy_rhs, dtSfz_rhs,
+        TZ_rhs,
+        rho, Sx, Sy, Sz,
+        Sxx, Sxy, Sxz, Syy, Syz, Szz,
+        Gamxxx, Gamxxy, Gamxxz, Gamxyy, Gamxyz, Gamxzz,
+        Gamyxx, Gamyxy, Gamyxz, Gamyyy, Gamyyz, Gamyzz,
+        Gamzxx, Gamzxy, Gamzxz, Gamzyy, Gamzyz, Gamzzz,
+        Rxx, Rxy, Rxz, Ryy, Ryz, Rzz,
+        Hcon, Mxcon, Mycon, Mzcon, Gmxcon, Gmycon, Gmzcon,
+        Symmetry, Lev, eps, co);
+
+    if (ret != 0 || co != 0)
+        return ret;
+
+#if (ABV == 0)
+    f_ricci_gamma(ex, X, Y, Z,
+                  chi,
+                  dxx, gxy, gxz, dyy, gyz, dzz,
+                  Gamx, Gamy, Gamz,
+                  Gamxxx, Gamxxy, Gamxxz, Gamxyy, Gamxyz, Gamxzz,
+                  Gamyxx, Gamyxy, Gamyxz, Gamyyy, Gamyyz, Gamyzz,
+                  Gamzxx, Gamzxy, Gamzxz, Gamzyy, Gamzyz, Gamzzz,
+                  Rxx, Rxy, Rxz, Ryy, Ryz, Rzz,
+                  Symmetry);
+#endif
+    f_constraint_bssn(ex, X, Y, Z,
+                      chi, trK,
+                      dxx, gxy, gxz, dyy, gyz, dzz,
+                      Axx, Axy, Axz, Ayy, Ayz, Azz,
+                      Gamx, Gamy, Gamz,
+                      Lap, betax, betay, betaz, rho, Sx, Sy, Sz,
+                      Gamxxx, Gamxxy, Gamxxz, Gamxyy, Gamxyz, Gamxzz,
+                      Gamyxx, Gamyxy, Gamyxz, Gamyyy, Gamyyz, Gamyzz,
+                      Gamzxx, Gamzxy, Gamzxz, Gamzyy, Gamzyz, Gamzzz,
+                      Rxx, Rxy, Rxz, Ryy, Ryz, Rzz,
+                      Hcon, Mxcon, Mycon, Mzcon, Gmxcon, Gmycon, Gmzcon,
+                      Symmetry);
+    return ret;
+}

AMSS_NCKU_source/z4c_rhs_cuda.h

@@ -0,0 +1,258 @@
+#ifndef Z4C_RHS_CUDA_H
+#define Z4C_RHS_CUDA_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+enum {
+    Z4C_CUDA_STATE_COUNT = 25
+};
+
+int z4c_cuda_rk4_substep(void *block_tag,
+                         int *ex, double *X, double *Y, double *Z,
+                         double **state_host_in,
+                         double **state_host_out,
+                         const double *propspeed,
+                         const double *soa_flat,
+                         const double *bbox,
+                         double &dT,
+                         double &T,
+                         int &RK4,
+                         int &apply_bam_bc,
+                         int &Symmetry,
+                         int &Lev,
+                         double &eps,
+                         int &co,
+                         int &keep_resident_state,
+                         int &apply_enforce_ga,
+                         double &chitiny);
+
+int z4c_cuda_download_resident_state(void *block_tag,
+                                     int *ex,
+                                     double **state_host_out);
+
+int z4c_cuda_pack_state_region_to_host_buffer(void *block_tag,
+                                              int state_index,
+                                              double *host_buffer,
+                                              int *ex,
+                                              int i0, int j0, int k0,
+                                              int sx, int sy, int sz);
+
+int z4c_cuda_unpack_state_region_from_host_buffer(void *block_tag,
+                                                  int state_index,
+                                                  double *host_buffer,
+                                                  int *ex,
+                                                  int i0, int j0, int k0,
+                                                  int sx, int sy, int sz);
+
+int z4c_cuda_pack_state_batch_to_host_buffer(void *block_tag,
+                                             int state_count,
+                                             double *host_buffer,
+                                             int *ex,
+                                             int i0, int j0, int k0,
+                                             int sx, int sy, int sz);
+
+int z4c_cuda_pack_state_batch_to_host_buffer_for_host_views(void *block_tag,
+                                                            double **state_host_key,
+                                                            int state_count,
+                                                            double *host_buffer,
+                                                            int *ex,
+                                                            int i0, int j0, int k0,
+                                                            int sx, int sy, int sz);
+
+int z4c_cuda_unpack_state_batch_from_host_buffer(void *block_tag,
+                                                 int state_count,
+                                                 double *host_buffer,
+                                                 int *ex,
+                                                 int i0, int j0, int k0,
+                                                 int sx, int sy, int sz);
+
+int z4c_cuda_unpack_state_batch_from_host_buffer_for_host_views(void *block_tag,
+                                                                double **state_host_key,
+                                                                int state_count,
+                                                                double *host_buffer,
+                                                                int *ex,
+                                                                int i0, int j0, int k0,
+                                                                int sx, int sy, int sz);
+
+int z4c_cuda_pack_state_batch_to_device_buffer(void *block_tag,
+                                               int state_count,
+                                               double *device_buffer,
+                                               int *ex,
+                                               int i0, int j0, int k0,
+                                               int sx, int sy, int sz);
+
+int z4c_cuda_pack_state_batch_to_device_buffer_for_host_views(void *block_tag,
+                                                              double **state_host_key,
+                                                              int state_count,
+                                                              double *device_buffer,
+                                                              int *ex,
+                                                              int i0, int j0, int k0,
+                                                              int sx, int sy, int sz);
+
+int z4c_cuda_unpack_state_batch_from_device_buffer(void *block_tag,
+                                                   int state_count,
+                                                   double *device_buffer,
+                                                   int *ex,
+                                                   int i0, int j0, int k0,
+                                                   int sx, int sy, int sz);
+
+int z4c_cuda_unpack_state_batch_from_device_buffer_for_host_views(void *block_tag,
+                                                                  double **state_host_key,
+                                                                  int state_count,
+                                                                  double *device_buffer,
+                                                                  int *ex,
+                                                                  int i0, int j0, int k0,
+                                                                  int sx, int sy, int sz);
+
+int z4c_cuda_pack_state_segments_to_device_buffer(void *block_tag,
+                                                  int state_count,
+                                                  double *device_buffer,
+                                                  int *ex,
+                                                  int segment_count,
+                                                  const int *segment_meta);
+
+int z4c_cuda_pack_state_segments_to_device_buffer_for_host_views(void *block_tag,
+                                                                 double **state_host_key,
+                                                                 int state_count,
+                                                                 double *device_buffer,
+                                                                 int *ex,
+                                                                 int segment_count,
+                                                                 const int *segment_meta);
+
+int z4c_cuda_unpack_state_segments_from_device_buffer(void *block_tag,
+                                                      int state_count,
+                                                      double *device_buffer,
+                                                      int *ex,
+                                                      int segment_count,
+                                                      const int *segment_meta);
+
+int z4c_cuda_unpack_state_segments_from_device_buffer_for_host_views(void *block_tag,
+                                                                     double **state_host_key,
+                                                                     int state_count,
+                                                                     double *device_buffer,
+                                                                     int *ex,
+                                                                     int segment_count,
+                                                                     const int *segment_meta);
+
+int z4c_cuda_restrict_state_segments_to_device_buffer(void *block_tag,
+                                                      int state_count,
+                                                      double *device_buffer,
+                                                      int *ex,
+                                                      int segment_count,
+                                                      const int *segment_meta,
+                                                      const double *state_soa);
+
+int z4c_cuda_restrict_state_segments_to_device_buffer_for_host_views(void *block_tag,
+                                                                     double **state_host_key,
+                                                                     int state_count,
+                                                                     double *device_buffer,
+                                                                     int *ex,
+                                                                     int segment_count,
+                                                                     const int *segment_meta,
+                                                                     const double *state_soa);
+
+int z4c_cuda_prolong_state_segments_to_device_buffer(void *block_tag,
+                                                     int state_count,
+                                                     double *device_buffer,
+                                                     int *ex,
+                                                     int segment_count,
+                                                     const int *segment_meta,
+                                                     const double *state_soa);
+
+int z4c_cuda_prolong_state_segments_to_device_buffer_for_host_views(void *block_tag,
+                                                                    double **state_host_key,
+                                                                    int state_count,
+                                                                    double *device_buffer,
+                                                                    int *ex,
+                                                                    int segment_count,
+                                                                    const int *segment_meta,
+                                                                    const double *state_soa);
+
+int z4c_cuda_restrict_state_batch_to_device_buffer(void *block_tag,
+                                                   int state_count,
+                                                   double *device_buffer,
+                                                   int *ex,
+                                                   int sx, int sy, int sz,
+                                                   int fi0, int fj0, int fk0,
+                                                   const double *state_soa);
+
+int z4c_cuda_restrict_state_batch_to_device_buffer_for_host_views(void *block_tag,
+                                                                  double **state_host_key,
+                                                                  int state_count,
+                                                                  double *device_buffer,
+                                                                  int *ex,
+                                                                  int sx, int sy, int sz,
+                                                                  int fi0, int fj0, int fk0,
+                                                                  const double *state_soa);
+
+int z4c_cuda_prolong_state_batch_to_device_buffer(void *block_tag,
+                                                  int state_count,
+                                                  double *device_buffer,
+                                                  int *ex,
+                                                  int sx, int sy, int sz,
+                                                  int ii0, int jj0, int kk0,
+                                                  int lbc_i, int lbc_j, int lbc_k,
+                                                  const double *state_soa);
+
+int z4c_cuda_prolong_state_batch_to_device_buffer_for_host_views(void *block_tag,
+                                                                 double **state_host_key,
+                                                                 int state_count,
+                                                                 double *device_buffer,
+                                                                 int *ex,
+                                                                 int sx, int sy, int sz,
+                                                                 int ii0, int jj0, int kk0,
+                                                                 int lbc_i, int lbc_j, int lbc_k,
+                                                                 const double *state_soa);
+
+int z4c_cuda_download_state_subset(void *block_tag,
+                                   int *ex,
+                                   int subset_count,
+                                   const int *state_indices,
+                                   double **state_host_out);
+
+int z4c_cuda_upload_state_subset(void *block_tag,
+                                 int *ex,
+                                 int subset_count,
+                                 const int *state_indices,
+                                 double **state_host_in);
+
+int z4c_cuda_compute_constraints_resident(void *block_tag,
+                                          int *ex, double *X, double *Y, double *Z,
+                                          int Symmetry, double eps, int co,
+                                          double **constraint_host_out);
+
+int z4c_cuda_interp_state_point3(void *block_tag,
+                                 int *ex,
+                                 int state0,
+                                 int state1,
+                                 int state2,
+                                 double x0,
+                                 double y0,
+                                 double z0,
+                                 double dx,
+                                 double dy,
+                                 double dz,
+                                 double px,
+                                 double py,
+                                 double pz,
+                                 int ordn,
+                                 int symmetry,
+                                 const double *soa3,
+                                 double *out3);
+
+int z4c_cuda_download_constraint_outputs(int *ex,
+                                         double **constraint_host_out);
+
+int z4c_cuda_has_resident_state(void *block_tag);
+int z4c_cuda_resident_state_matches(void *block_tag,
+                                    double **state_host_key);
+
+void z4c_cuda_release_step_ctx(void *block_tag);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

generate_interp_lb_header.py

@@ -0,0 +1,72 @@
+#!/usr/bin/env python3
+"""Convert interp_lb_profile.bin to a C header for compile-time embedding."""
+import struct, sys
+
+if len(sys.argv) < 3:
+    print(f"Usage: {sys.argv[0]} <profile.bin> <output.h>")
+    sys.exit(1)
+
+with open(sys.argv[1], 'rb') as f:
+    magic, version, nprocs, num_heavy = struct.unpack('IIii', f.read(16))
+    threshold = struct.unpack('d', f.read(8))[0]
+    times = list(struct.unpack(f'{nprocs}d', f.read(nprocs * 8)))
+    heavy = list(struct.unpack(f'{num_heavy}i', f.read(num_heavy * 4)))
+
+# For each heavy rank, compute split: left half -> lighter neighbor, right half -> heavy rank
+# (or vice versa depending on which neighbor is lighter)
+splits = []
+for hr in heavy:
+    prev_t = times[hr - 1] if hr > 0 else 1e30
+    next_t = times[hr + 1] if hr < nprocs - 1 else 1e30
+    if prev_t <= next_t:
+        splits.append((hr, hr - 1, hr))  # (block_id, r_left, r_right)
+    else:
+        splits.append((hr, hr, hr + 1))
+
+# Also remap the displaced neighbor blocks
+remaps = {}
+for hr, r_l, r_r in splits:
+    if r_l != hr:
+        # We took r_l's slot, so remap block r_l to its other neighbor
+        displaced = r_l
+        if displaced > 0 and displaced - 1 not in [s[0] for s in splits]:
+            remaps[displaced] = displaced - 1
+        elif displaced < nprocs - 1:
+            remaps[displaced] = displaced + 1
+    else:
+        displaced = r_r
+        if displaced < nprocs - 1 and displaced + 1 not in [s[0] for s in splits]:
+            remaps[displaced] = displaced + 1
+        elif displaced > 0:
+            remaps[displaced] = displaced - 1
+
+with open(sys.argv[2], 'w') as out:
+    out.write("/* Auto-generated from interp_lb_profile.bin — do not edit */\n")
+    out.write("#ifndef INTERP_LB_PROFILE_DATA_H\n")
+    out.write("#define INTERP_LB_PROFILE_DATA_H\n\n")
+    out.write(f"#define INTERP_LB_NPROCS {nprocs}\n")
+    out.write(f"#define INTERP_LB_NUM_HEAVY {num_heavy}\n\n")
+    out.write(f"static const int interp_lb_heavy_blocks[{num_heavy}] = {{")
+    out.write(", ".join(str(h) for h in heavy))
+    out.write("};\n\n")
+    out.write("/* Split table: {block_id, r_left, r_right} */\n")
+    out.write(f"static const int interp_lb_splits[{num_heavy}][3] = {{\n")
+    for bid, rl, rr in splits:
+        out.write(f"    {{{bid}, {rl}, {rr}}},\n")
+    out.write("};\n\n")
+    out.write("/* Rank remap for displaced neighbor blocks */\n")
+    out.write(f"static const int interp_lb_num_remaps = {len(remaps)};\n")
+    out.write(f"static const int interp_lb_remaps[][2] = {{\n")
+    for src, dst in sorted(remaps.items()):
+        out.write(f"    {{{src}, {dst}}},\n")
+    if not remaps:
+        out.write("    {-1, -1},\n")
+    out.write("};\n\n")
+    out.write("#endif /* INTERP_LB_PROFILE_DATA_H */\n")
+
+print(f"Generated {sys.argv[2]}:")
+print(f"  {num_heavy} heavy blocks to split: {heavy}")
+for bid, rl, rr in splits:
+    print(f"    block {bid}: split -> rank {rl} (left), rank {rr} (right)")
+for src, dst in sorted(remaps.items()):
+    print(f"    block {src}: remap -> rank {dst}")

generate_macrodef.py

@@ -144,11 +144,67 @@ def generate_macrodef_h():
     print( "#define REGLEV 0",      file=file1 )
     print(                          file=file1 )
 
+    # Define fine-grained timing/debug macros.
+    # All of them default to OFF so production builds do not pay profiling overhead.
+
+    fine_timing = getattr(input_data, "Fine_Timing",
+                  getattr(input_data, "Finegrained_Timing", "no"))
+    kernel_fine_timing = getattr(input_data, "Kernel_Fine_Timing",
+                          getattr(input_data, "BSSN_Kernel_Fine_Timing", "no"))
+    stdin_abort_poll = getattr(input_data, "Enable_Stdin_Abort_Poll",
+                       getattr(input_data, "Stdin_Abort_Poll", "no"))
+    timing_report_every = max(1, int(getattr(
+        input_data, "Timing_Every_Steps",
+        getattr(input_data, "Timing_Report_Every", 1))))
+    timing_top_hotspots = max(1, int(getattr(
+        input_data, "Timing_Top_Hotspots", 8)))
+
+    if ( fine_timing == "yes" ):
+        print( "#define BSSN_FINE_TIMING 1", file=file1 )
+        print(                               file=file1 )
+    elif ( fine_timing == "no" ):
+        print( "#define BSSN_FINE_TIMING 0", file=file1 )
+        print(                               file=file1 )
+    else:
+        print( "Fine_Timing setting error!!!" )
+        print()
+        print( "# Fine_Timing setting error!!!", file=file1 )
+        print(                                   file=file1 )
+
+    print( f"#define BSSN_FINE_TIMING_EVERY {timing_report_every}", file=file1 )
+    print(                                                          file=file1 )
+    print( f"#define BSSN_FINE_TIMING_TOPN {timing_top_hotspots}",  file=file1 )
+    print(                                                          file=file1 )
+
+    if ( kernel_fine_timing == "yes" ):
+        print( "#define BSSN_KERNEL_FINE_TIMING 1", file=file1 )
+        print(                                      file=file1 )
+    elif ( kernel_fine_timing == "no" ):
+        print( "#define BSSN_KERNEL_FINE_TIMING 0", file=file1 )
+        print(                                      file=file1 )
+    else:
+        print( "Kernel_Fine_Timing setting error!!!" )
+        print()
+        print( "# Kernel_Fine_Timing setting error!!!", file=file1 )
+        print(                                          file=file1 )
+
+    if ( stdin_abort_poll == "yes" ):
+        print( "#define BSSN_ENABLE_STDIN_ABORT_POLL 1", file=file1 )
+        print(                                           file=file1 )
+    elif ( stdin_abort_poll == "no" ):
+        print( "#define BSSN_ENABLE_STDIN_ABORT_POLL 0", file=file1 )
+        print(                                           file=file1 )
+    else:
+        print( "Enable_Stdin_Abort_Poll setting error!!!" )
+        print()
+        print( "# Enable_Stdin_Abort_Poll setting error!!!", file=file1 )
+        print(                                               file=file1 )
+
     # Define macro USE_GPU
     # use GPU or not
     
     if ( input_data.GPU_Calculation == "yes"):
-        print( "#define USE_GPU",   file=file1 )
+        print( "//#define USE_GPU",   file=file1 )
         print(                      file=file1 )
     elif ( input_data.GPU_Calculation == "no"):
         print( "//#define USE_GPU", file=file1 )
@@ -224,6 +280,21 @@ def generate_macrodef_h():
     print( "//     0: for every level;",                                                 file=file1 ) 
     print( "//     1: for all",                                                          file=file1 )
     print( "//",                                                                         file=file1 )
+    print( "// define BSSN_FINE_TIMING",                                                 file=file1 )
+    print( "//     enable fine-grained per-timestep timing monitor",                     file=file1 )
+    print( "//",                                                                         file=file1 )
+    print( "// define BSSN_FINE_TIMING_EVERY",                                           file=file1 )
+    print( "//     report timing every N coarse timesteps",                              file=file1 )
+    print( "//",                                                                         file=file1 )
+    print( "// define BSSN_FINE_TIMING_TOPN",                                            file=file1 )
+    print( "//     number of hottest timing buckets shown in stdout",                    file=file1 )
+    print( "//",                                                                         file=file1 )
+    print( "// define BSSN_KERNEL_FINE_TIMING",                                          file=file1 )
+    print( "//     enable split timing inside compute_rhs_bssn",                         file=file1 )
+    print( "//",                                                                         file=file1 )
+    print( "// define BSSN_ENABLE_STDIN_ABORT_POLL",                                     file=file1 )
+    print( "//     poll stdin and broadcast abort flag every coarse step",               file=file1 )
+    print( "//",                                                                         file=file1 )
     print( "// define USE_GPU",                                                          file=file1 )
     print( "//     use gpu or not",                                                      file=file1 )
     print( "//",                                                                         file=file1 )

makefile_and_run.py

@@ -9,7 +9,169 @@
 
 
 import AMSS_NCKU_Input as input_data
+import os
+import shutil
 import subprocess
+import time
+
+
+def get_last_n_cores_per_socket(n=32):
+    """
+    Read CPU topology via lscpu and return a taskset -c string
+    selecting the last `n` cores of each NUMA node (socket).
+
+    Example: 2 sockets x 56 cores each, n=32 -> node0: 24-55, node1: 80-111
+    -> "taskset -c 24-55,80-111"
+    """
+    result = subprocess.run(["lscpu", "--parse=NODE,CPU"], capture_output=True, text=True)
+
+    # Build a dict: node_id -> sorted list of CPU ids
+    node_cpus = {}
+    for line in result.stdout.splitlines():
+        if line.startswith("#") or not line.strip():
+            continue
+        parts = line.split(",")
+        if len(parts) < 2:
+            continue
+        node_id, cpu_id = int(parts[0]), int(parts[1])
+        node_cpus.setdefault(node_id, []).append(cpu_id)
+
+    segments = []
+    for node_id in sorted(node_cpus):
+        cpus = sorted(node_cpus[node_id])
+        selected = cpus[-n:]          # last n cores of this socket
+        segments.append(f"{selected[0]}-{selected[-1]}")
+
+    cpu_str = ",".join(segments)
+    total = len(segments) * n
+    print(f" CPU binding: taskset -c {cpu_str}  ({total} cores, last {n} per socket)")
+    #return f"taskset -c {cpu_str}"
+    return f""
+
+
+## CPU core binding: dynamically select the last 32 cores of each socket (64 cores total)
+NUMACTL_CPU_BIND = get_last_n_cores_per_socket(n=32)
+
+## Build parallelism: match the number of bound cores
+BUILD_JOBS = 64
+
+
+##################################################################
+
+def _truthy(value, default=False):
+    if value is None:
+        return default
+    if isinstance(value, bool):
+        return value
+    text = str(value).strip().lower()
+    if text == "":
+        return default
+    return text in ("1", "yes", "y", "true", "on", "enable", "enabled")
+
+
+def _input_or_env(input_name, env_name, default=None):
+    if env_name in os.environ:
+        return os.environ[env_name]
+    return getattr(input_data, input_name, default)
+
+
+def _start_cuda_mps_if_requested(runtime_env):
+    if input_data.GPU_Calculation != "yes":
+        return False
+
+    default_auto_mps = int(getattr(input_data, "MPI_processes", 1)) > 1
+    auto_mps = _truthy(
+        _input_or_env("CUDA_Auto_MPS", "AMSS_CUDA_AUTO_MPS", default_auto_mps),
+        default=default_auto_mps,
+    )
+    if not auto_mps:
+        return False
+
+    mps_control = shutil.which("nvidia-cuda-mps-control")
+    if not mps_control:
+        print(" CUDA MPS control command was not found; running without MPS.")
+        return False
+
+    uid = os.getuid()
+    pipe_dir = str(_input_or_env("CUDA_MPS_PIPE_DIRECTORY", "CUDA_MPS_PIPE_DIRECTORY",
+                                 f"/tmp/amss-ncku-mps-{uid}"))
+    log_dir = str(_input_or_env("CUDA_MPS_LOG_DIRECTORY", "CUDA_MPS_LOG_DIRECTORY",
+                                f"/tmp/amss-ncku-mps-log-{uid}"))
+    os.makedirs(pipe_dir, exist_ok=True)
+    os.makedirs(log_dir, exist_ok=True)
+
+    mps_env = runtime_env.copy()
+    mps_env["CUDA_MPS_PIPE_DIRECTORY"] = pipe_dir
+    mps_env["CUDA_MPS_LOG_DIRECTORY"] = log_dir
+
+    if os.path.exists(os.path.join(pipe_dir, "control")):
+        runtime_env.update({
+            "CUDA_MPS_PIPE_DIRECTORY": pipe_dir,
+            "CUDA_MPS_LOG_DIRECTORY": log_dir,
+        })
+        print(f" Reusing CUDA MPS daemon: {pipe_dir}")
+        return False
+
+    print(f" Starting CUDA MPS daemon for this run: {pipe_dir}")
+    result = subprocess.run([mps_control, "-d"], env=mps_env, text=True,
+                            stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+    if result.returncode != 0:
+        print(" CUDA MPS daemon did not start; running without MPS.")
+        if result.stdout:
+            print(result.stdout, end="")
+        return False
+
+    runtime_env.update({
+        "CUDA_MPS_PIPE_DIRECTORY": pipe_dir,
+        "CUDA_MPS_LOG_DIRECTORY": log_dir,
+    })
+    return True
+
+
+def _stop_cuda_mps(runtime_env):
+    mps_control = shutil.which("nvidia-cuda-mps-control")
+    if not mps_control:
+        return
+    subprocess.run([mps_control], input="quit\n", env=runtime_env, text=True,
+                   stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+
+
+def _gpu_runtime_env():
+    runtime_env = os.environ.copy()
+
+    defaults = {
+        "AMSS_INTERP_FAST": "1",
+        "AMSS_INTERP_GPU": "1",
+        "AMSS_ANALYSIS_MAP_EVERY": "1000000",
+        "AMSS_CUDA_AWARE_MPI": "1",
+        "AMSS_CUDA_KEEP_RESIDENT_AFTER_STEP": "1",
+        "AMSS_CUDA_Z4C_KEEP_RESIDENT_AFTER_STEP": "1",
+        "AMSS_CUDA_KEEP_ALL_LEVELS": "1",
+        "AMSS_CUDA_Z4C_AMR_DEVICE": "0",
+        "AMSS_CUDA_AMR_RESTRICT_DEVICE": "1",
+        "AMSS_CUDA_AMR_RESTRICT_BATCH": "0",
+        "AMSS_CUDA_DEVICE_SEGMENT_BATCH": "0",
+        "AMSS_CUDA_PIN_ESCALAR_TRANSFERS": "0",
+        "AMSS_ESCALAR_GPU_RK": "0",
+    }
+    if getattr(input_data, "Equation_Class", "") == "Z4C":
+        defaults["AMSS_CUDA_Z4C_KEEP_RESIDENT_AFTER_STEP"] = "0"
+        defaults["AMSS_CUDA_KEEP_ALL_LEVELS"] = "0"
+    for key, value in defaults.items():
+        runtime_env.setdefault(key, value)
+
+    optional_overrides = {
+        "AMSS_INTERP_FAST_COMPARE": "AMSS_Interp_Fast_Compare",
+        "AMSS_INTERP_FAST_COMPARE_LIMIT": "AMSS_Interp_Fast_Compare_Limit",
+        "AMSS_INTERP_FAST_COMPARE_TOL": "AMSS_Interp_Fast_Compare_Tol",
+        "AMSS_GPU_STAGE_TIMING": "AMSS_GPU_Stage_Timing",
+        "AMSS_GPU_STAGE_TIMING_EVERY": "AMSS_GPU_Stage_Timing_Every",
+    }
+    for env_name, input_name in optional_overrides.items():
+        if env_name not in runtime_env and hasattr(input_data, input_name):
+            runtime_env[env_name] = str(getattr(input_data, input_name))
+
+    return runtime_env
 
 
 ##################################################################
@@ -26,11 +188,11 @@ def makefile_ABE():
     print( " Compiling the AMSS-NCKU executable file ABE/ABEGPU " ) 
     print(                                                        )
 
-    ## Build command
+    ## Build command with CPU binding to nohz_full cores
     if (input_data.GPU_Calculation == "no"):
-        makefile_command  = "make -j4" + " ABE"
+        makefile_command  = f"{NUMACTL_CPU_BIND} make -j{BUILD_JOBS} INTERP_LB_MODE=off USE_CUDA_BSSN=0 USE_CUDA_Z4C=0 ABE"
     elif (input_data.GPU_Calculation == "yes"):
-        makefile_command  = "make -j4" + " ABEGPU"
+        makefile_command  = f"{NUMACTL_CPU_BIND} make -j{BUILD_JOBS} INTERP_LB_MODE=off USE_CUDA_BSSN=1 USE_CUDA_Z4C=1 ABE_CUDA"
     else:
         print( " CPU/GPU numerical calculation setting is wrong " )
         print(                                                    )
@@ -67,8 +229,8 @@ def makefile_TwoPunctureABE():
     print( " Compiling the AMSS-NCKU executable file TwoPunctureABE " )
     print(                                                            )
     
-    ## Build command
-    makefile_command = "make" + " TwoPunctureABE"
+    ## Build command with CPU binding to nohz_full cores
+    makefile_command = f"{NUMACTL_CPU_BIND} make -j{BUILD_JOBS} TwoPunctureABE"
 
     ## Execute the command with subprocess.Popen and stream output
     makefile_process = subprocess.Popen(makefile_command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True) 
@@ -103,28 +265,55 @@ def run_ABE():
     print(                                                      )
 
     ## Define the command to run; cast other values to strings as needed
+    mpi_env = None
+    started_mps = False
     
     if (input_data.GPU_Calculation == "no"):
-        mpi_command         = "mpirun -np " + str(input_data.MPI_processes) + " ./ABE"
+        mpi_command         = NUMACTL_CPU_BIND + " mpirun -np " + str(input_data.MPI_processes) + " ./ABE"
+        #mpi_command         = " mpirun -np " + str(input_data.MPI_processes) + " ./ABE"
         mpi_command_outfile = "ABE_out.log"
     elif (input_data.GPU_Calculation == "yes"):
-        mpi_command         = "mpirun -np " + str(input_data.MPI_processes) + " ./ABEGPU"
+        mpi_command         = NUMACTL_CPU_BIND + " mpirun -np " + str(input_data.MPI_processes) + " ./ABE_CUDA"
         mpi_command_outfile = "ABEGPU_out.log"
+        mpi_env = _gpu_runtime_env()
+        started_mps = _start_cuda_mps_if_requested(mpi_env)
+        print(" GPU optimized runtime switches:")
+        print(f"   AMSS_INTERP_FAST={mpi_env.get('AMSS_INTERP_FAST', '')}")
+        print(f"   AMSS_INTERP_GPU={mpi_env.get('AMSS_INTERP_GPU', '')}")
+        print(f"   AMSS_ANALYSIS_MAP_EVERY={mpi_env.get('AMSS_ANALYSIS_MAP_EVERY', '')}")
+        print(f"   AMSS_CUDA_AWARE_MPI={mpi_env.get('AMSS_CUDA_AWARE_MPI', '')}")
+        print(f"   AMSS_CUDA_KEEP_RESIDENT_AFTER_STEP={mpi_env.get('AMSS_CUDA_KEEP_RESIDENT_AFTER_STEP', '')}")
+        print(f"   AMSS_CUDA_Z4C_KEEP_RESIDENT_AFTER_STEP={mpi_env.get('AMSS_CUDA_Z4C_KEEP_RESIDENT_AFTER_STEP', '')}")
+        print(f"   AMSS_CUDA_KEEP_ALL_LEVELS={mpi_env.get('AMSS_CUDA_KEEP_ALL_LEVELS', '')}")
+        print(f"   AMSS_CUDA_Z4C_AMR_DEVICE={mpi_env.get('AMSS_CUDA_Z4C_AMR_DEVICE', '')}")
+        print(f"   AMSS_CUDA_AMR_RESTRICT_DEVICE={mpi_env.get('AMSS_CUDA_AMR_RESTRICT_DEVICE', '')}")
+        print(f"   AMSS_CUDA_AMR_RESTRICT_BATCH={mpi_env.get('AMSS_CUDA_AMR_RESTRICT_BATCH', '')}")
+        print(f"   AMSS_CUDA_DEVICE_SEGMENT_BATCH={mpi_env.get('AMSS_CUDA_DEVICE_SEGMENT_BATCH', '')}")
+        print(f"   AMSS_CUDA_PIN_ESCALAR_TRANSFERS={mpi_env.get('AMSS_CUDA_PIN_ESCALAR_TRANSFERS', '')}")
+        print(f"   AMSS_ESCALAR_GPU_RK={mpi_env.get('AMSS_ESCALAR_GPU_RK', '')}")
+        if "CUDA_MPS_PIPE_DIRECTORY" in mpi_env:
+            print(f"   CUDA_MPS_PIPE_DIRECTORY={mpi_env['CUDA_MPS_PIPE_DIRECTORY']}")
  
-    ## Execute the MPI command and stream output
-    mpi_process = subprocess.Popen(mpi_command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)
+    try:
+        ## Execute the MPI command and stream output
+        mpi_process = subprocess.Popen(mpi_command, shell=True, stdout=subprocess.PIPE,
+                                       stderr=subprocess.STDOUT, text=True, env=mpi_env)
 
-    ## Write ABE run output to file while printing to stdout
-    with open(mpi_command_outfile, 'w') as file0:  
-        ## Read and print output lines; also write each line to file
-        for line in mpi_process.stdout:
-            print(line, end='')  # stream output in real time
-            file0.write(line)    # write the line to file
-            file0.flush()        # flush to ensure each line is written immediately (optional)            
-    file0.close()
+        ## Write ABE run output to file while printing to stdout
+        with open(mpi_command_outfile, 'w') as file0:
+            ## Read and print output lines; also write each line to file
+            for line in mpi_process.stdout:
+                print(line, end='')  # stream output in real time
+                file0.write(line)    # write the line to file
+                file0.flush()        # flush to ensure each line is written immediately (optional)
 
-    ## Wait for the process to finish
-    mpi_return_code = mpi_process.wait()
+        ## Wait for the process to finish
+        mpi_return_code = mpi_process.wait()
+        if mpi_return_code != 0:
+            raise subprocess.CalledProcessError(mpi_return_code, mpi_command)
+    finally:
+        if started_mps:
+            _stop_cuda_mps(mpi_env)
     
     print(                                           )
     print( " The ABE/ABEGPU simulation is finished " ) 
@@ -141,13 +330,14 @@ def run_ABE():
 ## Run the AMSS-NCKU TwoPuncture program TwoPunctureABE
 
 def run_TwoPunctureABE():
-
+    tp_time1=time.time()
     print(                                                          )
     print( " Running the AMSS-NCKU executable file TwoPunctureABE " ) 
     print(                                                          )
     
     ## Define the command to run
-    TwoPuncture_command         = "./TwoPunctureABE"
+    #TwoPuncture_command         = NUMACTL_CPU_BIND + " ./TwoPunctureABE"
+    TwoPuncture_command         = " ./TwoPunctureABE"
     TwoPuncture_command_outfile = "TwoPunctureABE_out.log"
 
     ## Execute the command with subprocess.Popen and stream output
@@ -168,7 +358,9 @@ def run_TwoPunctureABE():
     print(                                               )
     print( " The TwoPunctureABE simulation is finished " ) 
     print(                                               )
-    
+    tp_time2=time.time()
+    et=tp_time2-tp_time1
+    print(f"Used time: {et}")
     return
 
 ##################################################################

parallel_plot_helper.py

@@ -0,0 +1,29 @@
+import multiprocessing
+
+def run_plot_task(task):
+    """Execute a single plotting task.
+    
+    Parameters
+    ----------
+    task : tuple
+        A tuple of (function, args_tuple) where function is a callable
+        plotting function and args_tuple contains its arguments.
+    """
+    func, args = task
+    return func(*args)
+
+
+def run_plot_tasks_parallel(plot_tasks):
+    """Execute a list of independent plotting tasks in parallel.
+
+    Uses the 'fork' context to create worker processes so that the main
+    script is NOT re-imported/re-executed in child processes.
+
+    Parameters
+    ----------
+    plot_tasks : list of tuples
+        Each element is (function, args_tuple).
+    """
+    ctx = multiprocessing.get_context('fork')
+    with ctx.Pool() as pool:
+        pool.map(run_plot_task, plot_tasks)

plot_GW_strain_amplitude_xiaoqu.py

@@ -11,6 +11,8 @@
 import numpy                               ## numpy for array operations
 import scipy                               ## scipy for interpolation and signal processing
 import math
+import matplotlib
+matplotlib.use('Agg')                      ## use non-interactive backend for multiprocessing safety
 import matplotlib.pyplot    as     plt     ## matplotlib for plotting
 import os                                  ## os for system/file operations
 

plot_binary_data.py

@@ -8,16 +8,23 @@
 ##
 #################################################
 
+## Restrict OpenMP to one thread per process so that running
+## many workers in parallel does not create an O(workers * BLAS_threads)
+## thread explosion.  The variable MUST be set before numpy/scipy
+## are imported, because the BLAS library reads them only at load time.
+import os
+os.environ.setdefault("OMP_NUM_THREADS",        "1")
+
 import numpy
 import scipy
+import matplotlib
+matplotlib.use('Agg')                      ## use non-interactive backend for multiprocessing safety
 import matplotlib.pyplot    as     plt
 from   matplotlib.colors    import LogNorm
 from   mpl_toolkits.mplot3d import Axes3D
 ## import torch
 import AMSS_NCKU_Input      as input_data
 
-import os
-
 
 #########################################################################################
 
@@ -192,3 +199,19 @@ def get_data_xy( Rmin, Rmax, n, data0, time, figure_title, figure_outdir ):
 
 ####################################################################################
 
+
+####################################################################################
+## Allow this module to be run as a standalone script so that each
+## binary-data plot can be executed in a fresh subprocess whose BLAS
+## environment variables (set above) take effect before numpy loads.
+##
+## Usage:  python3 plot_binary_data.py <filename> <binary_outdir> <figure_outdir>
+####################################################################################
+
+if __name__ == '__main__':
+    import sys
+    if len(sys.argv) != 4:
+        print(f"Usage: {sys.argv[0]} <filename> <binary_outdir> <figure_outdir>")
+        sys.exit(1)
+    plot_binary_data(sys.argv[1], sys.argv[2], sys.argv[3])
+

plot_xiaoqu.py

@@ -8,6 +8,8 @@
 #################################################
 
 import numpy                               ## numpy for array operations
+import matplotlib
+matplotlib.use('Agg')                      ## use non-interactive backend for multiprocessing safety
 import matplotlib.pyplot    as     plt     ## matplotlib for plotting
 from   mpl_toolkits.mplot3d import Axes3D  ## needed for 3D plots
 import glob
@@ -15,6 +17,9 @@ import os                                  ## operating system utilities
 
 import plot_binary_data
 import AMSS_NCKU_Input as input_data
+import subprocess
+import sys
+import multiprocessing
 
 # plt.rcParams['text.usetex'] = True  ## enable LaTeX fonts in plots
 
@@ -50,10 +55,40 @@ def generate_binary_data_plot( binary_outdir, figure_outdir ):
         file_list.append(x)
         print(x)
 
-    ## Plot each file in the list
+    ## Plot each file in parallel using subprocesses.
+    ## Each subprocess is a fresh Python process where the BLAS thread-count
+    ## environment variables (set at the top of plot_binary_data.py) take
+    ## effect before numpy is imported.  This avoids the thread explosion
+    ## that occurs when multiprocessing.Pool with 'fork' context inherits
+    ## already-initialized multi-threaded BLAS from the parent.
+    script = os.path.join( os.path.dirname(__file__), "plot_binary_data.py" )
+    max_workers = min( multiprocessing.cpu_count(), len(file_list) ) if file_list else 0
+
+    running = []
+    failed  = []
     for filename in file_list:
         print(filename)
-        plot_binary_data.plot_binary_data(filename, binary_outdir, figure_outdir)
+        proc = subprocess.Popen(
+            [sys.executable, script, filename, binary_outdir, figure_outdir],
+        )
+        running.append( (proc, filename) )
+        ## Keep at most max_workers subprocesses active at a time
+        if len(running) >= max_workers:
+            p, fn = running.pop(0)
+            p.wait()
+            if p.returncode != 0:
+                failed.append(fn)
+
+    ## Wait for all remaining subprocesses to finish
+    for p, fn in running:
+        p.wait()
+        if p.returncode != 0:
+            failed.append(fn)
+
+    if failed:
+        print( " WARNING: the following binary data plots failed:" )
+        for fn in failed:
+            print( "   ", fn )
 
     print(                        )
     print( " Binary Data Plot Has been Finished " )