[TEST]UPSTREAM: Pick some source changes from 48080d0a97

* Sync new folder structure
2026-04-23 20:55:40 +08:00
parent c185f99ee3
commit 17109fde9b
211 changed files with 189504 additions and 189280 deletions
--- a/AMSS_NCKU_source/Parallel/Parallel.C
+++ b/AMSS_NCKU_source/Parallel/Parallel.C
--- a/AMSS_NCKU_source/Parallel/Parallel.h
+++ b/AMSS_NCKU_source/Parallel/Parallel.h
@@ -0,0 +1,231 @@
+
+#ifndef PARALLEL_H
+#define PARALLEL_H
+
+#include <iostream>
+#include <iomanip>
+#include <fstream>
+#include <cstdlib>
+#include <cstdio>
+#include <string>
+#include <cmath>
+#include <new>
+using namespace std;
+
+#include "Parallel_bam.h"
+#include "var.h"
+#include "MPatch.h"
+#include "Block.h"
+#include "MyList.h"
+#include "macrodef.h" //need dim; ghost_width; CONTRACT
+namespace Parallel
+{
+  struct gridseg
+  {
+    double llb[dim];
+    double uub[dim];
+    int shape[dim];
+    double illb[dim], iuub[dim]; // only use for OutBdLow2Hi
+    Block *Bg;
+  };
+  int partition1(int &nx, int split_size, int min_width, int cpusize, int shape);    // special for 1 diemnsion
+  int partition2(int *nxy, int split_size, int *min_width, int cpusize, int *shape); // special for 2 diemnsions
+  int partition3(int *nxyz, int split_size, int *min_width, int cpusize, int *shape);
+  MyList<Block> *distribute(MyList<Patch> *PatchLIST, int cpusize, int ingfsi, int fngfs, bool periodic, int nodes = 0); // produce corresponding Blocks
+  MyList<Block> *distribute_optimize(MyList<Patch> *PatchLIST, int cpusize, int ingfsi, int fngfs, bool periodic, int nodes = 0);
+  Block* splitHotspotBlock(MyList<Block>* &BlL, int _dim,
+                           int ib0_orig, int ib3_orig,
+                           int jb1_orig, int jb4_orig,
+                           int kb2_orig, int kb5_orig,
+                           Patch* PP, int r_left, int r_right,
+                           int ingfsi, int fngfsi, bool periodic,
+                           Block* &split_first_block, Block* &split_last_block);
+  Block* createMappedBlock(MyList<Block>* &BlL, int _dim, int* shape, double* bbox,
+                           int block_id, int ingfsi, int fngfsi, int lev);
+  void KillBlocks(MyList<Patch> *PatchLIST);
+
+  void setfunction(MyList<Block> *BlL, var *vn, double func(double x, double y, double z));
+  void setfunction(int rank, MyList<Block> *BlL, var *vn, double func(double x, double y, double z));
+  void writefile(double time, int nx, int ny, int nz, double xmin, double xmax, double ymin, double ymax,
+                 double zmin, double zmax, char *filename, double *data_out);
+  void writefile(double time, int nx, int ny, double xmin, double xmax, double ymin, double ymax,
+                 char *filename, double *datain);
+  void getarrayindex(int DIM, int *shape, int *index, int n);
+  int getarraylocation(int DIM, int *shape, int *index);
+  void copy(int DIM, double *llbout, double *uubout, int *Dshape, double *DD, double *llbin, double *uubin,
+            int *shape, double *datain, double *llb, double *uub);
+  void Dump_CPU_Data(MyList<Block> *BlL, MyList<var> *DumpList, char *tag, double time, double dT);
+  void Dump_Data(MyList<Patch> *PL, MyList<var> *DumpList, char *tag, double time, double dT);
+  void Dump_Data(Patch *PP, MyList<var> *DumpList, char *tag, double time, double dT, int grd);
+  double *Collect_Data(Patch *PP, var *VP);
+  void d2Dump_Data(MyList<Patch> *PL, MyList<var> *DumpList, char *tag, double time, double dT);
+  void d2Dump_Data(Patch *PP, MyList<var> *DumpList, char *tag, double time, double dT, int grd);
+  void Dump_Data0(Patch *PP, MyList<var> *DumpList, char *tag, double time, double dT);
+  double global_interp(int DIM, int *ext, double **CoX, double *datain,
+                       double *poX, int ordn, double *SoA, int Symmetry);
+  double global_interp(int DIM, int *ext, double **CoX, double *datain,
+                       double *poX, int ordn);
+  double Lagrangian_Int(double x, int npts, double *xpts, double *funcvals);
+  double LagrangePoly(double x, int pt, int npts, double *xpts);
+  MyList<gridseg> *build_complete_gsl(Patch *Pat);
+  MyList<gridseg> *build_complete_gsl(MyList<Patch> *PatL);
+  MyList<gridseg> *build_complete_gsl_virtual(MyList<Patch> *PatL);
+  MyList<gridseg> *build_complete_gsl_virtual2(MyList<Patch> *PatL);        // - buffer
+  MyList<gridseg> *build_owned_gsl0(Patch *Pat, int rank_in);               // - ghost without extension, special for Sync usage
+  MyList<gridseg> *build_owned_gsl1(Patch *Pat, int rank_in);               // - ghost, similar to build_owned_gsl0 but extend one point on left side for vertex grid
+  MyList<gridseg> *build_owned_gsl2(Patch *Pat, int rank_in);               // - buffer - ghost
+  MyList<gridseg> *build_owned_gsl3(Patch *Pat, int rank_in, int Symmetry); // - ghost - BD ghost
+  MyList<gridseg> *build_owned_gsl4(Patch *Pat, int rank_in, int Symmetry); // - buffer - ghost - BD ghost
+  MyList<gridseg> *build_owned_gsl5(Patch *Pat, int rank_in);               // similar to build_owned_gsl2 but no extension
+  MyList<gridseg> *build_owned_gsl(MyList<Patch> *PatL, int rank_in, int type, int Symmetry);
+  void build_gstl(MyList<gridseg> *srci, MyList<gridseg> *dsti, MyList<gridseg> **out_src, MyList<gridseg> **out_dst);
+  int data_packer(double *data, MyList<gridseg> *src, MyList<gridseg> *dst, int rank_in, int dir,
+                  MyList<var> *VarLists, MyList<var> *VarListd, int Symmetry);
+  void transfer(MyList<gridseg> **src, MyList<gridseg> **dst,
+                MyList<var> *VarList1 /* source */, MyList<var> *VarList2 /*target */,
+                int Symmetry);
+  int data_packermix(double *data, MyList<gridseg> *src, MyList<gridseg> *dst, int rank_in, int dir,
+                     MyList<var> *VarLists, MyList<var> *VarListd, int Symmetry);
+  void transfermix(MyList<gridseg> **src, MyList<gridseg> **dst,
+                   MyList<var> *VarList1 /* source */, MyList<var> *VarList2 /*target */,
+                   int Symmetry);
+  void Sync(Patch *Pat, MyList<var> *VarList, int Symmetry);
+  void Sync(MyList<Patch> *PatL, MyList<var> *VarList, int Symmetry);
+  void Sync_merged(MyList<Patch> *PatL, MyList<var> *VarList, int Symmetry);
+
+  struct SyncCache {
+    bool valid;
+    int cpusize;
+    MyList<gridseg> **combined_src;
+    MyList<gridseg> **combined_dst;
+    int *send_lengths;
+    int *recv_lengths;
+    double **send_bufs;
+    double **recv_bufs;
+    int *send_buf_caps;
+    int *recv_buf_caps;
+    MPI_Request *reqs;
+    MPI_Status *stats;
+    int max_reqs;
+    bool lengths_valid;
+    int *tc_req_node;
+    int *tc_req_is_recv;
+    int *tc_completed;
+    SyncCache();
+    void invalidate();
+    void destroy();
+  };
+
+  void Sync_cached(MyList<Patch> *PatL, MyList<var> *VarList, int Symmetry, SyncCache &cache);
+  void transfer_cached(MyList<gridseg> **src, MyList<gridseg> **dst,
+                       MyList<var> *VarList1, MyList<var> *VarList2,
+                       int Symmetry, SyncCache &cache);
+
+  struct AsyncSyncState {
+    int req_no;
+    bool active;
+    int *req_node;
+    int *req_is_recv;
+    int pending_recv;
+    AsyncSyncState() : req_no(0), active(false), req_node(0), req_is_recv(0), pending_recv(0) {}
+  };
+
+  void Sync_start(MyList<Patch> *PatL, MyList<var> *VarList, int Symmetry,
+                  SyncCache &cache, AsyncSyncState &state);
+  void Sync_finish(SyncCache &cache, AsyncSyncState &state,
+                   MyList<var> *VarList, int Symmetry);
+  void OutBdLow2Hi(Patch *Patc, Patch *Patf,
+                   MyList<var> *VarList1 /* source */, MyList<var> *VarList2 /* target */,
+                   int Symmetry);
+  void OutBdLow2Hi(MyList<Patch> *PatcL, MyList<Patch> *PatfL,
+                   MyList<var> *VarList1 /* source */, MyList<var> *VarList2 /* target */,
+                   int Symmetry);
+  void OutBdLow2Himix(Patch *Patc, Patch *Patf,
+                      MyList<var> *VarList1 /* source */, MyList<var> *VarList2 /* target */,
+                      int Symmetry);
+  void OutBdLow2Himix(MyList<Patch> *PatcL, MyList<Patch> *PatfL,
+                      MyList<var> *VarList1 /* source */, MyList<var> *VarList2 /* target */,
+                      int Symmetry);
+  void Restrict_cached(MyList<Patch> *PatcL, MyList<Patch> *PatfL,
+                       MyList<var> *VarList1, MyList<var> *VarList2,
+                       int Symmetry, SyncCache &cache);
+  void OutBdLow2Hi_cached(MyList<Patch> *PatcL, MyList<Patch> *PatfL,
+                          MyList<var> *VarList1, MyList<var> *VarList2,
+                          int Symmetry, SyncCache &cache);
+  void OutBdLow2Himix_cached(MyList<Patch> *PatcL, MyList<Patch> *PatfL,
+                             MyList<var> *VarList1, MyList<var> *VarList2,
+                             int Symmetry, SyncCache &cache);
+  void Prolong(Patch *Patc, Patch *Patf,
+               MyList<var> *VarList1 /* source */, MyList<var> *VarList2 /* target */,
+               int Symmetry);
+  void Prolongint(Patch *Patc, Patch *Patf,
+                  MyList<var> *VarList1 /* source */, MyList<var> *VarList2 /* target */,
+                  int Symmetry);
+  void Restrict(MyList<Patch> *PatcL, MyList<Patch> *PatfL,
+                MyList<var> *VarList1 /* source */, MyList<var> *VarList2 /* target */,
+                int Symmetry);
+  void Restrict_after(MyList<Patch> *PatcL, MyList<Patch> *PatfL,
+                      MyList<var> *VarList1 /* source */, MyList<var> *VarList2 /* target */,
+                      int Symmetry); // for -ghost - BDghost
+  MyList<Parallel::gridseg> *build_PhysBD_gsl(Patch *Pat);
+  MyList<Parallel::gridseg> *build_ghost_gsl(MyList<Patch> *PatL);
+  MyList<Parallel::gridseg> *build_ghost_gsl(Patch *Pat);
+  MyList<Parallel::gridseg> *build_buffer_gsl(Patch *Pat);
+  MyList<Parallel::gridseg> *build_buffer_gsl(MyList<Patch> *PatL);
+  MyList<Parallel::gridseg> *gsl_subtract(MyList<Parallel::gridseg> *A, MyList<Parallel::gridseg> *B);
+  MyList<Parallel::gridseg> *gs_subtract(MyList<Parallel::gridseg> *A, MyList<Parallel::gridseg> *B);
+  MyList<Parallel::gridseg> *gsl_and(MyList<Parallel::gridseg> *A, MyList<Parallel::gridseg> *B);
+  MyList<Parallel::gridseg> *gs_and(MyList<Parallel::gridseg> *A, MyList<Parallel::gridseg> *B);
+  MyList<Parallel::gridseg> *clone_gsl(MyList<Parallel::gridseg> *p, bool first_only);
+  MyList<Parallel::gridseg> *build_bulk_gsl(Patch *Pat); // similar to build_owned_gsl0 but does not care rank issue
+  MyList<Parallel::gridseg> *build_bulk_gsl(Block *bp, Patch *Pat);
+  void build_PhysBD_gstl(Patch *Pat, MyList<Parallel::gridseg> *srci, MyList<Parallel::gridseg> *dsti,
+                         MyList<Parallel::gridseg> **out_src, MyList<Parallel::gridseg> **out_dst);
+  void PeriodicBD(Patch *Pat, MyList<var> *VarList, int Symmetry);
+  double L2Norm(Patch *Pat, var *vf);
+  void L2Norm7(Patch *Pat, var **vf, double *norms);
+  void checkgsl(MyList<Parallel::gridseg> *pp, bool first_only);
+  void checkvarl(MyList<var> *pp, bool first_only);
+  MyList<Parallel::gridseg> *divide_gsl(MyList<Parallel::gridseg> *p, Patch *Pat);
+  MyList<Parallel::gridseg> *divide_gs(MyList<Parallel::gridseg> *p, Patch *Pat);
+  void prepare_inter_time_level(Patch *Pat,
+                                MyList<var> *VarList1 /* source (t+dt) */, MyList<var> *VarList2 /* source (t) */,
+                                MyList<var> *VarList3 /* target (t+a*dt) */, int tindex);
+  void prepare_inter_time_level(Patch *Pat,
+                                MyList<var> *VarList1 /* source (t+dt) */, MyList<var> *VarList2 /* source (t) */,
+                                MyList<var> *VarList3 /* source (t-dt) */, MyList<var> *VarList4 /* target (t+a*dt) */, int tindex);
+  void prepare_inter_time_level(MyList<Patch> *PatL,
+                                MyList<var> *VarList1 /* source (t+dt) */, MyList<var> *VarList2 /* source (t) */,
+                                MyList<var> *VarList3 /* target (t+a*dt) */, int tindex);
+  void prepare_inter_time_level(MyList<Patch> *Pat,
+                                MyList<var> *VarList1 /* source (t+dt) */, MyList<var> *VarList2 /* source (t) */,
+                                MyList<var> *VarList3 /* source (t-dt) */, MyList<var> *VarList4 /* target (t+a*dt) */, int tindex);
+  void merge_gsl(MyList<gridseg> *&A, const double ratio);
+  bool merge_gs(MyList<gridseg> *D, MyList<gridseg> *B, MyList<gridseg> *&C, const double ratio);
+  // Add ghost region to tangent plane
+  // we assume the grids have the same resolution
+  void add_ghost_touch(MyList<gridseg> *&A);
+  void cut_gsl(MyList<gridseg> *&A);
+  bool cut_gs(MyList<gridseg> *D, MyList<gridseg> *B, MyList<gridseg> *&C);
+  MyList<Parallel::gridseg> *gs_subtract_virtual(MyList<Parallel::gridseg> *A, MyList<Parallel::gridseg> *B);
+  void fill_level_data(MyList<Patch> *PatLd, MyList<Patch> *PatLs, MyList<Patch> *PatcL,
+                       MyList<var> *OldList, MyList<var> *StateList, MyList<var> *FutureList,
+                       MyList<var> *tmList, int Symmetry, bool BB, bool CC);
+  bool PatList_Interp_Points(MyList<Patch> *PatL, MyList<var> *VarList,
+                             int NN, double **XX,
+                             double *Shellf, int Symmetry);
+  void aligncheck(double *bbox0, double *bboxl, int lev, double *DH0, int *shape);
+  bool point_locat_gsl(double *pox, MyList<Parallel::gridseg> *gsl);
+  void checkpatchlist(MyList<Patch> *PatL, bool buflog);
+
+  double L2Norm(Patch *Pat, var *vf, MPI_Comm Comm_here);
+  void L2Norm7(Patch *Pat, var **vf, double *norms, MPI_Comm Comm_here);
+  bool PatList_Interp_Points(MyList<Patch> *PatL, MyList<var> *VarList,
+                             int NN, double **XX,
+                             double *Shellf, int Symmetry, MPI_Comm Comm_here);
+#if (PSTR == 1 || PSTR == 2 || PSTR == 3)
+  MyList<Block> *distribute(MyList<Patch> *PatchLIST, int cpusize, int ingfsi, int fngfsi,
+                            bool periodic, int start_rank, int end_rank, int nodes = 0);
+#endif
+}
+#endif /*PARALLEL_H */
--- a/AMSS_NCKU_source/Parallel/Parallel_bam.C
+++ b/AMSS_NCKU_source/Parallel/Parallel_bam.C
@@ -0,0 +1,662 @@
+
+#include "Parallel.h"
+#include "fmisc.h"
+#include "prolongrestrict.h"
+#include "misc.h"
+
+void Parallel::OutBdLow2Hi_bam(MyList<Patch> *PLc, MyList<Patch> *PLf,
+                               MyList<var> *VarList1 /* source */, MyList<var> *VarList2 /* target */,
+                               int Symmetry)
+{
+   MyList<Parallel::pointstru_bam> *bdsul;
+   Constr_pointstr_OutBdLow2Hi(PLf, PLc, bdsul);
+
+   intertransfer(bdsul, VarList1, VarList2, Symmetry);
+
+   destroypsuList_bam(bdsul);
+}
+void Parallel::Restrict_bam(MyList<Patch> *PLc, MyList<Patch> *PLf,
+                            MyList<var> *VarList1 /* source */, MyList<var> *VarList2 /* target */,
+                            int Symmetry)
+{
+   MyList<Parallel::pointstru_bam> *rsul;
+   Constr_pointstr_Restrict(PLf, PLc, rsul);
+
+   intertransfer(rsul, VarList1, VarList2, Symmetry);
+
+   destroypsuList_bam(rsul);
+}
+void Parallel::OutBdLow2Hi_bam(MyList<Patch> *PLc, MyList<Patch> *PLf,
+                               MyList<var> *VarList1 /* source */, MyList<var> *VarList2 /* target */,
+                               MyList<Parallel::pointstru_bam> *bdsul, int Symmetry)
+{
+   intertransfer(bdsul, VarList1, VarList2, Symmetry);
+}
+void Parallel::Restrict_bam(MyList<Patch> *PLc, MyList<Patch> *PLf,
+                            MyList<var> *VarList1 /* source */, MyList<var> *VarList2 /* target */,
+                            MyList<Parallel::pointstru_bam> *rsul, int Symmetry)
+{
+   intertransfer(rsul, VarList1, VarList2, Symmetry);
+}
+void Parallel::Constr_pointstr_OutBdLow2Hi(MyList<Patch> *PLf, MyList<Patch> *PLc,
+                                           MyList<Parallel::pointstru_bam> *&bdsul)
+{
+   MyList<Patch> *PL;
+
+   MyList<Parallel::pointstru_bam> *ps;
+   bdsul = 0;
+
+   // find out points
+   PL = PLf;
+   while (PL)
+   {
+      double dx, dy, dz;
+
+      dx = PL->data->blb->data->getdX(0);
+      dy = PL->data->blb->data->getdX(1);
+      dz = PL->data->blb->data->getdX(2);
+
+      double uub[3], llb[3];
+
+      llb[0] = PL->data->bbox[0] + PL->data->lli[0] * dx;
+      llb[1] = PL->data->bbox[1] + PL->data->lli[1] * dy;
+      llb[2] = PL->data->bbox[2] + PL->data->lli[2] * dz;
+      uub[0] = PL->data->bbox[3] - PL->data->uui[0] * dx;
+      uub[1] = PL->data->bbox[4] - PL->data->uui[1] * dy;
+      uub[2] = PL->data->bbox[5] - PL->data->uui[2] * dz;
+
+      double x, y, z;
+
+      for (int i = 0; i < PL->data->shape[0]; i++)
+      {
+#ifdef Vertex
+#ifdef Cell
+#error Both Cell and Vertex are defined
+#endif
+         x = PL->data->bbox[0] + i * dx;
+#else
+#ifdef Cell
+         x = PL->data->bbox[0] + (0.5 + i) * dx;
+#else
+#error Not define Vertex nor Cell
+#endif
+#endif
+         for (int j = 0; j < PL->data->shape[1]; j++)
+         {
+#ifdef Vertex
+#ifdef Cell
+#error Both Cell and Vertex are defined
+#endif
+            y = PL->data->bbox[1] + j * dy;
+#else
+#ifdef Cell
+            y = PL->data->bbox[1] + (0.5 + j) * dy;
+#else
+#error Not define Vertex nor Cell
+#endif
+#endif
+            for (int k = 0; k < PL->data->shape[2]; k++)
+            {
+#ifdef Vertex
+#ifdef Cell
+#error Both Cell and Vertex are defined
+#endif
+               z = PL->data->bbox[2] + k * dz;
+#else
+#ifdef Cell
+               z = PL->data->bbox[2] + (0.5 + k) * dz;
+#else
+#error Not define Vertex nor Cell
+#endif
+#endif
+               if (!(llb[0] - TINY < x && uub[0] + TINY > x &&
+                     llb[1] - TINY < y && uub[1] + TINY > y &&
+                     llb[2] - TINY < z && uub[2] + TINY > z)) // not in the inner part
+               {
+                  if (bdsul)
+                  {
+                     ps->next = new MyList<Parallel::pointstru_bam>;
+                     ps = ps->next;
+                     ps->data = new Parallel::pointstru_bam;
+                  }
+                  else
+                  {
+                     bdsul = ps = new MyList<Parallel::pointstru_bam>;
+                     ps->data = new Parallel::pointstru_bam;
+                  }
+
+                  ps->data->pox[0] = x;
+                  ps->data->pox[1] = y;
+                  ps->data->pox[2] = z;
+                  ps->data->Bgs = 0;
+                  ps->data->Bgd = 0;
+                  ps->data->coef = 0;
+
+                  ps->next = 0;
+               }
+            }
+         }
+      }
+
+      PL = PL->next;
+   }
+
+   // find out blocks
+   ps = bdsul;
+   while (ps)
+   {
+      double x, y, z;
+      x = ps->data->pox[0];
+      y = ps->data->pox[1];
+      z = ps->data->pox[2];
+      bool flag;
+      // find target block
+      flag = true;
+      PL = PLf;
+      while (flag && PL)
+      {
+         MyList<Block> *BP = PL->data->blb;
+         while (flag && BP)
+         {
+            double llb[3], uub[3];
+
+            for (int i = 0; i < dim; i++)
+            {
+               double DH = BP->data->getdX(i);
+               uub[i] = (feq(BP->data->bbox[dim + i], PL->data->bbox[dim + i], DH / 2)) ? BP->data->bbox[dim + i] : BP->data->bbox[dim + i] - ghost_width * DH;
+               llb[i] = (feq(BP->data->bbox[i], PL->data->bbox[i], DH / 2)) ? BP->data->bbox[i] : BP->data->bbox[i] + ghost_width * DH;
+            }
+
+            if (llb[0] - TINY < x && uub[0] + TINY > x &&
+                llb[1] - TINY < y && uub[1] + TINY > y &&
+                llb[2] - TINY < z && uub[2] + TINY > z)
+            {
+               ps->data->Bgd = BP->data;
+               flag = false;
+            }
+
+            if (BP == PL->data->ble)
+               break;
+            BP = BP->next;
+         }
+         PL = PL->next;
+      }
+      if (flag)
+      {
+         cout << "error in Parallel::Constr_pointstr_OutBdLow2Hi 2" << endl;
+         MPI_Abort(MPI_COMM_WORLD, 1);
+      }
+      // find source block
+      flag = true;
+      PL = PLc;
+      while (flag && PL)
+      {
+         MyList<Block> *BP = PL->data->blb;
+         while (flag && BP)
+         {
+            double llb[3], uub[3];
+
+            for (int i = 0; i < dim; i++)
+            {
+               double DH = BP->data->getdX(i);
+               uub[i] = (feq(BP->data->bbox[dim + i], PL->data->bbox[dim + i], DH / 2)) ? BP->data->bbox[dim + i] : BP->data->bbox[dim + i] - ghost_width * DH;
+               llb[i] = (feq(BP->data->bbox[i], PL->data->bbox[i], DH / 2)) ? BP->data->bbox[i] : BP->data->bbox[i] + ghost_width * DH;
+            }
+
+            if (llb[0] - TINY < x && uub[0] + TINY > x &&
+                llb[1] - TINY < y && uub[1] + TINY > y &&
+                llb[2] - TINY < z && uub[2] + TINY > z)
+            {
+               ps->data->Bgs = BP->data;
+               flag = false;
+            }
+
+            if (BP == PL->data->ble)
+               break;
+            BP = BP->next;
+         }
+         PL = PL->next;
+      }
+      if (flag)
+      {
+         cout << "error in Parallel::Constr_pointstr_OutBdLow2Hi 3" << endl;
+         MPI_Abort(MPI_COMM_WORLD, 1);
+      }
+
+      ps = ps->next;
+   }
+}
+void Parallel::Constr_pointstr_Restrict(MyList<Patch> *PLf, MyList<Patch> *PLc,
+                                        MyList<Parallel::pointstru_bam> *&rsul)
+{
+   MyList<Parallel::gridseg> *gdlf = 0, *gs;
+   MyList<Patch> *PL = PLf;
+   while (PL)
+   {
+      if (gdlf)
+      {
+         gs->next = new MyList<Parallel::gridseg>;
+         gs = gs->next;
+         gs->data = new Parallel::gridseg;
+      }
+      else
+      {
+         gdlf = gs = new MyList<Parallel::gridseg>;
+         gs->data = new Parallel::gridseg;
+      }
+
+      gs->next = 0;
+
+      for (int i = 0; i < dim; i++)
+      {
+         double DH = PL->data->blb->data->getdX(i);
+
+         gs->data->llb[i] = PL->data->bbox[i] + PL->data->lli[i] * DH;
+         gs->data->uub[i] = PL->data->bbox[dim + i] - PL->data->uui[i] * DH;
+      }
+
+      PL = PL->next;
+   }
+
+   MyList<Parallel::pointstru_bam> *ps;
+   rsul = 0;
+
+   // find out points
+   gs = gdlf;
+   while (gs)
+   {
+      PL = PLc;
+      bool flag = true;
+      while (flag)
+      {
+         if (!PL)
+         {
+            int myrank;
+            MPI_Comm_rank(MPI_COMM_WORLD, &myrank);
+            if (myrank == 0)
+            {
+               cout << "error in Parallel::Constr_pointstr_Restrict: fail to find grid segment [" << gs->data->llb[0] << ":" << gs->data->uub[0] << ","
+                    << gs->data->llb[1] << ":" << gs->data->uub[1] << ","
+                    << gs->data->llb[2] << ":" << gs->data->uub[2] << "]"
+                    << endl;
+               PL = PLc;
+               while (PL)
+               {
+                  PL->data->checkPatch(0);
+                  PL = PL->next;
+               }
+            }
+
+            misc::tillherecheck("for wait.");
+            MPI_Abort(MPI_COMM_WORLD, 1);
+         }
+         if (gs->data->llb[0] > PL->data->bbox[0] - TINY && gs->data->uub[0] < PL->data->bbox[3] + TINY &&
+             gs->data->llb[1] > PL->data->bbox[1] - TINY && gs->data->uub[1] < PL->data->bbox[4] + TINY &&
+             gs->data->llb[2] > PL->data->bbox[2] - TINY && gs->data->uub[2] < PL->data->bbox[5] + TINY)
+            flag = false;
+
+         if (flag)
+            PL = PL->next;
+      }
+
+      double dx, dy, dz;
+
+      dx = PL->data->blb->data->getdX(0);
+      dy = PL->data->blb->data->getdX(1);
+      dz = PL->data->blb->data->getdX(2);
+
+      double x, y, z;
+
+      for (int i = 0; i < PL->data->shape[0]; i++)
+      {
+#ifdef Vertex
+#ifdef Cell
+#error Both Cell and Vertex are defined
+#endif
+         x = PL->data->bbox[0] + i * dx;
+#else
+#ifdef Cell
+         x = PL->data->bbox[0] + (0.5 + i) * dx;
+#else
+#error Not define Vertex nor Cell
+#endif
+#endif
+         for (int j = 0; j < PL->data->shape[1]; j++)
+         {
+#ifdef Vertex
+#ifdef Cell
+#error Both Cell and Vertex are defined
+#endif
+            y = PL->data->bbox[1] + j * dy;
+#else
+#ifdef Cell
+            y = PL->data->bbox[1] + (0.5 + j) * dy;
+#else
+#error Not define Vertex nor Cell
+#endif
+#endif
+            for (int k = 0; k < PL->data->shape[2]; k++)
+            {
+#ifdef Vertex
+#ifdef Cell
+#error Both Cell and Vertex are defined
+#endif
+               z = PL->data->bbox[2] + k * dz;
+#else
+#ifdef Cell
+               z = PL->data->bbox[2] + (0.5 + k) * dz;
+#else
+#error Not define Vertex nor Cell
+#endif
+#endif
+               if (gs->data->llb[0] - TINY < x && gs->data->uub[0] + TINY > x &&
+                   gs->data->llb[1] - TINY < y && gs->data->uub[1] + TINY > y &&
+                   gs->data->llb[2] - TINY < z && gs->data->uub[2] + TINY > z) // in the inner part
+               {
+                  if (rsul)
+                  {
+                     ps->next = new MyList<Parallel::pointstru_bam>;
+                     ps = ps->next;
+                     ps->data = new Parallel::pointstru_bam;
+                  }
+                  else
+                  {
+                     rsul = ps = new MyList<Parallel::pointstru_bam>;
+                     ps->data = new Parallel::pointstru_bam;
+                  }
+
+                  ps->data->pox[0] = x;
+                  ps->data->pox[1] = y;
+                  ps->data->pox[2] = z;
+                  ps->data->Bgs = 0;
+                  ps->data->Bgd = 0;
+                  ps->data->coef = 0;
+
+                  ps->next = 0;
+               }
+            }
+         }
+      }
+
+      gs = gs->next;
+   }
+
+   gdlf->destroyList();
+
+   // find out blocks
+   ps = rsul;
+   while (ps)
+   {
+      double x, y, z;
+      x = ps->data->pox[0];
+      y = ps->data->pox[1];
+      z = ps->data->pox[2];
+      bool flag;
+      // find source block
+      flag = true;
+      PL = PLf;
+      while (flag && PL)
+      {
+         MyList<Block> *BP = PL->data->blb;
+         while (flag && BP)
+         {
+            double llb[3], uub[3];
+
+            for (int i = 0; i < dim; i++)
+            {
+               double DH = BP->data->getdX(i);
+               uub[i] = (feq(BP->data->bbox[dim + i], PL->data->bbox[dim + i], DH / 2)) ? BP->data->bbox[dim + i] : BP->data->bbox[dim + i] - ghost_width * DH;
+               llb[i] = (feq(BP->data->bbox[i], PL->data->bbox[i], DH / 2)) ? BP->data->bbox[i] : BP->data->bbox[i] + ghost_width * DH;
+            }
+
+            if (llb[0] - TINY < x && uub[0] + TINY > x &&
+                llb[1] - TINY < y && uub[1] + TINY > y &&
+                llb[2] - TINY < z && uub[2] + TINY > z)
+            {
+               ps->data->Bgs = BP->data;
+               flag = false;
+            }
+
+            if (BP == PL->data->ble)
+               break;
+            BP = BP->next;
+         }
+         PL = PL->next;
+      }
+      if (flag)
+      {
+         cout << "error in Parallel::Constr_pointstr_Restrict 2" << endl;
+         MPI_Abort(MPI_COMM_WORLD, 1);
+      }
+      // find target block
+      flag = true;
+      PL = PLc;
+      while (flag && PL)
+      {
+         MyList<Block> *BP = PL->data->blb;
+         while (flag && BP)
+         {
+            double llb[3], uub[3];
+
+            for (int i = 0; i < dim; i++)
+            {
+               double DH = BP->data->getdX(i);
+               uub[i] = (feq(BP->data->bbox[dim + i], PL->data->bbox[dim + i], DH / 2)) ? BP->data->bbox[dim + i] : BP->data->bbox[dim + i] - ghost_width * DH;
+               llb[i] = (feq(BP->data->bbox[i], PL->data->bbox[i], DH / 2)) ? BP->data->bbox[i] : BP->data->bbox[i] + ghost_width * DH;
+            }
+
+            if (llb[0] - TINY < x && uub[0] + TINY > x &&
+                llb[1] - TINY < y && uub[1] + TINY > y &&
+                llb[2] - TINY < z && uub[2] + TINY > z)
+            {
+               ps->data->Bgd = BP->data;
+               flag = false;
+            }
+
+            if (BP == PL->data->ble)
+               break;
+            BP = BP->next;
+         }
+         PL = PL->next;
+      }
+      if (flag)
+      {
+         cout << "error in Parallel::Constr_pointstr_Restrict 3" << endl;
+         MPI_Abort(MPI_COMM_WORLD, 1);
+      }
+
+      ps = ps->next;
+   }
+}
+
+void Parallel::intertransfer(MyList<Parallel::pointstru_bam> *&sul,
+                             MyList<var> *VarList1 /* source */, MyList<var> *VarList2 /*target */,
+                             int Symmetry)
+{
+   int myrank, cpusize;
+   MPI_Comm_size(MPI_COMM_WORLD, &cpusize);
+   MPI_Comm_rank(MPI_COMM_WORLD, &myrank);
+
+   int node;
+
+   MPI_Request *reqs;
+   MPI_Status *stats;
+   reqs = new MPI_Request[2 * cpusize];
+   stats = new MPI_Status[2 * cpusize];
+   int req_no = 0;
+
+   double **send_data, **rec_data;
+   send_data = new double *[cpusize];
+   rec_data = new double *[cpusize];
+   int length;
+
+   for (node = 0; node < cpusize; node++)
+   {
+      send_data[node] = rec_data[node] = 0;
+      if (node == myrank)
+      {
+         // myrank: local; node : remote
+         if (length = interdata_packer(0, sul, myrank, node, PACK, VarList1, VarList2, Symmetry))
+         {
+            rec_data[node] = new double[length];
+            if (!rec_data[node])
+            {
+               cout << "Parallel::intertransfer: out of memory when new in short transfer, place 1" << endl;
+               MPI_Abort(MPI_COMM_WORLD, 1);
+            }
+            interdata_packer(rec_data[node], sul, myrank, node, PACK, VarList1, VarList2, Symmetry);
+         }
+      }
+      else
+      {
+         // send from this cpu to cpu#node
+         if (length = interdata_packer(0, sul, myrank, node, PACK, VarList1, VarList2, Symmetry))
+         {
+            send_data[node] = new double[length];
+            if (!send_data[node])
+            {
+               cout << "Parallel::intertransfer: out of memory when new in short transfer, place 2" << endl;
+               MPI_Abort(MPI_COMM_WORLD, 1);
+            }
+            interdata_packer(send_data[node], sul, myrank, node, PACK, VarList1, VarList2, Symmetry);
+            MPI_Isend((void *)send_data[node], length, MPI_DOUBLE, node, 1, MPI_COMM_WORLD, reqs + req_no++);
+         }
+         // receive from cpu#node to this cpu
+         if (length = interdata_packer(0, sul, myrank, node, UNPACK, VarList1, VarList2, Symmetry))
+         {
+            rec_data[node] = new double[length];
+            if (!rec_data[node])
+            {
+               cout << "Parallel::intertransfer: out of memory when new in short transfer, place 3" << endl;
+               MPI_Abort(MPI_COMM_WORLD, 1);
+            }
+            MPI_Irecv((void *)rec_data[node], length, MPI_DOUBLE, node, 1, MPI_COMM_WORLD, reqs + req_no++);
+         }
+      }
+   }
+   // wait for all requests to complete
+   MPI_Waitall(req_no, reqs, stats);
+
+   for (node = 0; node < cpusize; node++)
+      if (rec_data[node])
+         interdata_packer(rec_data[node], sul, myrank, node, UNPACK, VarList1, VarList2, Symmetry);
+
+   for (node = 0; node < cpusize; node++)
+   {
+      if (send_data[node])
+         delete[] send_data[node];
+      if (rec_data[node])
+         delete[] rec_data[node];
+   }
+
+   delete[] reqs;
+   delete[] stats;
+   delete[] send_data;
+   delete[] rec_data;
+}
+//   PACK: prepare target data in 'data'
+// UNPACK: copy target data from 'data' to corresponding numerical grids
+int Parallel::interdata_packer(double *data, MyList<Parallel::pointstru_bam> *sul, int myrank, int node, int dir,
+                               MyList<var> *VarLists /* source */, MyList<var> *VarListd /* target */, int Symmetry)
+{
+   int DIM = dim;
+   int ordn = 2 * ghost_width;
+
+   if (dir != PACK && dir != UNPACK)
+   {
+      cout << "Parallel::interdata_packer: error dir " << dir << " for data_packer " << endl;
+      MPI_Abort(MPI_COMM_WORLD, 1);
+   }
+
+   int size_out = 0;
+
+   MyList<var> *varls, *varld;
+
+   varls = VarLists;
+   varld = VarListd;
+   while (varls && varld)
+   {
+      varls = varls->next;
+      varld = varld->next;
+   }
+
+   if (varls || varld)
+   {
+      cout << "error in short data packer, var lists does not match." << endl;
+      MPI_Abort(MPI_COMM_WORLD, 1);
+   }
+
+   while (sul)
+   {
+      if ((dir == PACK && sul->data->Bgs->rank == myrank && sul->data->Bgd->rank == node) ||
+          (dir == UNPACK && sul->data->Bgd->rank == myrank && sul->data->Bgs->rank == node))
+      {
+         varls = VarLists;
+         varld = VarListd;
+         while (varls && varld)
+         {
+            if (data)
+            {
+               if (dir == PACK)
+               {
+                  //         f_global_interp(sul->data->Bgs->shape,sul->data->Bgs->X[0],sul->data->Bgs->X[1],sul->data->Bgs->X[2],
+                  //			 sul->data->Bgs->fgfs[varls->data->sgfn],data[size_out],
+                  //			 sul->data->pox[0],sul->data->pox[1],sul->data->pox[2],ordn,varls->data->SoA,Symmetry);
+                  if (sul->data->coef == 0)
+                  {
+                     sul->data->coef = new double[ordn * dim];
+                     for (int i = 0; i < dim; i++)
+                     {
+                        double dd = sul->data->Bgs->getdX(i);
+                        sul->data->sind[i] = int((sul->data->pox[i] - sul->data->Bgs->X[i][0]) / dd) - ordn / 2 + 1;
+                        double h1, h2;
+                        for (int j = 0; j < ordn; j++)
+                        {
+                           h1 = sul->data->Bgs->X[i][0] + (sul->data->sind[i] + j) * dd;
+                           sul->data->coef[i * ordn + j] = 1;
+                           for (int k = 0; k < j; k++)
+                           {
+                              h2 = sul->data->Bgs->X[i][0] + (sul->data->sind[i] + k) * dd;
+                              sul->data->coef[i * ordn + j] *= (sul->data->pox[i] - h2) / (h1 - h2);
+                           }
+                           for (int k = j + 1; k < ordn; k++)
+                           {
+                              h2 = sul->data->Bgs->X[i][0] + (sul->data->sind[i] + k) * dd;
+                              sul->data->coef[i * ordn + j] *= (sul->data->pox[i] - h2) / (h1 - h2);
+                           }
+                        }
+                     }
+                  }
+                  int sst = -1;
+                  f_global_interpind(sul->data->Bgs->shape, sul->data->Bgs->X[0], sul->data->Bgs->X[1], sul->data->Bgs->X[2],
+                                     sul->data->Bgs->fgfs[varls->data->sgfn], data[size_out],
+                                     sul->data->pox[0], sul->data->pox[1], sul->data->pox[2], ordn, varls->data->SoA, Symmetry,
+                                     sul->data->sind, sul->data->coef, sst);
+               }
+               if (dir == UNPACK) // from target data to corresponding grid
+                  f_pointcopy(DIM, sul->data->Bgd->bbox, sul->data->Bgd->bbox + dim, sul->data->Bgd->shape, sul->data->Bgd->fgfs[varld->data->sgfn],
+                              sul->data->pox[0], sul->data->pox[1], sul->data->pox[2], data[size_out]);
+            }
+            size_out += 1;
+            varls = varls->next;
+            varld = varld->next;
+         }
+      }
+      sul = sul->next;
+   }
+
+   return size_out;
+}
+void Parallel::destroypsuList_bam(MyList<pointstru_bam> *ct)
+{
+   MyList<pointstru_bam> *n;
+   while (ct)
+   {
+      n = ct->next;
+      if (ct->data->coef)
+         delete[] ct->data->coef;
+      delete ct->data;
+      delete ct;
+      ct = n;
+   }
+}
--- a/AMSS_NCKU_source/Parallel/Parallel_bam.h
+++ b/AMSS_NCKU_source/Parallel/Parallel_bam.h
@@ -0,0 +1,53 @@
+
+#ifndef PARALLEL_BAM_H
+#define PARALLEL_BAM_H
+
+#include <iostream>
+#include <iomanip>
+#include <fstream>
+#include <cstdlib>
+#include <cstdio>
+#include <string>
+#include <cmath>
+#include <new>
+using namespace std;
+
+#include "var.h"
+#include "MPatch.h"
+#include "Block.h"
+#include "MyList.h"
+#include "macrodef.h"
+namespace Parallel
+{
+  struct pointstru_bam
+  {
+    double pox[dim]; // cordinate
+    Block *Bgs;      // interplate from
+    Block *Bgd;      // interplate for
+    double *coef;    // interpolation coefficients
+    int sind[dim];   // interpolation starting array index
+  };
+  void destroypsuList_bam(MyList<pointstru_bam> *ct);
+  void OutBdLow2Hi_bam(MyList<Patch> *PLc, MyList<Patch> *PLf,
+                       MyList<var> *VarList1 /* source */, MyList<var> *VarList2 /* target */,
+                       int Symmetry);
+  void OutBdLow2Hi_bam(MyList<Patch> *PLc, MyList<Patch> *PLf,
+                       MyList<var> *VarList1 /* source */, MyList<var> *VarList2 /* target */,
+                       MyList<Parallel::pointstru_bam> *bdsul, int Symmetry);
+  void Constr_pointstr_OutBdLow2Hi(MyList<Patch> *PLf, MyList<Patch> *PLc,
+                                   MyList<Parallel::pointstru_bam> *&bdsul);
+  void Restrict_bam(MyList<Patch> *PLc, MyList<Patch> *PLf,
+                    MyList<var> *VarList1 /* source */, MyList<var> *VarList2 /* target */,
+                    int Symmetry);
+  void Restrict_bam(MyList<Patch> *PLc, MyList<Patch> *PLf,
+                    MyList<var> *VarList1 /* source */, MyList<var> *VarList2 /* target */,
+                    MyList<Parallel::pointstru_bam> *rsul, int Symmetry);
+  void Constr_pointstr_Restrict(MyList<Patch> *PLf, MyList<Patch> *PLc,
+                                MyList<Parallel::pointstru_bam> *&rsul);
+  void intertransfer(MyList<Parallel::pointstru_bam> *&sul,
+                     MyList<var> *VarList1 /* source */, MyList<var> *VarList2 /*target */,
+                     int Symmetry);
+  int interdata_packer(double *data, MyList<Parallel::pointstru_bam> *sul, int myrank, int node, int dir,
+                       MyList<var> *VarLists /* source */, MyList<var> *VarListd /* target */, int Symmetry);
+}
+#endif /*PARALLEL_BAM_H */