64-BitBrainstorm_2026/AMSS-NCKU
8
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[TEST]UPSTREAM: Pick some source changes from 48080d0a97

Browse Source 
* Sync new folder structure
This commit is contained in:
ianchb
2026-04-23 20:55:40 +08:00
parent c185f99ee3
commit 17109fde9b
211 changed files with 189504 additions and 189280 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2908
AMSS_NCKU_source/BSSN_GPU/bssn_gpu.cu Normal file
View File

File diff suppressed because it is too large Load Diff

73
AMSS_NCKU_source/BSSN_GPU/bssn_gpu.h Normal file
View File

@@ -0,0 +1,73 @@
#ifndef BSSN_GPU_H_
#define BSSN_GPU_H_
#include "bssn_macro.h"
#include "macrodef.fh"
#define DEVICE_ID 0
// #define DEVICE_ID_BY_MPI_RANK
#define GRID_DIM 256
#define BLOCK_DIM 128
#define _FH2_(i, j, k) fh[(i) + (j) * _1D_SIZE[2] + (k) * _2D_SIZE[2]]
#define _FH3_(i, j, k) fh[(i) + (j) * _1D_SIZE[3] + (k) * _2D_SIZE[3]]
#define pow2(x) ((x) * (x))
#define TimeBetween(a, b) ((b.tv_sec - a.tv_sec) + (b.tv_usec - a.tv_usec) / 1000000.0f)
#define M_ metac.
#define Mh_ meta->
#define Ms_ metassc.
#define Msh_ metass->
// #define TIMING
#define RHS_SS_PARA int calledby, int mpi_rank, int *ex, double &T, double *crho, double *sigma, double *R, double *X, double *Y, double *Z, double *drhodx, double *drhody, double *drhodz, double *dsigmadx, double *dsigmady, double *dsigmadz, double *dRdx, double *dRdy, double *dRdz, double *drhodxx, double *drhodxy, double *drhodxz, double *drhodyy, double *drhodyz, double *drhodzz, double *dsigmadxx, double *dsigmadxy, double *dsigmadxz, double *dsigmadyy, double *dsigmadyz, double *dsigmadzz, double *dRdxx, double *dRdxy, double *dRdxz, double *dRdyy, double *dRdyz, double *dRdzz, double *chi, double *trK, double *dxx, double *gxy, double *gxz, double *dyy, double *gyz, double *dzz, double *Axx, double *Axy, double *Axz, double *Ayy, double *Ayz, double *Azz, double *Gamx, double *Gamy, double *Gamz, double *Lap, double *betax, double *betay, double *betaz, double *dtSfx, double *dtSfy, double *dtSfz, double *chi_rhs, double *trK_rhs, double *gxx_rhs, double *gxy_rhs, double *gxz_rhs, double *gyy_rhs, double *gyz_rhs, double *gzz_rhs, double *Axx_rhs, double *Axy_rhs, double *Axz_rhs, double *Ayy_rhs, double *Ayz_rhs, double *Azz_rhs, double *Gamx_rhs, double *Gamy_rhs, double *Gamz_rhs, double *Lap_rhs, double *betax_rhs, double *betay_rhs, double *betaz_rhs, double *dtSfx_rhs, double *dtSfy_rhs, double *dtSfz_rhs, double *rho, double *Sx, double *Sy, double *Sz, double *Sxx, double *Sxy, double *Sxz, double *Syy, double *Syz, double *Szz, double *Gamxxx, double *Gamxxy, double *Gamxxz, double *Gamxyy, double *Gamxyz, double *Gamxzz, double *Gamyxx, double *Gamyxy, double *Gamyxz, double *Gamyyy, double *Gamyyz, double *Gamyzz, double *Gamzxx, double *Gamzxy, double *Gamzxz, double *Gamzyy, double *Gamzyz, double *Gamzzz, double *Rxx, double *Rxy, double *Rxz, double *Ryy, double *Ryz, double *Rzz, double *ham_Res, double *movx_Res, double *movy_Res, double *movz_Res, double *Gmx_Res, double *Gmy_Res, double *Gmz_Res, int &Symmetry, int &Lev, double &eps, int &sst, int &co
/** main function */
int gpu_rhs(int calledby, int mpi_rank, int *ex, double &T,
double *X, double *Y, double *Z,
double *chi, double *trK,
double *dxx, double *gxy, double *gxz, double *dyy, double *gyz, double *dzz,
double *Axx, double *Axy, double *Axz, double *Ayy, double *Ayz, double *Azz,
double *Gamx, double *Gamy, double *Gamz,
double *Lap, double *betax, double *betay, double *betaz,
double *dtSfx, double *dtSfy, double *dtSfz,
double *chi_rhs, double *trK_rhs,
double *gxx_rhs, double *gxy_rhs, double *gxz_rhs, double *gyy_rhs, double *gyz_rhs, double *gzz_rhs,
double *Axx_rhs, double *Axy_rhs, double *Axz_rhs, double *Ayy_rhs, double *Ayz_rhs, double *Azz_rhs,
double *Gamx_rhs, double *Gamy_rhs, double *Gamz_rhs,
double *Lap_rhs, double *betax_rhs, double *betay_rhs, double *betaz_rhs,
double *dtSfx_rhs, double *dtSfy_rhs, double *dtSfz_rhs,
double *rho, double *Sx, double *Sy, double *Sz, double *Sxx,
double *Sxy, double *Sxz, double *Syy, double *Syz, double *Szz,
double *Gamxxx, double *Gamxxy, double *Gamxxz, double *Gamxyy, double *Gamxyz, double *Gamxzz,
double *Gamyxx, double *Gamyxy, double *Gamyxz, double *Gamyyy, double *Gamyyz, double *Gamyzz,
double *Gamzxx, double *Gamzxy, double *Gamzxz, double *Gamzyy, double *Gamzyz, double *Gamzzz,
double *Rxx, double *Rxy, double *Rxz, double *Ryy, double *Ryz, double *Rzz,
double *ham_Res, double *movx_Res, double *movy_Res, double *movz_Res,
double *Gmx_Res, double *Gmy_Res, double *Gmz_Res,
int &Symmetry, int &Lev, double &eps, int &co);
int gpu_rhs_ss(RHS_SS_PARA);
/** Init GPU side data in GPUMeta. */
// void init_fluid_meta_gpu(GPUMeta *gpu_meta);
#endif

7790
AMSS_NCKU_source/BSSN_GPU/bssn_gpu_class.C Normal file
View File

File diff suppressed because it is too large Load Diff

210
AMSS_NCKU_source/BSSN_GPU/bssn_gpu_class.h Normal file
View File

@@ -0,0 +1,210 @@
#ifndef BSSN_GPU_CLASS_H
#define BSSN_GPU_CLASS_H
#ifdef newc
#include <iostream>
#include <iomanip>
#include <fstream>
#include <cstdlib>
#include <string>
#include <cmath>
using namespace std;
#else
#include <iostream.h>
#include <iomanip.h>
#include <fstream.h>
#include <stdlib.h>
#include <string.h>
#include <math.h>
#endif
#include <mpi.h>
#include "macrodef.h"
#include "cgh.h"
#include "ShellPatch.h"
#include "misc.h"
#include "var.h"
#include "MyList.h"
#include "monitor.h"
#include "surface_integral.h"
#include "checkpoint.h"
// added by yangquan
#include "bssn_macro.h"
extern void setpbh(int iBHN, double **iPBH, double *iMass, int rBHN);
class bssn_class
{
public:
// added by yangquan
//----------------------
int gpu_num_mynode;
int cpu_core_num_mynode;
int mpi_process_num_mynode;
int my_sequence_mynode;
int mynode_id;
int use_gpu;
virtual void Step_GPU(int lev, int YN);
virtual void Get_runtime_envirment();
// virtual void Step_OPENMP(int lev,int YN);
//----------------------
int ngfs;
int nprocs, myrank;
cgh *GH;
ShellPatch *SH;
double PhysTime;
int checkrun;
char checkfilename[50];
int Steps;
double StartTime, TotalTime;
double AnasTime, DumpTime, d2DumpTime, CheckTime;
double LastAnas, LastConsOut;
double Courant;
double numepss, numepsb, numepsh;
int Symmetry;
int maxl, decn;
double maxrex, drex;
int trfls, a_lev;
double dT;
double chitiny;
double **Porg0, **Porgbr, **Porg, **Porg1, **Porg_rhs;
int BH_num, BH_num_input;
double *Mass, *Pmom, *Spin;
double ADMMass;
var *phio, *trKo;
var *gxxo, *gxyo, *gxzo, *gyyo, *gyzo, *gzzo;
var *Axxo, *Axyo, *Axzo, *Ayyo, *Ayzo, *Azzo;
var *Gmxo, *Gmyo, *Gmzo;
var *Lapo, *Sfxo, *Sfyo, *Sfzo;
var *dtSfxo, *dtSfyo, *dtSfzo;
var *phi0, *trK0;
var *gxx0, *gxy0, *gxz0, *gyy0, *gyz0, *gzz0;
var *Axx0, *Axy0, *Axz0, *Ayy0, *Ayz0, *Azz0;
var *Gmx0, *Gmy0, *Gmz0;
var *Lap0, *Sfx0, *Sfy0, *Sfz0;
var *dtSfx0, *dtSfy0, *dtSfz0;
var *phi, *trK;
var *gxx, *gxy, *gxz, *gyy, *gyz, *gzz;
var *Axx, *Axy, *Axz, *Ayy, *Ayz, *Azz;
var *Gmx, *Gmy, *Gmz;
var *Lap, *Sfx, *Sfy, *Sfz;
var *dtSfx, *dtSfy, *dtSfz;
var *phi1, *trK1;
var *gxx1, *gxy1, *gxz1, *gyy1, *gyz1, *gzz1;
var *Axx1, *Axy1, *Axz1, *Ayy1, *Ayz1, *Azz1;
var *Gmx1, *Gmy1, *Gmz1;
var *Lap1, *Sfx1, *Sfy1, *Sfz1;
var *dtSfx1, *dtSfy1, *dtSfz1;
var *phi_rhs, *trK_rhs;
var *gxx_rhs, *gxy_rhs, *gxz_rhs, *gyy_rhs, *gyz_rhs, *gzz_rhs;
var *Axx_rhs, *Axy_rhs, *Axz_rhs, *Ayy_rhs, *Ayz_rhs, *Azz_rhs;
var *Gmx_rhs, *Gmy_rhs, *Gmz_rhs;
var *Lap_rhs, *Sfx_rhs, *Sfy_rhs, *Sfz_rhs;
var *dtSfx_rhs, *dtSfy_rhs, *dtSfz_rhs;
var *rho, *Sx, *Sy, *Sz, *Sxx, *Sxy, *Sxz, *Syy, *Syz, *Szz;
var *Gamxxx, *Gamxxy, *Gamxxz, *Gamxyy, *Gamxyz, *Gamxzz;
var *Gamyxx, *Gamyxy, *Gamyxz, *Gamyyy, *Gamyyz, *Gamyzz;
var *Gamzxx, *Gamzxy, *Gamzxz, *Gamzyy, *Gamzyz, *Gamzzz;
var *Rxx, *Rxy, *Rxz, *Ryy, *Ryz, *Rzz;
var *Rpsi4, *Ipsi4;
var *t1Rpsi4, *t1Ipsi4, *t2Rpsi4, *t2Ipsi4;
var *Cons_Ham, *Cons_Px, *Cons_Py, *Cons_Pz, *Cons_Gx, *Cons_Gy, *Cons_Gz;
#ifdef Point_Psi4
var *phix, *phiy, *phiz;
var *trKx, *trKy, *trKz;
var *Axxx, *Axxy, *Axxz;
var *Axyx, *Axyy, *Axyz;
var *Axzx, *Axzy, *Axzz;
var *Ayyx, *Ayyy, *Ayyz;
var *Ayzx, *Ayzy, *Ayzz;
var *Azzx, *Azzy, *Azzz;
#endif
// FIXME: uc = StateList, up = OldStateList, upp = SynchList_cor; so never touch these three data
MyList<var> *StateList, *SynchList_pre, *SynchList_cor, *RHSList;
MyList<var> *OldStateList, *DumpList;
MyList<var> *ConstraintList;
monitor *ErrorMonitor, *Psi4Monitor, *BHMonitor, *MAPMonitor;
monitor *ConVMonitor;
surface_integral *Waveshell;
checkpoint *CheckPoint;
public:
bssn_class(double Couranti, double StartTimei, double TotalTimei, double DumpTimei, double d2DumpTimei, double CheckTimei, double AnasTimei,
int Symmetryi, int checkruni, char *checkfilenamei, double numepssi, double numepsbi, double numepshi,
int a_levi, int maxli, int decni, double maxrexi, double drexi);
~bssn_class();
void Evolve(int Steps);
void RecursiveStep(int lev);
#if (PSTR == 1)
void ParallelStep();
void SHStep();
#endif
void RestrictProlong(int lev, int YN, bool BB, MyList<var> *SL, MyList<var> *OL, MyList<var> *corL);
void RestrictProlong_aux(int lev, int YN, bool BB, MyList<var> *SL, MyList<var> *OL, MyList<var> *corL);
void RestrictProlong(int lev, int YN, bool BB);
void ProlongRestrict(int lev, int YN, bool BB);
void Setup_Black_Hole_position();
void compute_Porg_rhs(double **BH_PS, double **BH_RHS, var *forx, var *fory, var *forz, int lev);
bool read_Pablo_file(int *ext, double *datain, char *filename);
void write_Pablo_file(int *ext, double xmin, double xmax, double ymin, double ymax, double zmin, double zmax,
char *filename);
void AnalysisStuff(int lev, double dT_lev);
void Setup_KerrSchild();
void Enforce_algcon(int lev, int fg);
void testRestrict();
void testOutBd();
virtual void Setup_Initial_Data_Lousto();
virtual void Setup_Initial_Data_Cao();
virtual void Initialize();
virtual void Read_Ansorg();
virtual void Read_Pablo() {};
virtual void Compute_Psi4(int lev);
virtual void Step(int lev, int YN);
virtual void Interp_Constraint(bool infg);
virtual void Constraint_Out();
virtual void Compute_Constraint();
#ifdef With_AHF
protected:
MyList<var> *AHList, *AHDList, *GaugeList;
int AHfindevery;
double AHdumptime;
int *lastahdumpid, HN_num; // number of possible horizons
int *findeveryl;
double *xc, *yc, *zc, *xr, *yr, *zr;
bool *trigger;
double *dTT;
int *dumpid;
public:
void AH_Prepare_derivatives();
bool AH_Interp_Points(MyList<var> *VarList,
int NN, double **XX,
double *Shellf, int Symmetryi);
void AH_Step_Find(int lev, double dT_lev);
#endif
};
#endif /* BSSN_GPU_CLASS_H */

2525
AMSS_NCKU_source/BSSN_GPU/bssn_gpu_rhs_ss.cu Normal file
View File

File diff suppressed because it is too large Load Diff

124
AMSS_NCKU_source/BSSN_GPU/bssn_macro.C Normal file
View File

@@ -0,0 +1,124 @@
#include "bssn_macro.h"
#include <iostream>
#include <fstream>
#include <cstring>
using namespace std;
int compare_two_file(char *fname1, char *fname2, int data_num)
{
// read file
fstream file1(fname1, ios_base::in);
fstream file2(fname2, ios_base::in);
double *d1, *d2;
d1 = (double *)malloc(sizeof(double) * data_num);
d2 = (double *)malloc(sizeof(double) * data_num);
for (int i = 0; i < data_num; ++i)
{
file1.read((char *)(d1 + i), sizeof(double));
file2.read((char *)(d2 + i), sizeof(double));
}
// compare data
bool is_match = true;
for (int i = 0; i < data_num; ++i)
{
if (d1[i] != d2[i])
{
is_match = false;
cout << "miss match at position " << i << endl;
break;
}
}
if (is_match)
cout << "Result is right." << endl;
free(d1);
free(d2);
file1.close();
file2.close();
return 0;
}
void printMatrix(int ftag1, int ftag2, double *d1, double *d2, int ord)
{
char fname1[32];
char fname2[32];
// char ftag1[32]; char ftag2[32];
// sprintf(ftag1,"%d",ftag1);
strcpy(fname1, "matrix_f.show");
// strcat(fname1,ftag1);
// sprintf(ftag2,"%d",ftag2);
strcpy(fname2, "matrix_g.show");
// strcat(fname2,ftag2);
ofstream fout0, fout1, fout2;
fout1.open(fname1);
fout2.open(fname2);
for (int k = 0; k < 65; k++)
{
fout1 << "---------square " << k << " ----------" << endl;
fout2 << "---------square " << k << " ----------" << endl;
for (int j = 0; j < 67 + ord * 2; j++)
{
for (int i = 0; i < 67 + ord * 2; i++)
{
fout1 << d1[i + j * (67 + ord * 2) + k * ((67 + ord * 2) * (67 + ord * 2))] << ' ';
fout2 << d2[i + j * (67 + ord * 2) + k * ((67 + ord * 2) * (67 + ord * 2))] << ' ';
// fout1<<test_output_g[i+j*(cg->shape[0]) + k*(_2d_size)] <<' ';
// fout2<<test_fh_f [i+j*(cg->shape[0]) + k*(_2d_size)] <<' ';
}
fout1 << endl;
fout2 << endl;
}
}
}
int compare_result(int ftag1, double *d2, int data_num)
{
// read file
char fname1[32];
char ftag[32];
// itoa(filetag,ftag,10);
sprintf(ftag, "%d", ftag1);
strcpy(fname1, "matrix_f.out");
strcat(fname1, ftag);
fstream file1(fname1, ios_base::in);
double *d1;
d1 = (double *)malloc(sizeof(double) * data_num);
for (int i = 0; i < data_num; ++i)
{
file1.read((char *)(d1 + i), sizeof(double));
}
// compare data
bool is_match = true;
double delta;
for (int i = 0; i < data_num; ++i)
{
delta = d1[i] - d2[i];
if (delta < 0)
delta = -delta;
if (delta > 1e-14)
{
is_match = false;
cout << fname1 << "::miss match at position " << i << endl;
break;
}
// if(i<100 && i>50)
// cout<<d1[i]<<" "<<d2[i]<<endl;
}
if (is_match)
cout << ftag1 << "::matched." << endl;
if (ftag1 == 0)
{
printMatrix(1, 2, d1, d2, 3);
}
free(d1);
file1.close();
return 0;
}

94
AMSS_NCKU_source/BSSN_GPU/bssn_macro.h Normal file
View File

@@ -0,0 +1,94 @@
#ifndef BSSN_STEP_H
#define BSSN_STEP_H
//1---------------------FLAGS---------------------
#define USE_GPU
#define MAX_GPU_PROCESS_NUM 1
#define COUNT_CPU_RHS_TIME
//2---------------------TIMER---------------------
//2.1 TIMER_INIT
//2.2 TIMER_TIC_WITHOUT_OUTPUT
//2.3 TIMER_TIC(tag,order,label)
//2.4 TIMER_TIC_TAIL_OF_FUNC(tag,label)
#define TIME_COUNT_EACH_RANK 0
#define TIMER_INIT \
double clock_prev,clock_curr,step_begin_clock;\
if(1 == 1){\
clock_curr =MPI_Wtime();\
step_begin_clock = MPI_Wtime();\
}else{\
if(myrank == 0){\
clock_curr= MPI_Wtime();\
step_begin_clock = MPI_Wtime();\
}\
}
#define TIMER_TIC(tag,order,label) \
if(TIME_COUNT_EACH_RANK == 1){\
clock_prev= clock_curr;\
clock_curr = MPI_Wtime();\
cout<<#tag <<order <<":MPI Rank: "<<myrank<<" "<<#label <<" "<<(clock_curr-clock_prev)<<endl;\
}else{\
if(myrank==0){\
clock_prev= clock_curr;\
clock_curr = MPI_Wtime();\
cout<<#tag <<order <<" "<<#label " "<<(clock_curr-clock_prev)<<endl;\
}\
}
#define TIMER_TIC_EACH_PROC(tag,order,label) \
clock_prev= clock_curr;\
clock_curr = MPI_Wtime();\
cout<<#tag <<order <<":MPI Rank: "<<myrank<<" "<<#label <<" "<<(clock_curr-clock_prev)<<endl;\
}
#define TIMER_TIC_WITHOUT_OUTPUT \
if(TIME_COUNT_EACH_RANK == 1){\
clock_curr = MPI_Wtime();\
}else{\
if(myrank==0){\
clock_curr = MPI_Wtime();\
}\
}
#define TIMER_TIC_TAIL_OF_FUNC(tag,label) \
if(TIME_COUNT_EACH_RANK == 1){\
cout<<#tag <<"MPI Rank: "<<myrank<<" "<<#label <<" "<<(MPI_Wtime()-step_begin_clock)<<" seconds!"<<endl;\
}else{\
if(myrank==0)\
{\
cout<<#tag <<#label <<" "<<(MPI_Wtime()-step_begin_clock)<<" seconds!"<<endl;\
}\
}
//3---------------------GPU---------------------
#define CALLED_BY_STEP 0
#define CALLED_BY_CONSTRAINT 1
#define RHS_PARA_CALLED_FIRST_TIME cg->shape,TRK4,cg->X[0],cg->X[1],cg->X[2],cg->fgfs[phi0->sgfn],cg->fgfs[trK0->sgfn],cg->fgfs[gxx0->sgfn],cg->fgfs[gxy0->sgfn],cg->fgfs[gxz0->sgfn],cg->fgfs[gyy0->sgfn],cg->fgfs[gyz0->sgfn],cg->fgfs[gzz0->sgfn],cg->fgfs[Axx0->sgfn],cg->fgfs[Axy0->sgfn],cg->fgfs[Axz0->sgfn],cg->fgfs[Ayy0->sgfn],cg->fgfs[Ayz0->sgfn],cg->fgfs[Azz0->sgfn],cg->fgfs[Gmx0->sgfn],cg->fgfs[Gmy0->sgfn],cg->fgfs[Gmz0->sgfn],cg->fgfs[Lap0->sgfn],cg->fgfs[Sfx0->sgfn],cg->fgfs[Sfy0->sgfn],cg->fgfs[Sfz0->sgfn],cg->fgfs[dtSfx0->sgfn],cg->fgfs[dtSfy0->sgfn],cg->fgfs[dtSfz0->sgfn],cg->fgfs[phi_rhs->sgfn],cg->fgfs[trK_rhs->sgfn],cg->fgfs[gxx_rhs->sgfn],cg->fgfs[gxy_rhs->sgfn],cg->fgfs[gxz_rhs->sgfn],cg->fgfs[gyy_rhs->sgfn],cg->fgfs[gyz_rhs->sgfn],cg->fgfs[gzz_rhs->sgfn],cg->fgfs[Axx_rhs->sgfn],cg->fgfs[Axy_rhs->sgfn],cg->fgfs[Axz_rhs->sgfn],cg->fgfs[Ayy_rhs->sgfn],cg->fgfs[Ayz_rhs->sgfn],cg->fgfs[Azz_rhs->sgfn],cg->fgfs[Gmx_rhs->sgfn],cg->fgfs[Gmy_rhs->sgfn],cg->fgfs[Gmz_rhs->sgfn],cg->fgfs[Lap_rhs->sgfn],cg->fgfs[Sfx_rhs->sgfn],cg->fgfs[Sfy_rhs->sgfn],cg->fgfs[Sfz_rhs->sgfn],cg->fgfs[dtSfx_rhs->sgfn],cg->fgfs[dtSfy_rhs->sgfn],cg->fgfs[dtSfz_rhs->sgfn],cg->fgfs[rho->sgfn],cg->fgfs[Sx->sgfn],cg->fgfs[Sy->sgfn],cg->fgfs[Sz->sgfn],cg->fgfs[Sxx->sgfn],cg->fgfs[Sxy->sgfn],cg->fgfs[Sxz->sgfn],cg->fgfs[Syy->sgfn],cg->fgfs[Syz->sgfn],cg->fgfs[Szz->sgfn],cg->fgfs[Gamxxx->sgfn],cg->fgfs[Gamxxy->sgfn],cg->fgfs[Gamxxz->sgfn],cg->fgfs[Gamxyy->sgfn],cg->fgfs[Gamxyz->sgfn],cg->fgfs[Gamxzz->sgfn],cg->fgfs[Gamyxx->sgfn],cg->fgfs[Gamyxy->sgfn],cg->fgfs[Gamyxz->sgfn],cg->fgfs[Gamyyy->sgfn],cg->fgfs[Gamyyz->sgfn],cg->fgfs[Gamyzz->sgfn],cg->fgfs[Gamzxx->sgfn],cg->fgfs[Gamzxy->sgfn],cg->fgfs[Gamzxz->sgfn],cg->fgfs[Gamzyy->sgfn],cg->fgfs[Gamzyz->sgfn],cg->fgfs[Gamzzz->sgfn],cg->fgfs[Rxx->sgfn],cg->fgfs[Rxy->sgfn],cg->fgfs[Rxz->sgfn],cg->fgfs[Ryy->sgfn],cg->fgfs[Ryz->sgfn],cg->fgfs[Rzz->sgfn],cg->fgfs[Cons_Ham->sgfn],cg->fgfs[Cons_Px->sgfn],cg->fgfs[Cons_Py->sgfn],cg->fgfs[Cons_Pz->sgfn],cg->fgfs[Cons_Gx->sgfn],cg->fgfs[Cons_Gy->sgfn],cg->fgfs[Cons_Gz->sgfn],Symmetry,lev,ndeps,pre
#define RHS_PARA_CALLED_THEN cg->shape,TRK4,cg->X[0],cg->X[1],cg->X[2],cg->fgfs[phi->sgfn],cg->fgfs[trK->sgfn],cg->fgfs[gxx->sgfn],cg->fgfs[gxy->sgfn],cg->fgfs[gxz->sgfn],cg->fgfs[gyy->sgfn],cg->fgfs[gyz->sgfn],cg->fgfs[gzz->sgfn],cg->fgfs[Axx->sgfn],cg->fgfs[Axy->sgfn],cg->fgfs[Axz->sgfn],cg->fgfs[Ayy->sgfn],cg->fgfs[Ayz->sgfn],cg->fgfs[Azz->sgfn],cg->fgfs[Gmx->sgfn],cg->fgfs[Gmy->sgfn],cg->fgfs[Gmz->sgfn],cg->fgfs[Lap->sgfn],cg->fgfs[Sfx->sgfn],cg->fgfs[Sfy->sgfn],cg->fgfs[Sfz->sgfn],cg->fgfs[dtSfx->sgfn],cg->fgfs[dtSfy->sgfn],cg->fgfs[dtSfz->sgfn],cg->fgfs[phi1->sgfn],cg->fgfs[trK1->sgfn],cg->fgfs[gxx1->sgfn],cg->fgfs[gxy1->sgfn],cg->fgfs[gxz1->sgfn],cg->fgfs[gyy1->sgfn],cg->fgfs[gyz1->sgfn],cg->fgfs[gzz1->sgfn],cg->fgfs[Axx1->sgfn],cg->fgfs[Axy1->sgfn],cg->fgfs[Axz1->sgfn],cg->fgfs[Ayy1->sgfn],cg->fgfs[Ayz1->sgfn],cg->fgfs[Azz1->sgfn],cg->fgfs[Gmx1->sgfn],cg->fgfs[Gmy1->sgfn],cg->fgfs[Gmz1->sgfn],cg->fgfs[Lap1->sgfn],cg->fgfs[Sfx1->sgfn],cg->fgfs[Sfy1->sgfn],cg->fgfs[Sfz1->sgfn],cg->fgfs[dtSfx1->sgfn],cg->fgfs[dtSfy1->sgfn],cg->fgfs[dtSfz1->sgfn],cg->fgfs[rho->sgfn],cg->fgfs[Sx->sgfn],cg->fgfs[Sy->sgfn],cg->fgfs[Sz->sgfn],cg->fgfs[Sxx->sgfn],cg->fgfs[Sxy->sgfn],cg->fgfs[Sxz->sgfn],cg->fgfs[Syy->sgfn],cg->fgfs[Syz->sgfn],cg->fgfs[Szz->sgfn],cg->fgfs[Gamxxx->sgfn],cg->fgfs[Gamxxy->sgfn],cg->fgfs[Gamxxz->sgfn],cg->fgfs[Gamxyy->sgfn],cg->fgfs[Gamxyz->sgfn],cg->fgfs[Gamxzz->sgfn],cg->fgfs[Gamyxx->sgfn],cg->fgfs[Gamyxy->sgfn],cg->fgfs[Gamyxz->sgfn],cg->fgfs[Gamyyy->sgfn],cg->fgfs[Gamyyz->sgfn],cg->fgfs[Gamyzz->sgfn],cg->fgfs[Gamzxx->sgfn],cg->fgfs[Gamzxy->sgfn],cg->fgfs[Gamzxz->sgfn],cg->fgfs[Gamzyy->sgfn],cg->fgfs[Gamzyz->sgfn],cg->fgfs[Gamzzz->sgfn],cg->fgfs[Rxx->sgfn],cg->fgfs[Rxy->sgfn],cg->fgfs[Rxz->sgfn],cg->fgfs[Ryy->sgfn],cg->fgfs[Ryz->sgfn],cg->fgfs[Rzz->sgfn],cg->fgfs[Cons_Ham->sgfn],cg->fgfs[Cons_Px->sgfn],cg->fgfs[Cons_Py->sgfn],cg->fgfs[Cons_Pz->sgfn],cg->fgfs[Cons_Gx->sgfn],cg->fgfs[Cons_Gy->sgfn],cg->fgfs[Cons_Gz->sgfn],Symmetry,lev,ndeps,cor
#define RHS_PARA_CALLED_Constraint_Out cg->shape,TRK4,cg->X[0],cg->X[1],cg->X[2],cg->fgfs[phi0->sgfn],cg->fgfs[trK0->sgfn],cg->fgfs[gxx0->sgfn],cg->fgfs[gxy0->sgfn],cg->fgfs[gxz0->sgfn],cg->fgfs[gyy0->sgfn],cg->fgfs[gyz0->sgfn],cg->fgfs[gzz0->sgfn],cg->fgfs[Axx0->sgfn],cg->fgfs[Axy0->sgfn],cg->fgfs[Axz0->sgfn],cg->fgfs[Ayy0->sgfn],cg->fgfs[Ayz0->sgfn],cg->fgfs[Azz0->sgfn],cg->fgfs[Gmx0->sgfn],cg->fgfs[Gmy0->sgfn],cg->fgfs[Gmz0->sgfn],cg->fgfs[Lap0->sgfn],cg->fgfs[Sfx0->sgfn],cg->fgfs[Sfy0->sgfn],cg->fgfs[Sfz0->sgfn],cg->fgfs[dtSfx0->sgfn],cg->fgfs[dtSfy0->sgfn],cg->fgfs[dtSfz0->sgfn],cg->fgfs[phi_rhs->sgfn],cg->fgfs[trK_rhs->sgfn],cg->fgfs[gxx_rhs->sgfn],cg->fgfs[gxy_rhs->sgfn],cg->fgfs[gxz_rhs->sgfn],cg->fgfs[gyy_rhs->sgfn],cg->fgfs[gyz_rhs->sgfn],cg->fgfs[gzz_rhs->sgfn],cg->fgfs[Axx_rhs->sgfn],cg->fgfs[Axy_rhs->sgfn],cg->fgfs[Axz_rhs->sgfn],cg->fgfs[Ayy_rhs->sgfn],cg->fgfs[Ayz_rhs->sgfn],cg->fgfs[Azz_rhs->sgfn],cg->fgfs[Gmx_rhs->sgfn],cg->fgfs[Gmy_rhs->sgfn],cg->fgfs[Gmz_rhs->sgfn],cg->fgfs[Lap_rhs->sgfn],cg->fgfs[Sfx_rhs->sgfn],cg->fgfs[Sfy_rhs->sgfn],cg->fgfs[Sfz_rhs->sgfn],cg->fgfs[dtSfx_rhs->sgfn],cg->fgfs[dtSfy_rhs->sgfn],cg->fgfs[dtSfz_rhs->sgfn],cg->fgfs[rho->sgfn],cg->fgfs[Sx->sgfn],cg->fgfs[Sy->sgfn],cg->fgfs[Sz->sgfn],cg->fgfs[Sxx->sgfn],cg->fgfs[Sxy->sgfn],cg->fgfs[Sxz->sgfn],cg->fgfs[Syy->sgfn],cg->fgfs[Syz->sgfn],cg->fgfs[Szz->sgfn],cg->fgfs[Gamxxx->sgfn],cg->fgfs[Gamxxy->sgfn],cg->fgfs[Gamxxz->sgfn],cg->fgfs[Gamxyy->sgfn],cg->fgfs[Gamxyz->sgfn],cg->fgfs[Gamxzz->sgfn],cg->fgfs[Gamyxx->sgfn],cg->fgfs[Gamyxy->sgfn],cg->fgfs[Gamyxz->sgfn],cg->fgfs[Gamyyy->sgfn],cg->fgfs[Gamyyz->sgfn],cg->fgfs[Gamyzz->sgfn],cg->fgfs[Gamzxx->sgfn],cg->fgfs[Gamzxy->sgfn],cg->fgfs[Gamzxz->sgfn],cg->fgfs[Gamzyy->sgfn],cg->fgfs[Gamzyz->sgfn],cg->fgfs[Gamzzz->sgfn],cg->fgfs[Rxx->sgfn],cg->fgfs[Rxy->sgfn],cg->fgfs[Rxz->sgfn],cg->fgfs[Ryy->sgfn],cg->fgfs[Ryz->sgfn],cg->fgfs[Rzz->sgfn],cg->fgfs[Cons_Ham->sgfn],cg->fgfs[Cons_Px->sgfn],cg->fgfs[Cons_Py->sgfn],cg->fgfs[Cons_Pz->sgfn],cg->fgfs[Cons_Gx->sgfn],cg->fgfs[Cons_Gy->sgfn],cg->fgfs[Cons_Gz->sgfn],Symmetry,lev,ndeps,pre
#define RHS_PARA_CALLED_Interp_Constraint cg->shape,TRK4,cg->X[0],cg->X[1],cg->X[2],cg->fgfs[phi0->sgfn],cg->fgfs[trK0->sgfn],cg->fgfs[gxx0->sgfn],cg->fgfs[gxy0->sgfn],cg->fgfs[gxz0->sgfn],cg->fgfs[gyy0->sgfn],cg->fgfs[gyz0->sgfn],cg->fgfs[gzz0->sgfn],cg->fgfs[Axx0->sgfn],cg->fgfs[Axy0->sgfn],cg->fgfs[Axz0->sgfn],cg->fgfs[Ayy0->sgfn],cg->fgfs[Ayz0->sgfn],cg->fgfs[Azz0->sgfn],cg->fgfs[Gmx0->sgfn],cg->fgfs[Gmy0->sgfn],cg->fgfs[Gmz0->sgfn],cg->fgfs[Lap0->sgfn],cg->fgfs[Sfx0->sgfn],cg->fgfs[Sfy0->sgfn],cg->fgfs[Sfz0->sgfn],cg->fgfs[dtSfx0->sgfn],cg->fgfs[dtSfy0->sgfn],cg->fgfs[dtSfz0->sgfn],cg->fgfs[phi_rhs->sgfn],cg->fgfs[trK_rhs->sgfn],cg->fgfs[gxx_rhs->sgfn],cg->fgfs[gxy_rhs->sgfn],cg->fgfs[gxz_rhs->sgfn],cg->fgfs[gyy_rhs->sgfn],cg->fgfs[gyz_rhs->sgfn],cg->fgfs[gzz_rhs->sgfn],cg->fgfs[Axx_rhs->sgfn],cg->fgfs[Axy_rhs->sgfn],cg->fgfs[Axz_rhs->sgfn],cg->fgfs[Ayy_rhs->sgfn],cg->fgfs[Ayz_rhs->sgfn],cg->fgfs[Azz_rhs->sgfn],cg->fgfs[Gmx_rhs->sgfn],cg->fgfs[Gmy_rhs->sgfn],cg->fgfs[Gmz_rhs->sgfn],cg->fgfs[Lap_rhs->sgfn],cg->fgfs[Sfx_rhs->sgfn],cg->fgfs[Sfy_rhs->sgfn],cg->fgfs[Sfz_rhs->sgfn],cg->fgfs[dtSfx_rhs->sgfn],cg->fgfs[dtSfy_rhs->sgfn],cg->fgfs[dtSfz_rhs->sgfn],cg->fgfs[rho->sgfn],cg->fgfs[Sx->sgfn],cg->fgfs[Sy->sgfn],cg->fgfs[Sz->sgfn],cg->fgfs[Sxx->sgfn],cg->fgfs[Sxy->sgfn],cg->fgfs[Sxz->sgfn],cg->fgfs[Syy->sgfn],cg->fgfs[Syz->sgfn],cg->fgfs[Szz->sgfn],cg->fgfs[Gamxxx->sgfn],cg->fgfs[Gamxxy->sgfn],cg->fgfs[Gamxxz->sgfn],cg->fgfs[Gamxyy->sgfn],cg->fgfs[Gamxyz->sgfn],cg->fgfs[Gamxzz->sgfn],cg->fgfs[Gamyxx->sgfn],cg->fgfs[Gamyxy->sgfn],cg->fgfs[Gamyxz->sgfn],cg->fgfs[Gamyyy->sgfn],cg->fgfs[Gamyyz->sgfn],cg->fgfs[Gamyzz->sgfn],cg->fgfs[Gamzxx->sgfn],cg->fgfs[Gamzxy->sgfn],cg->fgfs[Gamzxz->sgfn],cg->fgfs[Gamzyy->sgfn],cg->fgfs[Gamzyz->sgfn],cg->fgfs[Gamzzz->sgfn],cg->fgfs[Rxx->sgfn],cg->fgfs[Rxy->sgfn],cg->fgfs[Rxz->sgfn],cg->fgfs[Ryy->sgfn],cg->fgfs[Ryz->sgfn],cg->fgfs[Rzz->sgfn],cg->fgfs[Cons_Ham->sgfn],cg->fgfs[Cons_Px->sgfn],cg->fgfs[Cons_Py->sgfn],cg->fgfs[Cons_Pz->sgfn],cg->fgfs[Cons_Gx->sgfn],cg->fgfs[Cons_Gy->sgfn],cg->fgfs[Cons_Gz->sgfn],Symmetry,lev,ndeps,pre
#define RHS_SS_PARA_CALLED_FIRST_TIME cg->shape,TRK4,cg->X[0],cg->X[1],cg->X[2],cg->fgfs[fngfs+ShellPatch::gx],cg->fgfs[fngfs+ShellPatch::gy],cg->fgfs[fngfs+ShellPatch::gz],cg->fgfs[fngfs+ShellPatch::drhodx],cg->fgfs[fngfs+ShellPatch::drhody],cg->fgfs[fngfs+ShellPatch::drhodz],cg->fgfs[fngfs+ShellPatch::dsigmadx],cg->fgfs[fngfs+ShellPatch::dsigmady],cg->fgfs[fngfs+ShellPatch::dsigmadz],cg->fgfs[fngfs+ShellPatch::dRdx],cg->fgfs[fngfs+ShellPatch::dRdy],cg->fgfs[fngfs+ShellPatch::dRdz],cg->fgfs[fngfs+ShellPatch::drhodxx],cg->fgfs[fngfs+ShellPatch::drhodxy],cg->fgfs[fngfs+ShellPatch::drhodxz],cg->fgfs[fngfs+ShellPatch::drhodyy],cg->fgfs[fngfs+ShellPatch::drhodyz],cg->fgfs[fngfs+ShellPatch::drhodzz],cg->fgfs[fngfs+ShellPatch::dsigmadxx],cg->fgfs[fngfs+ShellPatch::dsigmadxy],cg->fgfs[fngfs+ShellPatch::dsigmadxz],cg->fgfs[fngfs+ShellPatch::dsigmadyy],cg->fgfs[fngfs+ShellPatch::dsigmadyz],cg->fgfs[fngfs+ShellPatch::dsigmadzz],cg->fgfs[fngfs+ShellPatch::dRdxx],cg->fgfs[fngfs+ShellPatch::dRdxy],cg->fgfs[fngfs+ShellPatch::dRdxz],cg->fgfs[fngfs+ShellPatch::dRdyy],cg->fgfs[fngfs+ShellPatch::dRdyz],cg->fgfs[fngfs+ShellPatch::dRdzz],cg->fgfs[phi0->sgfn],cg->fgfs[trK0->sgfn],cg->fgfs[gxx0->sgfn],cg->fgfs[gxy0->sgfn],cg->fgfs[gxz0->sgfn],cg->fgfs[gyy0->sgfn],cg->fgfs[gyz0->sgfn],cg->fgfs[gzz0->sgfn],cg->fgfs[Axx0->sgfn],cg->fgfs[Axy0->sgfn],cg->fgfs[Axz0->sgfn],cg->fgfs[Ayy0->sgfn],cg->fgfs[Ayz0->sgfn],cg->fgfs[Azz0->sgfn],cg->fgfs[Gmx0->sgfn],cg->fgfs[Gmy0->sgfn],cg->fgfs[Gmz0->sgfn],cg->fgfs[Lap0->sgfn],cg->fgfs[Sfx0->sgfn],cg->fgfs[Sfy0->sgfn],cg->fgfs[Sfz0->sgfn],cg->fgfs[dtSfx0->sgfn],cg->fgfs[dtSfy0->sgfn],cg->fgfs[dtSfz0->sgfn],cg->fgfs[phi_rhs->sgfn],cg->fgfs[trK_rhs->sgfn],cg->fgfs[gxx_rhs->sgfn],cg->fgfs[gxy_rhs->sgfn],cg->fgfs[gxz_rhs->sgfn],cg->fgfs[gyy_rhs->sgfn],cg->fgfs[gyz_rhs->sgfn],cg->fgfs[gzz_rhs->sgfn],cg->fgfs[Axx_rhs->sgfn],cg->fgfs[Axy_rhs->sgfn],cg->fgfs[Axz_rhs->sgfn],cg->fgfs[Ayy_rhs->sgfn],cg->fgfs[Ayz_rhs->sgfn],cg->fgfs[Azz_rhs->sgfn],cg->fgfs[Gmx_rhs->sgfn],cg->fgfs[Gmy_rhs->sgfn],cg->fgfs[Gmz_rhs->sgfn],cg->fgfs[Lap_rhs->sgfn],cg->fgfs[Sfx_rhs->sgfn],cg->fgfs[Sfy_rhs->sgfn],cg->fgfs[Sfz_rhs->sgfn],cg->fgfs[dtSfx_rhs->sgfn],cg->fgfs[dtSfy_rhs->sgfn],cg->fgfs[dtSfz_rhs->sgfn],cg->fgfs[rho->sgfn],cg->fgfs[Sx->sgfn],cg->fgfs[Sy->sgfn],cg->fgfs[Sz->sgfn],cg->fgfs[Sxx->sgfn],cg->fgfs[Sxy->sgfn],cg->fgfs[Sxz->sgfn],cg->fgfs[Syy->sgfn],cg->fgfs[Syz->sgfn],cg->fgfs[Szz->sgfn],cg->fgfs[Gamxxx->sgfn],cg->fgfs[Gamxxy->sgfn],cg->fgfs[Gamxxz->sgfn],cg->fgfs[Gamxyy->sgfn],cg->fgfs[Gamxyz->sgfn],cg->fgfs[Gamxzz->sgfn],cg->fgfs[Gamyxx->sgfn],cg->fgfs[Gamyxy->sgfn],cg->fgfs[Gamyxz->sgfn],cg->fgfs[Gamyyy->sgfn],cg->fgfs[Gamyyz->sgfn],cg->fgfs[Gamyzz->sgfn],cg->fgfs[Gamzxx->sgfn],cg->fgfs[Gamzxy->sgfn],cg->fgfs[Gamzxz->sgfn],cg->fgfs[Gamzyy->sgfn],cg->fgfs[Gamzyz->sgfn],cg->fgfs[Gamzzz->sgfn],cg->fgfs[Rxx->sgfn],cg->fgfs[Rxy->sgfn],cg->fgfs[Rxz->sgfn],cg->fgfs[Ryy->sgfn],cg->fgfs[Ryz->sgfn],cg->fgfs[Rzz->sgfn],cg->fgfs[Cons_Ham->sgfn],cg->fgfs[Cons_Px->sgfn],cg->fgfs[Cons_Py->sgfn],cg->fgfs[Cons_Pz->sgfn],cg->fgfs[Cons_Gx->sgfn],cg->fgfs[Cons_Gy->sgfn],cg->fgfs[Cons_Gz->sgfn],Symmetry,lev,numepsh,sPp->data->sst,pre
#define RHS_SS_PARA_CALLED_THEN cg->shape,TRK4,cg->X[0],cg->X[1],cg->X[2],cg->fgfs[fngfs+ShellPatch::gx],cg->fgfs[fngfs+ShellPatch::gy],cg->fgfs[fngfs+ShellPatch::gz],cg->fgfs[fngfs+ShellPatch::drhodx],cg->fgfs[fngfs+ShellPatch::drhody],cg->fgfs[fngfs+ShellPatch::drhodz],cg->fgfs[fngfs+ShellPatch::dsigmadx],cg->fgfs[fngfs+ShellPatch::dsigmady],cg->fgfs[fngfs+ShellPatch::dsigmadz],cg->fgfs[fngfs+ShellPatch::dRdx],cg->fgfs[fngfs+ShellPatch::dRdy],cg->fgfs[fngfs+ShellPatch::dRdz],cg->fgfs[fngfs+ShellPatch::drhodxx],cg->fgfs[fngfs+ShellPatch::drhodxy],cg->fgfs[fngfs+ShellPatch::drhodxz],cg->fgfs[fngfs+ShellPatch::drhodyy],cg->fgfs[fngfs+ShellPatch::drhodyz],cg->fgfs[fngfs+ShellPatch::drhodzz],cg->fgfs[fngfs+ShellPatch::dsigmadxx],cg->fgfs[fngfs+ShellPatch::dsigmadxy],cg->fgfs[fngfs+ShellPatch::dsigmadxz],cg->fgfs[fngfs+ShellPatch::dsigmadyy],cg->fgfs[fngfs+ShellPatch::dsigmadyz],cg->fgfs[fngfs+ShellPatch::dsigmadzz],cg->fgfs[fngfs+ShellPatch::dRdxx],cg->fgfs[fngfs+ShellPatch::dRdxy],cg->fgfs[fngfs+ShellPatch::dRdxz],cg->fgfs[fngfs+ShellPatch::dRdyy],cg->fgfs[fngfs+ShellPatch::dRdyz],cg->fgfs[fngfs+ShellPatch::dRdzz],cg->fgfs[phi->sgfn],cg->fgfs[trK->sgfn],cg->fgfs[gxx->sgfn],cg->fgfs[gxy->sgfn],cg->fgfs[gxz->sgfn],cg->fgfs[gyy->sgfn],cg->fgfs[gyz->sgfn],cg->fgfs[gzz->sgfn],cg->fgfs[Axx->sgfn],cg->fgfs[Axy->sgfn],cg->fgfs[Axz->sgfn],cg->fgfs[Ayy->sgfn],cg->fgfs[Ayz->sgfn],cg->fgfs[Azz->sgfn],cg->fgfs[Gmx->sgfn],cg->fgfs[Gmy->sgfn],cg->fgfs[Gmz->sgfn],cg->fgfs[Lap->sgfn],cg->fgfs[Sfx->sgfn],cg->fgfs[Sfy->sgfn],cg->fgfs[Sfz->sgfn],cg->fgfs[dtSfx->sgfn],cg->fgfs[dtSfy->sgfn],cg->fgfs[dtSfz->sgfn],cg->fgfs[phi1->sgfn],cg->fgfs[trK1->sgfn],cg->fgfs[gxx1->sgfn],cg->fgfs[gxy1->sgfn],cg->fgfs[gxz1->sgfn],cg->fgfs[gyy1->sgfn],cg->fgfs[gyz1->sgfn],cg->fgfs[gzz1->sgfn],cg->fgfs[Axx1->sgfn],cg->fgfs[Axy1->sgfn],cg->fgfs[Axz1->sgfn],cg->fgfs[Ayy1->sgfn],cg->fgfs[Ayz1->sgfn],cg->fgfs[Azz1->sgfn],cg->fgfs[Gmx1->sgfn],cg->fgfs[Gmy1->sgfn],cg->fgfs[Gmz1->sgfn],cg->fgfs[Lap1->sgfn],cg->fgfs[Sfx1->sgfn],cg->fgfs[Sfy1->sgfn],cg->fgfs[Sfz1->sgfn],cg->fgfs[dtSfx1->sgfn],cg->fgfs[dtSfy1->sgfn],cg->fgfs[dtSfz1->sgfn],cg->fgfs[rho->sgfn],cg->fgfs[Sx->sgfn],cg->fgfs[Sy->sgfn],cg->fgfs[Sz->sgfn],cg->fgfs[Sxx->sgfn],cg->fgfs[Sxy->sgfn],cg->fgfs[Sxz->sgfn],cg->fgfs[Syy->sgfn],cg->fgfs[Syz->sgfn],cg->fgfs[Szz->sgfn],cg->fgfs[Gamxxx->sgfn],cg->fgfs[Gamxxy->sgfn],cg->fgfs[Gamxxz->sgfn],cg->fgfs[Gamxyy->sgfn],cg->fgfs[Gamxyz->sgfn],cg->fgfs[Gamxzz->sgfn],cg->fgfs[Gamyxx->sgfn],cg->fgfs[Gamyxy->sgfn],cg->fgfs[Gamyxz->sgfn],cg->fgfs[Gamyyy->sgfn],cg->fgfs[Gamyyz->sgfn],cg->fgfs[Gamyzz->sgfn],cg->fgfs[Gamzxx->sgfn],cg->fgfs[Gamzxy->sgfn],cg->fgfs[Gamzxz->sgfn],cg->fgfs[Gamzyy->sgfn],cg->fgfs[Gamzyz->sgfn],cg->fgfs[Gamzzz->sgfn],cg->fgfs[Rxx->sgfn],cg->fgfs[Rxy->sgfn],cg->fgfs[Rxz->sgfn],cg->fgfs[Ryy->sgfn],cg->fgfs[Ryz->sgfn],cg->fgfs[Rzz->sgfn],cg->fgfs[Cons_Ham->sgfn],cg->fgfs[Cons_Px->sgfn],cg->fgfs[Cons_Py->sgfn],cg->fgfs[Cons_Pz->sgfn],cg->fgfs[Cons_Gx->sgfn],cg->fgfs[Cons_Gy->sgfn],cg->fgfs[Cons_Gz->sgfn],Symmetry,lev,numepsh,sPp->data->sst,cor
#define RHS_PARA_CALLED_Constraint_Out_SS cg->shape,TRK4,cg->X[0],cg->X[1],cg->X[2],cg->fgfs[fngfs+ShellPatch::gx],cg->fgfs[fngfs+ShellPatch::gy],cg->fgfs[fngfs+ShellPatch::gz],cg->fgfs[fngfs+ShellPatch::drhodx],cg->fgfs[fngfs+ShellPatch::drhody],cg->fgfs[fngfs+ShellPatch::drhodz],cg->fgfs[fngfs+ShellPatch::dsigmadx],cg->fgfs[fngfs+ShellPatch::dsigmady],cg->fgfs[fngfs+ShellPatch::dsigmadz],cg->fgfs[fngfs+ShellPatch::dRdx],cg->fgfs[fngfs+ShellPatch::dRdy],cg->fgfs[fngfs+ShellPatch::dRdz],cg->fgfs[fngfs+ShellPatch::drhodxx],cg->fgfs[fngfs+ShellPatch::drhodxy],cg->fgfs[fngfs+ShellPatch::drhodxz],cg->fgfs[fngfs+ShellPatch::drhodyy],cg->fgfs[fngfs+ShellPatch::drhodyz],cg->fgfs[fngfs+ShellPatch::drhodzz],cg->fgfs[fngfs+ShellPatch::dsigmadxx],cg->fgfs[fngfs+ShellPatch::dsigmadxy],cg->fgfs[fngfs+ShellPatch::dsigmadxz],cg->fgfs[fngfs+ShellPatch::dsigmadyy],cg->fgfs[fngfs+ShellPatch::dsigmadyz],cg->fgfs[fngfs+ShellPatch::dsigmadzz],cg->fgfs[fngfs+ShellPatch::dRdxx],cg->fgfs[fngfs+ShellPatch::dRdxy],cg->fgfs[fngfs+ShellPatch::dRdxz],cg->fgfs[fngfs+ShellPatch::dRdyy],cg->fgfs[fngfs+ShellPatch::dRdyz],cg->fgfs[fngfs+ShellPatch::dRdzz],cg->fgfs[phi0->sgfn],cg->fgfs[trK0->sgfn],cg->fgfs[gxx0->sgfn],cg->fgfs[gxy0->sgfn],cg->fgfs[gxz0->sgfn],cg->fgfs[gyy0->sgfn],cg->fgfs[gyz0->sgfn],cg->fgfs[gzz0->sgfn],cg->fgfs[Axx0->sgfn],cg->fgfs[Axy0->sgfn],cg->fgfs[Axz0->sgfn],cg->fgfs[Ayy0->sgfn],cg->fgfs[Ayz0->sgfn],cg->fgfs[Azz0->sgfn],cg->fgfs[Gmx0->sgfn],cg->fgfs[Gmy0->sgfn],cg->fgfs[Gmz0->sgfn],cg->fgfs[Lap0->sgfn],cg->fgfs[Sfx0->sgfn],cg->fgfs[Sfy0->sgfn],cg->fgfs[Sfz0->sgfn],cg->fgfs[dtSfx0->sgfn],cg->fgfs[dtSfy0->sgfn],cg->fgfs[dtSfz0->sgfn],cg->fgfs[phi_rhs->sgfn],cg->fgfs[trK_rhs->sgfn],cg->fgfs[gxx_rhs->sgfn],cg->fgfs[gxy_rhs->sgfn],cg->fgfs[gxz_rhs->sgfn],cg->fgfs[gyy_rhs->sgfn],cg->fgfs[gyz_rhs->sgfn],cg->fgfs[gzz_rhs->sgfn],cg->fgfs[Axx_rhs->sgfn],cg->fgfs[Axy_rhs->sgfn],cg->fgfs[Axz_rhs->sgfn],cg->fgfs[Ayy_rhs->sgfn],cg->fgfs[Ayz_rhs->sgfn],cg->fgfs[Azz_rhs->sgfn],cg->fgfs[Gmx_rhs->sgfn],cg->fgfs[Gmy_rhs->sgfn],cg->fgfs[Gmz_rhs->sgfn],cg->fgfs[Lap_rhs->sgfn],cg->fgfs[Sfx_rhs->sgfn],cg->fgfs[Sfy_rhs->sgfn],cg->fgfs[Sfz_rhs->sgfn],cg->fgfs[dtSfx_rhs->sgfn],cg->fgfs[dtSfy_rhs->sgfn],cg->fgfs[dtSfz_rhs->sgfn],cg->fgfs[rho->sgfn],cg->fgfs[Sx->sgfn],cg->fgfs[Sy->sgfn],cg->fgfs[Sz->sgfn],cg->fgfs[Sxx->sgfn],cg->fgfs[Sxy->sgfn],cg->fgfs[Sxz->sgfn],cg->fgfs[Syy->sgfn],cg->fgfs[Syz->sgfn],cg->fgfs[Szz->sgfn],cg->fgfs[Gamxxx->sgfn],cg->fgfs[Gamxxy->sgfn],cg->fgfs[Gamxxz->sgfn],cg->fgfs[Gamxyy->sgfn],cg->fgfs[Gamxyz->sgfn],cg->fgfs[Gamxzz->sgfn],cg->fgfs[Gamyxx->sgfn],cg->fgfs[Gamyxy->sgfn],cg->fgfs[Gamyxz->sgfn],cg->fgfs[Gamyyy->sgfn],cg->fgfs[Gamyyz->sgfn],cg->fgfs[Gamyzz->sgfn],cg->fgfs[Gamzxx->sgfn],cg->fgfs[Gamzxy->sgfn],cg->fgfs[Gamzxz->sgfn],cg->fgfs[Gamzyy->sgfn],cg->fgfs[Gamzyz->sgfn],cg->fgfs[Gamzzz->sgfn],cg->fgfs[Rxx->sgfn],cg->fgfs[Rxy->sgfn],cg->fgfs[Rxz->sgfn],cg->fgfs[Ryy->sgfn],cg->fgfs[Ryz->sgfn],cg->fgfs[Rzz->sgfn],cg->fgfs[Cons_Ham->sgfn],cg->fgfs[Cons_Px->sgfn],cg->fgfs[Cons_Py->sgfn],cg->fgfs[Cons_Pz->sgfn],cg->fgfs[Cons_Gx->sgfn],cg->fgfs[Cons_Gy->sgfn],cg->fgfs[Cons_Gz->sgfn],Symmetry,lev,numepsh,sPp->data->sst,pre
#define RHS_PARA_CALLED_Intrp_Constraint_Out_SS cg->shape,TRK4,cg->X[0],cg->X[1],cg->X[2],cg->fgfs[fngfs+ShellPatch::gx],cg->fgfs[fngfs+ShellPatch::gy],cg->fgfs[fngfs+ShellPatch::gz],cg->fgfs[fngfs+ShellPatch::drhodx],cg->fgfs[fngfs+ShellPatch::drhody],cg->fgfs[fngfs+ShellPatch::drhodz],cg->fgfs[fngfs+ShellPatch::dsigmadx],cg->fgfs[fngfs+ShellPatch::dsigmady],cg->fgfs[fngfs+ShellPatch::dsigmadz],cg->fgfs[fngfs+ShellPatch::dRdx],cg->fgfs[fngfs+ShellPatch::dRdy],cg->fgfs[fngfs+ShellPatch::dRdz],cg->fgfs[fngfs+ShellPatch::drhodxx],cg->fgfs[fngfs+ShellPatch::drhodxy],cg->fgfs[fngfs+ShellPatch::drhodxz],cg->fgfs[fngfs+ShellPatch::drhodyy],cg->fgfs[fngfs+ShellPatch::drhodyz],cg->fgfs[fngfs+ShellPatch::drhodzz],cg->fgfs[fngfs+ShellPatch::dsigmadxx],cg->fgfs[fngfs+ShellPatch::dsigmadxy],cg->fgfs[fngfs+ShellPatch::dsigmadxz],cg->fgfs[fngfs+ShellPatch::dsigmadyy],cg->fgfs[fngfs+ShellPatch::dsigmadyz],cg->fgfs[fngfs+ShellPatch::dsigmadzz],cg->fgfs[fngfs+ShellPatch::dRdxx],cg->fgfs[fngfs+ShellPatch::dRdxy],cg->fgfs[fngfs+ShellPatch::dRdxz],cg->fgfs[fngfs+ShellPatch::dRdyy],cg->fgfs[fngfs+ShellPatch::dRdyz],cg->fgfs[fngfs+ShellPatch::dRdzz],cg->fgfs[phi0->sgfn],cg->fgfs[trK0->sgfn],cg->fgfs[gxx0->sgfn],cg->fgfs[gxy0->sgfn],cg->fgfs[gxz0->sgfn],cg->fgfs[gyy0->sgfn],cg->fgfs[gyz0->sgfn],cg->fgfs[gzz0->sgfn],cg->fgfs[Axx0->sgfn],cg->fgfs[Axy0->sgfn],cg->fgfs[Axz0->sgfn],cg->fgfs[Ayy0->sgfn],cg->fgfs[Ayz0->sgfn],cg->fgfs[Azz0->sgfn],cg->fgfs[Gmx0->sgfn],cg->fgfs[Gmy0->sgfn],cg->fgfs[Gmz0->sgfn],cg->fgfs[Lap0->sgfn],cg->fgfs[Sfx0->sgfn],cg->fgfs[Sfy0->sgfn],cg->fgfs[Sfz0->sgfn],cg->fgfs[dtSfx0->sgfn],cg->fgfs[dtSfy0->sgfn],cg->fgfs[dtSfz0->sgfn],cg->fgfs[phi_rhs->sgfn],cg->fgfs[trK_rhs->sgfn],cg->fgfs[gxx_rhs->sgfn],cg->fgfs[gxy_rhs->sgfn],cg->fgfs[gxz_rhs->sgfn],cg->fgfs[gyy_rhs->sgfn],cg->fgfs[gyz_rhs->sgfn],cg->fgfs[gzz_rhs->sgfn],cg->fgfs[Axx_rhs->sgfn],cg->fgfs[Axy_rhs->sgfn],cg->fgfs[Axz_rhs->sgfn],cg->fgfs[Ayy_rhs->sgfn],cg->fgfs[Ayz_rhs->sgfn],cg->fgfs[Azz_rhs->sgfn],cg->fgfs[Gmx_rhs->sgfn],cg->fgfs[Gmy_rhs->sgfn],cg->fgfs[Gmz_rhs->sgfn],cg->fgfs[Lap_rhs->sgfn],cg->fgfs[Sfx_rhs->sgfn],cg->fgfs[Sfy_rhs->sgfn],cg->fgfs[Sfz_rhs->sgfn],cg->fgfs[dtSfx_rhs->sgfn],cg->fgfs[dtSfy_rhs->sgfn],cg->fgfs[dtSfz_rhs->sgfn],cg->fgfs[rho->sgfn],cg->fgfs[Sx->sgfn],cg->fgfs[Sy->sgfn],cg->fgfs[Sz->sgfn],cg->fgfs[Sxx->sgfn],cg->fgfs[Sxy->sgfn],cg->fgfs[Sxz->sgfn],cg->fgfs[Syy->sgfn],cg->fgfs[Syz->sgfn],cg->fgfs[Szz->sgfn],cg->fgfs[Gamxxx->sgfn],cg->fgfs[Gamxxy->sgfn],cg->fgfs[Gamxxz->sgfn],cg->fgfs[Gamxyy->sgfn],cg->fgfs[Gamxyz->sgfn],cg->fgfs[Gamxzz->sgfn],cg->fgfs[Gamyxx->sgfn],cg->fgfs[Gamyxy->sgfn],cg->fgfs[Gamyxz->sgfn],cg->fgfs[Gamyyy->sgfn],cg->fgfs[Gamyyz->sgfn],cg->fgfs[Gamyzz->sgfn],cg->fgfs[Gamzxx->sgfn],cg->fgfs[Gamzxy->sgfn],cg->fgfs[Gamzxz->sgfn],cg->fgfs[Gamzyy->sgfn],cg->fgfs[Gamzyz->sgfn],cg->fgfs[Gamzzz->sgfn],cg->fgfs[Rxx->sgfn],cg->fgfs[Rxy->sgfn],cg->fgfs[Rxz->sgfn],cg->fgfs[Ryy->sgfn],cg->fgfs[Ryz->sgfn],cg->fgfs[Rzz->sgfn],cg->fgfs[Cons_Ham->sgfn],cg->fgfs[Cons_Px->sgfn],cg->fgfs[Cons_Py->sgfn],cg->fgfs[Cons_Pz->sgfn],cg->fgfs[Cons_Gx->sgfn],cg->fgfs[Cons_Gy->sgfn],cg->fgfs[Cons_Gz->sgfn],Symmetry,lev,numepsh,sPp->data->sst,pre
//4------------tool------------------------------
int compare_result(int ftag1,double * d2,int data_num);
#endif

1942
AMSS_NCKU_source/BSSN_GPU/bssn_step_gpu.C Normal file
View File

File diff suppressed because it is too large Load Diff

146
AMSS_NCKU_source/BSSN_GPU/gpu_mem.h Normal file
View File

@@ -0,0 +1,146 @@
#ifndef GPU_MEM_H_
#define GPU_MEM_H_
#include "macrodef.fh"
struct Meta
{
//---------------in/out-------------------
// int * ex;
// int* Symmetry,Lev,co; //not array //in
// double * T; //not array //in
double *X, *Y, *Z; // in
double *chi, *dxx, *dyy, *dzz; // inout
double *trK; // in
double *gxy, *gxz, *gyz; // in
double *Axx, *Axy, *Axz, *Ayy, *Ayz, *Azz; // in
double *Gamx, *Gamy, *Gamz; // in
double *Lap, *betax, *betay, *betaz; // inout
double *dtSfx, *dtSfy, *dtSfz; // in
double *chi_rhs, *trK_rhs; // out
double *gxx_rhs, *gxy_rhs, *gxz_rhs; // out
double *gyy_rhs, *gyz_rhs, *gzz_rhs; // out
double *Axx_rhs, *Axy_rhs, *Axz_rhs; // out
double *Ayy_rhs, *Ayz_rhs, *Azz_rhs; // out
double *Gamx_rhs, *Gamy_rhs, *Gamz_rhs; // out
double *Lap_rhs, *betax_rhs, *betay_rhs, *betaz_rhs; // out
double *dtSfx_rhs, *dtSfy_rhs, *dtSfz_rhs; // out
double *rho, *Sx, *Sy, *Sz; // in
double *Sxx, *Sxy, *Sxz, *Syy, *Syz, *Szz; // in
// when out, physical second kind of connection //out
double *Gamxxx, *Gamxxy, *Gamxxz;
double *Gamxyy, *Gamxyz, *Gamxzz;
double *Gamyxx, *Gamyxy, *Gamyxz;
double *Gamyyy, *Gamyyz, *Gamyzz;
double *Gamzxx, *Gamzxy, *Gamzxz;
double *Gamzyy, *Gamzyz, *Gamzzz;
// when out, physical Ricci tensor
double *Rxx, *Rxy, *Rxz, *Ryy, *Ryz, *Rzz; // out
// double * eps; //in
double *ham_Res, *movx_Res, *movy_Res, *movz_Res; // inout
double *Gmx_Res, *Gmy_Res, *Gmz_Res; // inout
//---------------local-------------------
double *gxx, *gyy, *gzz, *chix, *chiy, *chiz, *gxxx, *gxyx, *gxzx, *gyyx, *gyzx, *gzzx, *gxxy, *gxyy, *gxzy, *gyyy, *gyzy, *gzzy, *gxxz, *gxyz, *gxzz, *gyyz, *gyzz, *gzzz, *Lapx, *Lapy, *Lapz, *betaxx, *betaxy, *betaxz, *betayx, *betayy, *betayz, *betazx, *betazy, *betazz, *Gamxx, *Gamxy, *Gamxz, *Gamyx, *Gamyy, *Gamyz, *Gamzx, *Gamzy, *Gamzz, *Kx, *Ky, *Kz, *div_beta, *S, *f, *fxx, *fxy, *fxz, *fyy, *fyz, *fzz, *Gamxa, *Gamya, *Gamza, *alpn1, *chin1, *gupxx, *gupxy, *gupxz, *gupyy, *gupyz, *gupzz;
//---------------subroutine----------------
double *fh;
double *fh2;
/*double *SSS;
double *AAS;
double *ASA;
double *SAA;
double *ASS;
double *SAS;
double *SSA;*/
//---------------GAUGE--------------
#if (GAUGE == 2 || GAUGE == 3 || GAUGE == 4 || GAUGE == 5 || GAUGE == 6 || GAUGE == 7)
double *reta;
#endif
};
//------init constant memory---------
// 1-----for compute_rhs_bssn---------
__constant__ Meta metac;
__constant__ int ex_c[3];
__constant__ double T_c;
__constant__ int Symmetry_c;
__constant__ int Lev_c;
__constant__ int co_c;
__constant__ double eps_c;
// local
__constant__ double dX; // dX,dY,dZ
__constant__ double dY;
__constant__ double dZ;
__constant__ double ZEO = 1.0;
__constant__ double ONE = 1.0;
__constant__ double TWO = 2.0;
__constant__ double FOUR = 4.0;
__constant__ double EIGHT = 8.0;
__constant__ double HALF = 0.5;
__constant__ double THR = 3.0;
__constant__ double SYM = 1.0;
__constant__ double ANTI = -1.0;
__constant__ double FF = 0.75;
__constant__ double eta = 2.0;
__constant__ double F1o3;
__constant__ double F2o3;
__constant__ double F3o2 = 1.5;
__constant__ double F1o6;
__constant__ double F8 = 8.0;
__constant__ double F16 = 16.0;
__constant__ double PI;
/*__constant__ double SSS[3] = {1,1,1};
__constant__ double AAS[3] = {-1,-1,1};
__constant__ double ASA[3] = {-1,1,-1};
__constant__ double SAA[3] = {1,-1,-1};
__constant__ double ASS[3] = {-1,1,1};
__constant__ double SAS[3] = {1,-1,1};
__constant__ double SSA[3] = {1,1,-1};*/
// 2--------for fderivs------------
__constant__ int ijk_min[3];
__constant__ int ijk_min2[3];
__constant__ int ijk_min3[3];
__constant__ int ijk_max[3];
__constant__ double d12dxyz[3];
__constant__ double d2dxyz[3];
// 3--------for fdderivs------------
__constant__ double Sdxdx;
__constant__ double Sdydy;
__constant__ double Sdzdz;
__constant__ double Fdxdx;
__constant__ double Fdydy;
__constant__ double Fdzdz;
__constant__ double Sdxdy;
__constant__ double Sdxdz;
__constant__ double Sdydz;
__constant__ double Fdxdy;
__constant__ double Fdxdz;
__constant__ double Fdydz;
// my own
__constant__ int STEP_SIZE;
/*__constant__ int MATRIX_SIZE;
__constant__ int MATRIX_SIZE_FH;
__constant__ int SQUARE_SIZE;
__constant__ int SQUARE_SIZE_FH;
__constant__ int LINE_SIZE_FH;*/
__constant__ int _1D_SIZE[4]; // start from 0 !!
__constant__ int _2D_SIZE[4]; ////start from 0 !!
__constant__ int _3D_SIZE[4]; ////start from 0 !!
#if (GAUGE == 6 || GAUGE == 7)
__constant__ int BHN;
__constant__ double Porg[9];
__constant__ double Mass[3];
__constant__ double /*r1,r2*/, M, A, /*w1,w2 (== 12)*/, C1, C2;
#endif
/**/
#endif

198
AMSS_NCKU_source/BSSN_GPU/gpu_rhsSS_mem.h Normal file
View File

@@ -0,0 +1,198 @@
#ifndef GPU_MEM_H_
#define GPU_MEM_H_
#include "macrodef.fh"
#ifdef WithShell
struct Metass
{
double *crho,* sigma,* R,*
drhodx,* drhody,* drhodz,*
dsigmadx,* dsigmady,* dsigmadz,*
dRdx,* dRdy,* dRdz,*
drhodxx,* drhodxy,* drhodxz,*
drhodyy,* drhodyz,* drhodzz,*
dsigmadxx,* dsigmadxy,* dsigmadxz,*
dsigmadyy,* dsigmadyz,* dsigmadzz,*
dRdxx,* dRdxy,* dRdxz,*
dRdyy,* dRdyz,* dRdzz;
//local
double *gx,*gy,*gz,*gxx,*gxy,*gxz,*gyy,*gyz,*gzz;
};
__constant__ Metass metassc;
Metass * metass;
#endif //WithShell
struct Meta
{
//SS
//---------------in/out-------------------
//int * ex;
//int* Symmetry,Lev,co; //not array //in
//double * T; //not array //in
double * X,*Y,*Z; //in
double * chi,*dxx,*dyy,*dzz; //inout
double * trK ; //in
double * gxy,*gxz,*gyz; //in
double * Axx,*Axy,*Axz,*Ayy,*Ayz,*Azz; //in
double * Gamx,*Gamy,*Gamz ; //in
double * Lap, *betax, *betay, *betaz; //inout
double * dtSfx, *dtSfy, *dtSfz ; //in
double * chi_rhs,*trK_rhs ; //out
double * gxx_rhs,*gxy_rhs,*gxz_rhs; //out
double * gyy_rhs,*gyz_rhs,*gzz_rhs; //out
double * Axx_rhs,*Axy_rhs,*Axz_rhs; //out
double * Ayy_rhs,*Ayz_rhs,*Azz_rhs; //out
double * Gamx_rhs,*Gamy_rhs,*Gamz_rhs;//out
double * Lap_rhs, *betax_rhs, *betay_rhs, *betaz_rhs;//out
double * dtSfx_rhs,*dtSfy_rhs,*dtSfz_rhs;//out
double * rho,*Sx,*Sy,*Sz ; //in
double * Sxx,*Sxy,*Sxz,*Syy,*Syz,*Szz; //in
// when out, physical second kind of connection //out
double * Gamxxx, *Gamxxy, *Gamxxz;
double * Gamxyy, *Gamxyz, *Gamxzz;
double * Gamyxx, *Gamyxy, *Gamyxz;
double * Gamyyy, *Gamyyz, *Gamyzz;
double * Gamzxx, *Gamzxy,* Gamzxz;
double * Gamzyy, *Gamzyz, *Gamzzz;
//when out, physical Ricci tensor
double * Rxx,*Rxy,*Rxz,*Ryy,*Ryz,*Rzz; //out
//double * eps; //in
double * ham_Res, *movx_Res, *movy_Res, *movz_Res; //inout
double * Gmx_Res, *Gmy_Res, *Gmz_Res; //inout
//---------------local-------------------
double * gxx,*gyy,*gzz
, *chix,*chiy,*chiz
, *gxxx,*gxyx,*gxzx,*gyyx,*gyzx,*gzzx
, *gxxy,*gxyy,*gxzy,*gyyy,*gyzy,*gzzy
, *gxxz,*gxyz,*gxzz,*gyyz,*gyzz,*gzzz
, *Lapx,*Lapy,*Lapz
, *betaxx,*betaxy,*betaxz
, *betayx,*betayy,*betayz
, *betazx,*betazy,*betazz
, *Gamxx,*Gamxy,*Gamxz
, *Gamyx,*Gamyy,*Gamyz
, *Gamzx,*Gamzy,*Gamzz
, *Kx,*Ky,*Kz,*div_beta,*S
, *f,*fxx,*fxy,*fxz,*fyy,*fyz,*fzz
, *Gamxa,*Gamya,*Gamza,*alpn1,*chin1
, *gupxx,*gupxy,*gupxz
, *gupyy,*gupyz,*gupzz;
//---------------subroutine----------------
double * fh;
double * fh2;
/*double *SSS;
double *AAS;
double *ASA;
double *SAA;
double *ASS;
double *SAS;
double *SSA;*/
//---------------GAUGE--------------
#if (GAUGE == 2 || GAUGE == 3 || GAUGE == 4 || GAUGE == 5 || GAUGE == 6 || GAUGE == 7)
double * reta;
#endif
};
//------init constant memory---------
//1-----for compute_rhs_bssn---------
__constant__ Meta metac;
__constant__ int ex_c[3];
__constant__ double T_c;
__constant__ int Symmetry_c;
__constant__ int Lev_c;
__constant__ int co_c;
__constant__ double eps_c;
__constant__ int sst_c;
//local
__constant__ double dX; //dX,dY,dZ
__constant__ double dY;
__constant__ double dZ;
__constant__ double ZEO = 1.0;
__constant__ double ONE = 1.0;
__constant__ double TWO = 2.0;
__constant__ double FOUR = 4.0;
__constant__ double EIGHT = 8.0;
__constant__ double HALF = 0.5;
__constant__ double THR = 3.0;
__constant__ double SYM = 1.0;
__constant__ double ANTI = -1.0;
__constant__ double FF = 0.75;
__constant__ double eta = 2.0;
__constant__ double F1o3;
__constant__ double F2o3;
__constant__ double F3o2 = 1.5;
__constant__ double F1o6;
__constant__ double F8 = 8.0;
__constant__ double F16 = 16.0;
__constant__ double PI;
/*__constant__ double SSS[3] = {1,1,1};
__constant__ double AAS[3] = {-1,-1,1};
__constant__ double ASA[3] = {-1,1,-1};
__constant__ double SAA[3] = {1,-1,-1};
__constant__ double ASS[3] = {-1,1,1};
__constant__ double SAS[3] = {1,-1,1};
__constant__ double SSA[3] = {1,1,-1};*/
//2--------for fderivs------------
__constant__ int ijk_min[3];
__constant__ int ijk_min2[3];
__constant__ int ijk_min3[3];
__constant__ int ijk_max[3];
__constant__ int ijk_max3[3];
__constant__ double d12dxyz[3];
__constant__ double d2dxyz[3];
//3--------for fdderivs------------
__constant__ double Sdxdx;
__constant__ double Sdydy;
__constant__ double Sdzdz;
__constant__ double Fdxdx;
__constant__ double Fdydy;
__constant__ double Fdzdz;
__constant__ double Sdxdy;
__constant__ double Sdxdz;
__constant__ double Sdydz;
__constant__ double Fdxdy;
__constant__ double Fdxdz;
__constant__ double Fdydz;
//my own
__constant__ int STEP_SIZE;
/*__constant__ int MATRIX_SIZE;
__constant__ int MATRIX_SIZE_FH;
__constant__ int SQUARE_SIZE;
__constant__ int SQUARE_SIZE_FH;
__constant__ int LINE_SIZE_FH;*/
__constant__ int _1D_SIZE[4]; //start from 0 !!
__constant__ int _2D_SIZE[4]; ////start from 0 !!
__constant__ int _3D_SIZE[4]; ////start from 0 !!
int h_1D_SIZE[4]; //start from 0 !!
int h_2D_SIZE[4]; ////start from 0 !!
int h_3D_SIZE[4]; ////start from 0 !!
Meta * meta;
#if (GAUGE == 6 || GAUGE == 7)
__constant__ int BHN;
__constant__ double Porg[9];
__constant__ double Mass[3];
__constant__ double /*r1,r2*/,M,A,/*w1,w2 (== 12)*/,C1,C2;
#endif
/**/
#endif