64-BitBrainstorm_2026/AMSS-NCKU
8
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
50e2a845f8c3e936e334514a66051082dcc41505
AMSS-NCKU/AMSS_NCKU_source/bssn_class.C
CGH0S7 738498cb28 Optimize MPI communication in RestrictProlong and surface_integral 
Cache Sync in RestrictProlong: replace 11 basic Parallel::Sync() calls
with Parallel::Sync_cached() across RestrictProlong, RestrictProlong_aux,
and ProlongRestrict to avoid rebuilding grid segment lists every call.

Merge paired MPI_Allreduce in surface_integral: combine 9 pairs of
consecutive RP/IP Allreduce calls into single calls with count=2*NN.

Merge scalar MPI_Allreduce in surf_MassPAng: combine 3 groups of 7
scalar Allreduce calls (mass + angular/linear momentum) into single
calls with count=7.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-02-09 22:07:12 +08:00

8492 lines
339 KiB
C
Raw Blame History

#ifdef newc
#include <typeinfo>
#include <sstream>
#include <cstdio>
#include <map>
#include <string>
#include <cstring>
#include <iostream>
using namespace std;
#else
#include <stdio.h>
#include <map.h>
#include <string.h>
#endif
#include <time.h>
#include "macrodef.h"
#include "misc.h"
#include "Ansorg.h"
#include "fmisc.h"
#include "Parallel.h"
#include "bssn_class.h"
#include "bssn_rhs.h"
#include "initial_puncture.h"
#include "enforce_algebra.h"
#include "rungekutta4_rout.h"
#include "sommerfeld_rout.h"
#include "getnp4.h"
#include "shellfunctions.h"
#include "parameters.h"
#ifdef With_AHF
#include "derivatives.h"
#include "myglobal.h"
#endif
#include "perf.h"
#include "derivatives.h"
#include "ricci_gamma.h"
//================================================================================================
// define bssn_class
//================================================================================================
bssn_class::bssn_class(double Couranti, double StartTimei, double TotalTimei,
double DumpTimei, double d2DumpTimei, double CheckTimei, double AnasTimei,
int Symmetryi, int checkruni, char *checkfilenamei,
double numepssi, double numepsbi, double numepshi,
int a_levi, int maxli, int decni, double maxrexi, double drexi)
: Courant(Couranti), StartTime(StartTimei), TotalTime(TotalTimei),
DumpTime(DumpTimei), d2DumpTime(d2DumpTimei), CheckTime(CheckTimei), AnasTime(AnasTimei),
Symmetry(Symmetryi), checkrun(checkruni), numepss(numepssi), numepsb(numepsbi), numepsh(numepshi),
#ifdef With_AHF
xc(0), yc(0), zc(0), xr(0), yr(0), zr(0), trigger(0), dTT(0), dumpid(0),
#endif
a_lev(a_levi), maxl(maxli), decn(decni), maxrex(maxrexi), drex(drexi),
CheckPoint(0)
// CheckPoint(0)
{
MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
MPI_Comm_rank(MPI_COMM_WORLD, &myrank);
// setup Monitors
{
stringstream a_stream;
a_stream.setf(ios::left);
a_stream << "# Error log information";
ErrorMonitor = new monitor("Error.log", myrank, a_stream.str());
ErrorMonitor->print_message("Warning: we always assume intput parameter in cell center style.");
a_stream.clear();
a_stream.str("");
a_stream << setw(15) << "# time";
char str[50];
for (int pl = 2; pl < maxl + 1; pl++)
for (int pm = -pl; pm < pl + 1; pm++)
{
sprintf(str, "R%02dm%03d", pl, pm);
a_stream << setw(16) << str;
sprintf(str, "I%02dm%03d", pl, pm);
a_stream << setw(16) << str;
}
Psi4Monitor = new monitor("bssn_psi4.dat", myrank, a_stream.str());
a_stream.clear();
a_stream.str("");
a_stream << setw(15) << "# time";
BHMonitor = new monitor("bssn_BH.dat", myrank, a_stream.str());
a_stream.clear();
a_stream.str("");
a_stream << setw(15) << "# time ADMmass ADMPx ADMPy ADMPz ADMSx ADMSy ADMSz";
MAPMonitor = new monitor("bssn_ADMQs.dat", myrank, a_stream.str());
a_stream.clear();
a_stream.str("");
a_stream << setw(15) << "# time Ham Px Py Pz Gx Gy Gz";
ConVMonitor = new monitor("bssn_constraint.dat", myrank, a_stream.str());
}
// setup sphere integration engine
Waveshell = new surface_integral(Symmetry);
trfls = 0;
chitiny = 0;
// read parameter from file
{
char filename[50];
{
map<string, string>::iterator iter = parameters::str_par.find("inputpar");
if (iter != parameters::str_par.end())
{
strcpy(filename, (iter->second).c_str());
}
else
{
cout << "Error inputpar" << endl;
exit(0);
}
}
const int LEN = 256;
char pline[LEN];
string str, sgrp, skey, sval;
int sind;
ifstream inf(filename, ifstream::in);
if (!inf.good() && myrank == 0)
{
if (ErrorMonitor->outfile)
ErrorMonitor->outfile << "Can not open parameter file " << filename
<< " for inputing information of black holes" << endl;
MPI_Abort(MPI_COMM_WORLD, 1);
}
for (int i = 1; inf.good(); i++)
{
inf.getline(pline, LEN);
str = pline;
int status = misc::parse_parts(str, sgrp, skey, sval, sind);
if (status == -1)
{
if (ErrorMonitor->outfile)
ErrorMonitor->outfile << "error reading parameter file " << filename
<< " in line " << i << endl;
MPI_Abort(MPI_COMM_WORLD, 1);
}
else if (status == 0)
continue;
if (sgrp == "BSSN" && skey == "chitiny")
chitiny = atof(sval.c_str());
else if (sgrp == "BSSN" && skey == "time refinement start from level")
trfls = atoi(sval.c_str());
#ifdef With_AHF
else if (sgrp == "AHF" && skey == "AHfindevery")
AHfindevery = atoi(sval.c_str());
else if (sgrp == "AHF" && skey == "AHdumptime")
AHdumptime = atof(sval.c_str());
#endif
}
inf.close();
}
if (myrank == 0)
{
// echo information of lower bound of chi
cout << " chitiny = " << chitiny << endl;
cout << " time refinement start from level #" << trfls << endl;
#ifdef With_AHF
cout << " parameters for AHF:" << endl;
cout << " AHfindevery = " << AHfindevery << endl;
cout << " AHdumptime = " << AHdumptime << endl;
#endif
}
chitiny = chitiny - 1; // because we have subtracted one from chi
strcpy(checkfilename, checkfilenamei);
ngfs = 0;
phio = new var("phio", ngfs++, 1, 1, 1);
trKo = new var("trKo", ngfs++, 1, 1, 1);
gxxo = new var("gxxo", ngfs++, 1, 1, 1);
gxyo = new var("gxyo", ngfs++, -1, -1, 1);
gxzo = new var("gxzo", ngfs++, -1, 1, -1);
gyyo = new var("gyyo", ngfs++, 1, 1, 1);
gyzo = new var("gyzo", ngfs++, 1, -1, -1);
gzzo = new var("gzzo", ngfs++, 1, 1, 1);
Axxo = new var("Axxo", ngfs++, 1, 1, 1);
Axyo = new var("Axyo", ngfs++, -1, -1, 1);
Axzo = new var("Axzo", ngfs++, -1, 1, -1);
Ayyo = new var("Ayyo", ngfs++, 1, 1, 1);
Ayzo = new var("Ayzo", ngfs++, 1, -1, -1);
Azzo = new var("Azzo", ngfs++, 1, 1, 1);
Gmxo = new var("Gmxo", ngfs++, -1, 1, 1);
Gmyo = new var("Gmyo", ngfs++, 1, -1, 1);
Gmzo = new var("Gmzo", ngfs++, 1, 1, -1);
Lapo = new var("Lapo", ngfs++, 1, 1, 1);
Sfxo = new var("Sfxo", ngfs++, -1, 1, 1);
Sfyo = new var("Sfyo", ngfs++, 1, -1, 1);
Sfzo = new var("Sfzo", ngfs++, 1, 1, -1);
dtSfxo = new var("dtSfxo", ngfs++, -1, 1, 1);
dtSfyo = new var("dtSfyo", ngfs++, 1, -1, 1);
dtSfzo = new var("dtSfzo", ngfs++, 1, 1, -1);
phi0 = new var("phi0", ngfs++, 1, 1, 1);
trK0 = new var("trK0", ngfs++, 1, 1, 1);
gxx0 = new var("gxx0", ngfs++, 1, 1, 1);
gxy0 = new var("gxy0", ngfs++, -1, -1, 1);
gxz0 = new var("gxz0", ngfs++, -1, 1, -1);
gyy0 = new var("gyy0", ngfs++, 1, 1, 1);
gyz0 = new var("gyz0", ngfs++, 1, -1, -1);
gzz0 = new var("gzz0", ngfs++, 1, 1, 1);
Axx0 = new var("Axx0", ngfs++, 1, 1, 1);
Axy0 = new var("Axy0", ngfs++, -1, -1, 1);
Axz0 = new var("Axz0", ngfs++, -1, 1, -1);
Ayy0 = new var("Ayy0", ngfs++, 1, 1, 1);
Ayz0 = new var("Ayz0", ngfs++, 1, -1, -1);
Azz0 = new var("Azz0", ngfs++, 1, 1, 1);
Gmx0 = new var("Gmx0", ngfs++, -1, 1, 1);
Gmy0 = new var("Gmy0", ngfs++, 1, -1, 1);
Gmz0 = new var("Gmz0", ngfs++, 1, 1, -1);
Lap0 = new var("Lap0", ngfs++, 1, 1, 1);
Sfx0 = new var("Sfx0", ngfs++, -1, 1, 1);
Sfy0 = new var("Sfy0", ngfs++, 1, -1, 1);
Sfz0 = new var("Sfz0", ngfs++, 1, 1, -1);
dtSfx0 = new var("dtSfx0", ngfs++, -1, 1, 1);
dtSfy0 = new var("dtSfy0", ngfs++, 1, -1, 1);
dtSfz0 = new var("dtSfz0", ngfs++, 1, 1, -1);
phi = new var("phi", ngfs++, 1, 1, 1);
trK = new var("trK", ngfs++, 1, 1, 1);
gxx = new var("gxx", ngfs++, 1, 1, 1);
gxy = new var("gxy", ngfs++, -1, -1, 1);
gxz = new var("gxz", ngfs++, -1, 1, -1);
gyy = new var("gyy", ngfs++, 1, 1, 1);
gyz = new var("gyz", ngfs++, 1, -1, -1);
gzz = new var("gzz", ngfs++, 1, 1, 1);
Axx = new var("Axx", ngfs++, 1, 1, 1);
Axy = new var("Axy", ngfs++, -1, -1, 1);
Axz = new var("Axz", ngfs++, -1, 1, -1);
Ayy = new var("Ayy", ngfs++, 1, 1, 1);
Ayz = new var("Ayz", ngfs++, 1, -1, -1);
Azz = new var("Azz", ngfs++, 1, 1, 1);
Gmx = new var("Gmx", ngfs++, -1, 1, 1);
Gmy = new var("Gmy", ngfs++, 1, -1, 1);
Gmz = new var("Gmz", ngfs++, 1, 1, -1);
Lap = new var("Lap", ngfs++, 1, 1, 1);
Sfx = new var("Sfx", ngfs++, -1, 1, 1);
Sfy = new var("Sfy", ngfs++, 1, -1, 1);
Sfz = new var("Sfz", ngfs++, 1, 1, -1);
dtSfx = new var("dtSfx", ngfs++, -1, 1, 1);
dtSfy = new var("dtSfy", ngfs++, 1, -1, 1);
dtSfz = new var("dtSfz", ngfs++, 1, 1, -1);
phi1 = new var("phi1", ngfs++, 1, 1, 1);
trK1 = new var("trK1", ngfs++, 1, 1, 1);
gxx1 = new var("gxx1", ngfs++, 1, 1, 1);
gxy1 = new var("gxy1", ngfs++, -1, -1, 1);
gxz1 = new var("gxz1", ngfs++, -1, 1, -1);
gyy1 = new var("gyy1", ngfs++, 1, 1, 1);
gyz1 = new var("gyz1", ngfs++, 1, -1, -1);
gzz1 = new var("gzz1", ngfs++, 1, 1, 1);
Axx1 = new var("Axx1", ngfs++, 1, 1, 1);
Axy1 = new var("Axy1", ngfs++, -1, -1, 1);
Axz1 = new var("Axz1", ngfs++, -1, 1, -1);
Ayy1 = new var("Ayy1", ngfs++, 1, 1, 1);
Ayz1 = new var("Ayz1", ngfs++, 1, -1, -1);
Azz1 = new var("Azz1", ngfs++, 1, 1, 1);
Gmx1 = new var("Gmx1", ngfs++, -1, 1, 1);
Gmy1 = new var("Gmy1", ngfs++, 1, -1, 1);
Gmz1 = new var("Gmz1", ngfs++, 1, 1, -1);
Lap1 = new var("Lap1", ngfs++, 1, 1, 1);
Sfx1 = new var("Sfx1", ngfs++, -1, 1, 1);
Sfy1 = new var("Sfy1", ngfs++, 1, -1, 1);
Sfz1 = new var("Sfz1", ngfs++, 1, 1, -1);
dtSfx1 = new var("dtSfx1", ngfs++, -1, 1, 1);
dtSfy1 = new var("dtSfy1", ngfs++, 1, -1, 1);
dtSfz1 = new var("dtSfz1", ngfs++, 1, 1, -1);
phi_rhs = new var("phi_rhs", ngfs++, 1, 1, 1);
trK_rhs = new var("trK_rhs", ngfs++, 1, 1, 1);
gxx_rhs = new var("gxx_rhs", ngfs++, 1, 1, 1);
gxy_rhs = new var("gxy_rhs", ngfs++, -1, -1, 1);
gxz_rhs = new var("gxz_rhs", ngfs++, -1, 1, -1);
gyy_rhs = new var("gyy_rhs", ngfs++, 1, 1, 1);
gyz_rhs = new var("gyz_rhs", ngfs++, 1, -1, -1);
gzz_rhs = new var("gzz_rhs", ngfs++, 1, 1, 1);
Axx_rhs = new var("Axx_rhs", ngfs++, 1, 1, 1);
Axy_rhs = new var("Axy_rhs", ngfs++, -1, -1, 1);
Axz_rhs = new var("Axz_rhs", ngfs++, -1, 1, -1);
Ayy_rhs = new var("Ayy_rhs", ngfs++, 1, 1, 1);
Ayz_rhs = new var("Ayz_rhs", ngfs++, 1, -1, -1);
Azz_rhs = new var("Azz_rhs", ngfs++, 1, 1, 1);
Gmx_rhs = new var("Gmx_rhs", ngfs++, -1, 1, 1);
Gmy_rhs = new var("Gmy_rhs", ngfs++, 1, -1, 1);
Gmz_rhs = new var("Gmz_rhs", ngfs++, 1, 1, -1);
Lap_rhs = new var("Lap_rhs", ngfs++, 1, 1, 1);
Sfx_rhs = new var("Sfx_rhs", ngfs++, -1, 1, 1);
Sfy_rhs = new var("Sfy_rhs", ngfs++, 1, -1, 1);
Sfz_rhs = new var("Sfz_rhs", ngfs++, 1, 1, -1);
dtSfx_rhs = new var("dtSfx_rhs", ngfs++, -1, 1, 1);
dtSfy_rhs = new var("dtSfy_rhs", ngfs++, 1, -1, 1);
dtSfz_rhs = new var("dtSfz_rhs", ngfs++, 1, 1, -1);
rho = new var("rho", ngfs++, 1, 1, 1);
Sx = new var("Sx", ngfs++, -1, 1, 1);
Sy = new var("Sy", ngfs++, 1, -1, 1);
Sz = new var("Sz", ngfs++, 1, 1, -1);
Sxx = new var("Sxx", ngfs++, 1, 1, 1);
Sxy = new var("Sxy", ngfs++, -1, -1, 1);
Sxz = new var("Sxz", ngfs++, -1, 1, -1);
Syy = new var("Syy", ngfs++, 1, 1, 1);
Syz = new var("Syz", ngfs++, 1, -1, -1);
Szz = new var("Szz", ngfs++, 1, 1, 1);
Gamxxx = new var("Gamxxx", ngfs++, -1, 1, 1);
Gamxxy = new var("Gamxxy", ngfs++, 1, -1, 1);
Gamxxz = new var("Gamxxz", ngfs++, 1, 1, -1);
Gamxyy = new var("Gamxyy", ngfs++, -1, 1, 1);
Gamxyz = new var("Gamxyz", ngfs++, -1, -1, -1);
Gamxzz = new var("Gamxzz", ngfs++, -1, 1, 1);
Gamyxx = new var("Gamyxx", ngfs++, 1, -1, 1);
Gamyxy = new var("Gamyxy", ngfs++, -1, 1, 1);
Gamyxz = new var("Gamyxz", ngfs++, -1, -1, -1);
Gamyyy = new var("Gamyyy", ngfs++, 1, -1, 1);
Gamyyz = new var("Gamyyz", ngfs++, 1, 1, -1);
Gamyzz = new var("Gamyzz", ngfs++, 1, -1, 1);
Gamzxx = new var("Gamzxx", ngfs++, 1, 1, -1);
Gamzxy = new var("Gamzxy", ngfs++, -1, -1, -1);
Gamzxz = new var("Gamzxz", ngfs++, -1, 1, 1);
Gamzyy = new var("Gamzyy", ngfs++, 1, 1, -1);
Gamzyz = new var("Gamzyz", ngfs++, 1, -1, 1);
Gamzzz = new var("Gamzzz", ngfs++, 1, 1, -1);
Rxx = new var("Rxx", ngfs++, 1, 1, 1);
Rxy = new var("Rxy", ngfs++, -1, -1, 1);
Rxz = new var("Rxz", ngfs++, -1, 1, -1);
Ryy = new var("Ryy", ngfs++, 1, 1, 1);
Ryz = new var("Ryz", ngfs++, 1, -1, -1);
Rzz = new var("Rzz", ngfs++, 1, 1, 1);
// refer to PRD, 77, 024027 (2008)
Rpsi4 = new var("Rpsi4", ngfs++, 1, 1, 1);
Ipsi4 = new var("Ipsi4", ngfs++, -1, -1, -1);
t1Rpsi4 = new var("t1Rpsi4", ngfs++, 1, 1, 1);
t1Ipsi4 = new var("t1Ipsi4", ngfs++, -1, -1, -1);
t2Rpsi4 = new var("t2Rpsi4", ngfs++, 1, 1, 1);
t2Ipsi4 = new var("t2Ipsi4", ngfs++, -1, -1, -1);
// constraint violation monitor variables
Cons_Ham = new var("Cons_Ham", ngfs++, 1, 1, 1);
Cons_Px = new var("Cons_Px", ngfs++, -1, 1, 1);
Cons_Py = new var("Cons_Py", ngfs++, 1, -1, 1);
Cons_Pz = new var("Cons_Pz", ngfs++, 1, 1, -1);
Cons_Gx = new var("Cons_Gx", ngfs++, -1, 1, 1);
Cons_Gy = new var("Cons_Gy", ngfs++, 1, -1, 1);
Cons_Gz = new var("Cons_Gz", ngfs++, 1, 1, -1);
#ifdef Point_Psi4
phix = new var("phix", ngfs++, -1, 1, 1);
phiy = new var("phiy", ngfs++, 1, -1, 1);
phiz = new var("phiz", ngfs++, 1, 1, -1);
trKx = new var("trKx", ngfs++, -1, 1, 1);
trKy = new var("trKy", ngfs++, 1, -1, 1);
trKz = new var("trKz", ngfs++, 1, 1, -1);
Axxx = new var("Axxx", ngfs++, -1, 1, 1);
Axxy = new var("Axxy", ngfs++, 1, -1, 1);
Axxz = new var("Axxz", ngfs++, 1, 1, -1);
Axyx = new var("Axyx", ngfs++, 1, -1, 1);
Axyy = new var("Axyy", ngfs++, -1, 1, 1);
Axyz = new var("Axyz", ngfs++, -1, -1, -1);
Axzx = new var("Axzx", ngfs++, 1, 1, -1);
Axzy = new var("Axzy", ngfs++, -1, -1, -1);
Axzz = new var("Axzz", ngfs++, -1, 1, 1);
Ayyx = new var("Ayyx", ngfs++, -1, 1, 1);
Ayyy = new var("Ayyy", ngfs++, 1, -1, 1);
Ayyz = new var("Ayyz", ngfs++, 1, 1, -1);
Ayzx = new var("Ayzx", ngfs++, -1, -1, -1);
Ayzy = new var("Ayzy", ngfs++, 1, 1, -1);
Ayzz = new var("Ayzz", ngfs++, 1, -1, 1);
Azzx = new var("Azzx", ngfs++, -1, 1, 1);
Azzy = new var("Azzy", ngfs++, 1, -1, 1);
Azzz = new var("Azzz", ngfs++, 1, 1, -1);
#endif
// specific properspeed for 1+log slice
{
const double vl = sqrt(2);
trKo->setpropspeed(vl);
trK0->setpropspeed(vl);
trK->setpropspeed(vl);
trK1->setpropspeed(vl);
trK_rhs->setpropspeed(vl);
phio->setpropspeed(vl);
phi0->setpropspeed(vl);
phi->setpropspeed(vl);
phi1->setpropspeed(vl);
phi_rhs->setpropspeed(vl);
Lapo->setpropspeed(vl);
Lap0->setpropspeed(vl);
Lap->setpropspeed(vl);
Lap1->setpropspeed(vl);
Lap_rhs->setpropspeed(vl);
}
OldStateList = new MyList<var>(phio);
OldStateList->insert(trKo);
OldStateList->insert(gxxo);
OldStateList->insert(gxyo);
OldStateList->insert(gxzo);
OldStateList->insert(gyyo);
OldStateList->insert(gyzo);
OldStateList->insert(gzzo);
OldStateList->insert(Axxo);
OldStateList->insert(Axyo);
OldStateList->insert(Axzo);
OldStateList->insert(Ayyo);
OldStateList->insert(Ayzo);
OldStateList->insert(Azzo);
OldStateList->insert(Gmxo);
OldStateList->insert(Gmyo);
OldStateList->insert(Gmzo);
OldStateList->insert(Lapo);
OldStateList->insert(Sfxo);
OldStateList->insert(Sfyo);
OldStateList->insert(Sfzo);
OldStateList->insert(dtSfxo);
OldStateList->insert(dtSfyo);
OldStateList->insert(dtSfzo);
StateList = new MyList<var>(phi0);
StateList->insert(trK0);
StateList->insert(gxx0);
StateList->insert(gxy0);
StateList->insert(gxz0);
StateList->insert(gyy0);
StateList->insert(gyz0);
StateList->insert(gzz0);
StateList->insert(Axx0);
StateList->insert(Axy0);
StateList->insert(Axz0);
StateList->insert(Ayy0);
StateList->insert(Ayz0);
StateList->insert(Azz0);
StateList->insert(Gmx0);
StateList->insert(Gmy0);
StateList->insert(Gmz0);
StateList->insert(Lap0);
StateList->insert(Sfx0);
StateList->insert(Sfy0);
StateList->insert(Sfz0);
StateList->insert(dtSfx0);
StateList->insert(dtSfy0);
StateList->insert(dtSfz0);
RHSList = new MyList<var>(phi_rhs);
RHSList->insert(trK_rhs);
RHSList->insert(gxx_rhs);
RHSList->insert(gxy_rhs);
RHSList->insert(gxz_rhs);
RHSList->insert(gyy_rhs);
RHSList->insert(gyz_rhs);
RHSList->insert(gzz_rhs);
RHSList->insert(Axx_rhs);
RHSList->insert(Axy_rhs);
RHSList->insert(Axz_rhs);
RHSList->insert(Ayy_rhs);
RHSList->insert(Ayz_rhs);
RHSList->insert(Azz_rhs);
RHSList->insert(Gmx_rhs);
RHSList->insert(Gmy_rhs);
RHSList->insert(Gmz_rhs);
RHSList->insert(Lap_rhs);
RHSList->insert(Sfx_rhs);
RHSList->insert(Sfy_rhs);
RHSList->insert(Sfz_rhs);
RHSList->insert(dtSfx_rhs);
RHSList->insert(dtSfy_rhs);
RHSList->insert(dtSfz_rhs);
SynchList_pre = new MyList<var>(phi);
SynchList_pre->insert(trK);
SynchList_pre->insert(gxx);
SynchList_pre->insert(gxy);
SynchList_pre->insert(gxz);
SynchList_pre->insert(gyy);
SynchList_pre->insert(gyz);
SynchList_pre->insert(gzz);
SynchList_pre->insert(Axx);
SynchList_pre->insert(Axy);
SynchList_pre->insert(Axz);
SynchList_pre->insert(Ayy);
SynchList_pre->insert(Ayz);
SynchList_pre->insert(Azz);
SynchList_pre->insert(Gmx);
SynchList_pre->insert(Gmy);
SynchList_pre->insert(Gmz);
SynchList_pre->insert(Lap);
SynchList_pre->insert(Sfx);
SynchList_pre->insert(Sfy);
SynchList_pre->insert(Sfz);
SynchList_pre->insert(dtSfx);
SynchList_pre->insert(dtSfy);
SynchList_pre->insert(dtSfz);
SynchList_cor = new MyList<var>(phi1);
SynchList_cor->insert(trK1);
SynchList_cor->insert(gxx1);
SynchList_cor->insert(gxy1);
SynchList_cor->insert(gxz1);
SynchList_cor->insert(gyy1);
SynchList_cor->insert(gyz1);
SynchList_cor->insert(gzz1);
SynchList_cor->insert(Axx1);
SynchList_cor->insert(Axy1);
SynchList_cor->insert(Axz1);
SynchList_cor->insert(Ayy1);
SynchList_cor->insert(Ayz1);
SynchList_cor->insert(Azz1);
SynchList_cor->insert(Gmx1);
SynchList_cor->insert(Gmy1);
SynchList_cor->insert(Gmz1);
SynchList_cor->insert(Lap1);
SynchList_cor->insert(Sfx1);
SynchList_cor->insert(Sfy1);
SynchList_cor->insert(Sfz1);
SynchList_cor->insert(dtSfx1);
SynchList_cor->insert(dtSfy1);
SynchList_cor->insert(dtSfz1);
DumpList = new MyList<var>(phi0);
DumpList->insert(trK0);
DumpList->insert(gxx0);
DumpList->insert(gxy0);
DumpList->insert(gxz0);
DumpList->insert(gyy0);
DumpList->insert(gyz0);
DumpList->insert(gzz0);
// DumpList->insert(Axx0);
// DumpList->insert(Axy0);
// DumpList->insert(Axz0);
// DumpList->insert(Ayy0);
// DumpList->insert(Ayz0);
// DumpList->insert(Azz0);
// DumpList->insert(Gmx0);
// DumpList->insert(Gmy0);
// DumpList->insert(Gmz0);
DumpList->insert(Lap0);
// DumpList->insert(Sfx0);
// DumpList->insert(Sfy0);
// DumpList->insert(Sfz0);
// DumpList->insert(dtSfx0);
// DumpList->insert(dtSfy0);
// DumpList->insert(dtSfz0);
// DumpList->insert(Rpsi4);
// DumpList->insert(Ipsi4);
DumpList->insert(Cons_Ham);
DumpList->insert(Cons_Px);
DumpList->insert(Cons_Py);
DumpList->insert(Cons_Pz);
// DumpList->insert(Cons_Gx);
// DumpList->insert(Cons_Gy);
// DumpList->insert(Cons_Gz);
ConstraintList = new MyList<var>(Cons_Ham);
ConstraintList->insert(Cons_Px);
ConstraintList->insert(Cons_Py);
ConstraintList->insert(Cons_Pz);
ConstraintList->insert(Cons_Gx);
ConstraintList->insert(Cons_Gy);
ConstraintList->insert(Cons_Gz);
#ifdef With_AHF
// setup kinds of var list
// List for AparentHorizonFinderDirect
// special attension is payed to symmetry type
// gij gij,x gij,y gij,z
AHList = new MyList<var>(gxx0);
AHList->insert(Gamxxx);
AHList->insert(Gamyxx);
AHList->insert(Gamzxx);
AHList->insert(gxy0);
AHList->insert(Gamxxy);
AHList->insert(Gamyxy);
AHList->insert(Gamzxy);
AHList->insert(gxz0);
AHList->insert(Gamxxz);
AHList->insert(Gamyxz);
AHList->insert(Gamzxz);
AHList->insert(gyy0);
AHList->insert(Gamxyy);
AHList->insert(Gamyyy);
AHList->insert(Gamzyy);
AHList->insert(gyz0);
AHList->insert(Gamxyz);
AHList->insert(Gamyyz);
AHList->insert(Gamzyz);
AHList->insert(gzz0);
AHList->insert(Gamxzz);
AHList->insert(Gamyzz);
AHList->insert(Gamzzz);
// phi phi,x phi,y phi,z
AHList->insert(phi0);
AHList->insert(dtSfx_rhs);
AHList->insert(dtSfy_rhs);
AHList->insert(dtSfz_rhs);
// Aij
AHList->insert(Axx0);
AHList->insert(Axy0);
AHList->insert(Axz0);
AHList->insert(Ayy0);
AHList->insert(Ayz0);
AHList->insert(Azz0);
// trK
AHList->insert(trK0);
// gij,x gij,y gij,z
AHDList = new MyList<var>(Gamxxx);
AHDList->insert(Gamyxx);
AHDList->insert(Gamzxx);
AHDList->insert(Gamxxy);
AHDList->insert(Gamyxy);
AHDList->insert(Gamzxy);
AHDList->insert(Gamxxz);
AHDList->insert(Gamyxz);
AHDList->insert(Gamzxz);
AHDList->insert(Gamxyy);
AHDList->insert(Gamyyy);
AHDList->insert(Gamzyy);
AHDList->insert(Gamxyz);
AHDList->insert(Gamyyz);
AHDList->insert(Gamzyz);
AHDList->insert(Gamxzz);
AHDList->insert(Gamyzz);
AHDList->insert(Gamzzz);
// phi,x phi,y phi,z
AHDList->insert(dtSfx_rhs);
AHDList->insert(dtSfy_rhs);
AHDList->insert(dtSfz_rhs);
GaugeList = new MyList<var>(Lap0);
GaugeList->insert(Sfx0);
GaugeList->insert(Sfy0);
GaugeList->insert(Sfz0);
#endif
// Note: the first checkpoint-class variable is `bool` while the local variable is `int`;
// an explicit conversion may be required in some contexts.
// bool checkrun00 = checkrun;
// Note: the second checkpoint-class variable is `const char*` while the local variable is `char*`;
// an explicit conversion may be required.
// const char* checkfilename00 = checkfilename;
CheckPoint = new checkpoint(checkrun, checkfilename, myrank);
if (myrank==0) {
cout << " BSSN class successfully created " << endl;
}
}
//================================================================================================
//================================================================================================
// This member function initializes the class
//================================================================================================
void bssn_class::Initialize()
{
if (myrank == 0)
cout << " you have setted " << ngfs << " grid functions." << endl;
CheckPoint->addvariablelist(StateList);
CheckPoint->addvariablelist(OldStateList);
char pname[50];
{
map<string, string>::iterator iter = parameters::str_par.find("inputpar");
if (iter != parameters::str_par.end())
{
strcpy(pname, (iter->second).c_str());
}
else
{
cout << "Error inputpar" << endl;
exit(0);
}
}
GH = new cgh(0, ngfs, Symmetry, pname, checkrun, ErrorMonitor);
if (checkrun)
CheckPoint->readcheck_cgh(PhysTime, GH, myrank, nprocs, Symmetry);
else
GH->compose_cgh(nprocs);
#ifdef WithShell
SH = new ShellPatch(0, ngfs, pname, Symmetry, myrank, ErrorMonitor);
SH->matchcheck(GH->PatL[0]);
SH->compose_sh(nprocs);
// SH->compose_shr(nprocs); //sh is faster than shr
SH->setupcordtrans();
SH->Dump_xyz(0, 0, 1);
SH->setupintintstuff(nprocs, GH->PatL[0], Symmetry);
if (checkrun)
CheckPoint->readcheck_sh(SH, myrank);
#else
SH = 0;
#endif
double h = GH->PatL[0]->data->blb->data->getdX(0);
for (int i = 1; i < dim; i++)
h = Mymin(h, GH->PatL[0]->data->blb->data->getdX(i));
dT = Courant * h;
if (checkrun)
{
CheckPoint->read_Black_Hole_position(BH_num_input, BH_num, Porg0, Pmom, Spin, Mass, Porgbr, Porg, Porg1, Porg_rhs);
setpbh(BH_num, Porg0, Mass, BH_num_input);
}
else
{
PhysTime = StartTime;
Setup_Black_Hole_position();
}
// Initialize sync caches (per-level, for predictor and corrector)
sync_cache_pre = new Parallel::SyncCache[GH->levels];
sync_cache_cor = new Parallel::SyncCache[GH->levels];
sync_cache_rp_coarse = new Parallel::SyncCache[GH->levels];
sync_cache_rp_fine = new Parallel::SyncCache[GH->levels];
}
//================================================================================================
//================================================================================================
// This member function is the destructor; it releases allocated variables
//================================================================================================
bssn_class::~bssn_class()
{
#ifdef With_AHF
AHList->clearList();
AHDList->clearList();
GaugeList->clearList();
if (lastahdumpid)
delete[] lastahdumpid;
if (findeveryl)
delete[] findeveryl;
if (xc)
{
delete[] xc;
delete[] yc;
delete[] zc;
delete[] xr;
delete[] yr;
delete[] zr;
delete[] trigger;
delete[] dumpid;
delete[] dTT;
}
AHFinderDirect::AHFinderDirect_cleanup();
#endif
StateList->clearList();
RHSList->clearList();
OldStateList->clearList();
SynchList_pre->clearList();
SynchList_cor->clearList();
DumpList->clearList();
ConstraintList->clearList();
delete phio;
delete trKo;
delete gxxo;
delete gxyo;
delete gxzo;
delete gyyo;
delete gyzo;
delete gzzo;
delete Axxo;
delete Axyo;
delete Axzo;
delete Ayyo;
delete Ayzo;
delete Azzo;
delete Gmxo;
delete Gmyo;
delete Gmzo;
delete Lapo;
delete Sfxo;
delete Sfyo;
delete Sfzo;
delete dtSfxo;
delete dtSfyo;
delete dtSfzo;
delete phi0;
delete trK0;
delete gxx0;
delete gxy0;
delete gxz0;
delete gyy0;
delete gyz0;
delete gzz0;
delete Axx0;
delete Axy0;
delete Axz0;
delete Ayy0;
delete Ayz0;
delete Azz0;
delete Gmx0;
delete Gmy0;
delete Gmz0;
delete Lap0;
delete Sfx0;
delete Sfy0;
delete Sfz0;
delete dtSfx0;
delete dtSfy0;
delete dtSfz0;
delete phi;
delete trK;
delete gxx;
delete gxy;
delete gxz;
delete gyy;
delete gyz;
delete gzz;
delete Axx;
delete Axy;
delete Axz;
delete Ayy;
delete Ayz;
delete Azz;
delete Gmx;
delete Gmy;
delete Gmz;
delete Lap;
delete Sfx;
delete Sfy;
delete Sfz;
delete dtSfx;
delete dtSfy;
delete dtSfz;
delete phi1;
delete trK1;
delete gxx1;
delete gxy1;
delete gxz1;
delete gyy1;
delete gyz1;
delete gzz1;
delete Axx1;
delete Axy1;
delete Axz1;
delete Ayy1;
delete Ayz1;
delete Azz1;
delete Gmx1;
delete Gmy1;
delete Gmz1;
delete Lap1;
delete Sfx1;
delete Sfy1;
delete Sfz1;
delete dtSfx1;
delete dtSfy1;
delete dtSfz1;
delete phi_rhs;
delete trK_rhs;
delete gxx_rhs;
delete gxy_rhs;
delete gxz_rhs;
delete gyy_rhs;
delete gyz_rhs;
delete gzz_rhs;
delete Axx_rhs;
delete Axy_rhs;
delete Axz_rhs;
delete Ayy_rhs;
delete Ayz_rhs;
delete Azz_rhs;
delete Gmx_rhs;
delete Gmy_rhs;
delete Gmz_rhs;
delete Lap_rhs;
delete Sfx_rhs;
delete Sfy_rhs;
delete Sfz_rhs;
delete dtSfx_rhs;
delete dtSfy_rhs;
delete dtSfz_rhs;
delete rho;
delete Sx;
delete Sy;
delete Sz;
delete Sxx;
delete Sxy;
delete Sxz;
delete Syy;
delete Syz;
delete Szz;
delete Gamxxx;
delete Gamxxy;
delete Gamxxz;
delete Gamxyy;
delete Gamxyz;
delete Gamxzz;
delete Gamyxx;
delete Gamyxy;
delete Gamyxz;
delete Gamyyy;
delete Gamyyz;
delete Gamyzz;
delete Gamzxx;
delete Gamzxy;
delete Gamzxz;
delete Gamzyy;
delete Gamzyz;
delete Gamzzz;
delete Rxx;
delete Rxy;
delete Rxz;
delete Ryy;
delete Ryz;
delete Rzz;
delete Rpsi4;
delete Ipsi4;
delete t1Rpsi4;
delete t1Ipsi4;
delete t2Rpsi4;
delete t2Ipsi4;
delete Cons_Ham;
delete Cons_Px;
delete Cons_Py;
delete Cons_Pz;
delete Cons_Gx;
delete Cons_Gy;
delete Cons_Gz;
#ifdef Point_Psi4
delete phix;
delete phiy;
delete phiz;
delete trKx;
delete trKy;
delete trKz;
delete Axxx;
delete Axxy;
delete Axxz;
delete Axyx;
delete Axyy;
delete Axyz;
delete Axzx;
delete Axzy;
delete Axzz;
delete Ayyx;
delete Ayyy;
delete Ayyz;
delete Ayzx;
delete Ayzy;
delete Ayzz;
delete Azzx;
delete Azzy;
delete Azzz;
#endif
// Destroy sync caches before GH
if (sync_cache_pre)
{
for (int i = 0; i < GH->levels; i++)
sync_cache_pre[i].destroy();
delete[] sync_cache_pre;
}
if (sync_cache_cor)
{
for (int i = 0; i < GH->levels; i++)
sync_cache_cor[i].destroy();
delete[] sync_cache_cor;
}
if (sync_cache_rp_coarse)
{
for (int i = 0; i < GH->levels; i++)
sync_cache_rp_coarse[i].destroy();
delete[] sync_cache_rp_coarse;
}
if (sync_cache_rp_fine)
{
for (int i = 0; i < GH->levels; i++)
sync_cache_rp_fine[i].destroy();
delete[] sync_cache_rp_fine;
}
delete GH;
#ifdef WithShell
delete SH;
#endif
for (int i = 0; i < BH_num; i++)
{
delete[] Porg0[i];
delete[] Porgbr[i];
delete[] Porg[i];
delete[] Porg1[i];
delete[] Porg_rhs[i];
}
delete[] Porg0;
delete[] Porgbr;
delete[] Porg;
delete[] Porg1;
delete[] Porg_rhs;
delete[] Mass;
delete[] Spin;
delete[] Pmom;
delete ErrorMonitor;
delete Psi4Monitor;
delete BHMonitor;
delete MAPMonitor;
delete ConVMonitor;
delete Waveshell;
delete CheckPoint;
}
//================================================================================================
//================================================================================================
// This member function computes initial data using Lousto's analytic method
//================================================================================================
void bssn_class::Setup_Initial_Data_Lousto()
{
if (!checkrun)
{
if (myrank == 0)
{
cout << endl;
cout << " Setup initial data with Lousto's analytical formula. " << endl;
cout << endl;
}
char filename[50];
{
map<string, string>::iterator iter = parameters::str_par.find("inputpar");
if (iter != parameters::str_par.end())
{
strcpy(filename, (iter->second).c_str());
}
else
{
cout << "Error inputpar" << endl;
exit(0);
}
}
int BH_NM;
double *Porg_here, *Pmom_here, *Spin_here, *Mass_here;
// read parameter from file
{
const int LEN = 256;
char pline[LEN];
string str, sgrp, skey, sval;
int sind;
ifstream inf(filename, ifstream::in);
if (!inf.good() && myrank == 0)
{
if (ErrorMonitor->outfile)
ErrorMonitor->outfile << "Can not open parameter file " << filename
<< " for inputing information of black holes" << endl;
MPI_Abort(MPI_COMM_WORLD, 1);
}
for (int i = 1; inf.good(); i++)
{
inf.getline(pline, LEN);
str = pline;
int status = misc::parse_parts(str, sgrp, skey, sval, sind);
if (status == -1)
{
if (ErrorMonitor->outfile)
ErrorMonitor->outfile << "error reading parameter file " << filename << " in line " << i << endl;
MPI_Abort(MPI_COMM_WORLD, 1);
}
else if (status == 0)
continue;
if (sgrp == "BSSN" && skey == "BH_num")
{
BH_NM = atoi(sval.c_str());
break;
}
}
inf.close();
}
Porg_here = new double[3 * BH_NM];
Pmom_here = new double[3 * BH_NM];
Spin_here = new double[3 * BH_NM];
Mass_here = new double[BH_NM];
// read parameter from file
{
const int LEN = 256;
char pline[LEN];
string str, sgrp, skey, sval;
int sind;
ifstream inf(filename, ifstream::in);
if (!inf.good() && myrank == 0)
{
if (ErrorMonitor->outfile)
ErrorMonitor->outfile << "Can not open parameter file " << filename
<< " for inputing information of black holes" << endl;
MPI_Abort(MPI_COMM_WORLD, 1);
}
for (int i = 1; inf.good(); i++)
{
inf.getline(pline, LEN);
str = pline;
int status = misc::parse_parts(str, sgrp, skey, sval, sind);
if (status == -1)
{
if (ErrorMonitor->outfile)
ErrorMonitor->outfile << "error reading parameter file " << filename << " in line " << i << endl;
MPI_Abort(MPI_COMM_WORLD, 1);
}
else if (status == 0)
continue;
if (sgrp == "BSSN" && sind < BH_NM)
{
if (skey == "Mass")
Mass_here[sind] = atof(sval.c_str());
else if (skey == "Porgx")
Porg_here[sind * 3] = atof(sval.c_str());
else if (skey == "Porgy")
Porg_here[sind * 3 + 1] = atof(sval.c_str());
else if (skey == "Porgz")
Porg_here[sind * 3 + 2] = atof(sval.c_str());
else if (skey == "Spinx")
Spin_here[sind * 3] = atof(sval.c_str());
else if (skey == "Spiny")
Spin_here[sind * 3 + 1] = atof(sval.c_str());
else if (skey == "Spinz")
Spin_here[sind * 3 + 2] = atof(sval.c_str());
else if (skey == "Pmomx")
Pmom_here[sind * 3] = atof(sval.c_str());
else if (skey == "Pmomy")
Pmom_here[sind * 3 + 1] = atof(sval.c_str());
else if (skey == "Pmomz")
Pmom_here[sind * 3 + 2] = atof(sval.c_str());
}
}
inf.close();
}
// set initial data
for (int lev = 0; lev < GH->levels; lev++)
{
MyList<Patch> *Pp = GH->PatL[lev];
while (Pp)
{
MyList<Block> *BL = Pp->data->blb;
while (BL)
{
Block *cg = BL->data;
if (myrank == cg->rank)
{
// Use Lousto's analytic formulas to compute initial data
f_get_lousto_nbhs(cg->shape, cg->X[0], cg->X[1], cg->X[2],
cg->fgfs[phi0->sgfn], cg->fgfs[trK0->sgfn],
cg->fgfs[gxx0->sgfn], cg->fgfs[gxy0->sgfn], cg->fgfs[gxz0->sgfn],
cg->fgfs[gyy0->sgfn], cg->fgfs[gyz0->sgfn], cg->fgfs[gzz0->sgfn],
cg->fgfs[Axx0->sgfn], cg->fgfs[Axy0->sgfn], cg->fgfs[Axz0->sgfn],
cg->fgfs[Ayy0->sgfn], cg->fgfs[Ayz0->sgfn], cg->fgfs[Azz0->sgfn],
cg->fgfs[Gmx0->sgfn], cg->fgfs[Gmy0->sgfn], cg->fgfs[Gmz0->sgfn],
cg->fgfs[Lap0->sgfn],
cg->fgfs[Sfx0->sgfn], cg->fgfs[Sfy0->sgfn], cg->fgfs[Sfz0->sgfn],
cg->fgfs[dtSfx0->sgfn], cg->fgfs[dtSfy0->sgfn], cg->fgfs[dtSfz0->sgfn],
Mass_here, Porg_here, Pmom_here, Spin_here, BH_NM);
}
if (BL == Pp->data->ble)
break;
BL = BL->next;
}
Pp = Pp->next;
}
}
// dump read_in initial data
for (int lev = 0; lev < GH->levels; lev++)
Parallel::Dump_Data(GH->PatL[lev], StateList, 0, PhysTime, dT);
#ifdef WithShell
// ShellPatch part
MyList<ss_patch> *Pp = SH->PatL;
while (Pp)
{
MyList<Block> *BL = Pp->data->blb;
while (BL)
{
Block *cg = BL->data;
if (myrank == cg->rank)
{
f_get_initial_nbhs_sh(cg->shape,
cg->fgfs[Pp->data->fngfs + ShellPatch::gx],
cg->fgfs[Pp->data->fngfs + ShellPatch::gy],
cg->fgfs[Pp->data->fngfs + ShellPatch::gz],
cg->fgfs[phi0->sgfn], cg->fgfs[trK0->sgfn],
cg->fgfs[gxx0->sgfn], cg->fgfs[gxy0->sgfn], cg->fgfs[gxz0->sgfn],
cg->fgfs[gyy0->sgfn], cg->fgfs[gyz0->sgfn], cg->fgfs[gzz0->sgfn],
cg->fgfs[Axx0->sgfn], cg->fgfs[Axy0->sgfn], cg->fgfs[Axz0->sgfn],
cg->fgfs[Ayy0->sgfn], cg->fgfs[Ayz0->sgfn], cg->fgfs[Azz0->sgfn],
cg->fgfs[Gmx0->sgfn], cg->fgfs[Gmy0->sgfn], cg->fgfs[Gmz0->sgfn],
cg->fgfs[Lap0->sgfn],
cg->fgfs[Sfx0->sgfn], cg->fgfs[Sfy0->sgfn], cg->fgfs[Sfz0->sgfn],
cg->fgfs[dtSfx0->sgfn], cg->fgfs[dtSfy0->sgfn], cg->fgfs[dtSfz0->sgfn],
Mass_here, Porg_here, Pmom_here, Spin_here, BH_NM);
}
if (BL == Pp->data->ble)
break;
BL = BL->next;
}
Pp = Pp->next;
}
// dump read_in initial data
SH->Dump_Data(StateList, 0, PhysTime, dT);
#endif
delete[] Porg_here;
delete[] Mass_here;
delete[] Pmom_here;
delete[] Spin_here;
// SH->Synch(GH->PatL[0],StateList,Symmetry);
// exit(0);
}
}
//================================================================================================
//================================================================================================
// This member function computes initial data using Cao's analytic formulas
//================================================================================================
void bssn_class::Setup_Initial_Data_Cao()
{
if (!checkrun)
{
if (myrank == 0)
{
cout << endl;
cout << " Setup initial data with Cao's analytical formula. " << endl;
cout << endl;
}
char filename[50];
{
map<string, string>::iterator iter = parameters::str_par.find("inputpar");
if (iter != parameters::str_par.end())
{
strcpy(filename, (iter->second).c_str());
}
else
{
cout << "Error inputpar" << endl;
exit(0);
}
}
int BH_NM;
double *Porg_here, *Pmom_here, *Spin_here, *Mass_here;
// read parameter from file
{
const int LEN = 256;
char pline[LEN];
string str, sgrp, skey, sval;
int sind;
ifstream inf(filename, ifstream::in);
if (!inf.good() && myrank == 0)
{
if (ErrorMonitor->outfile)
ErrorMonitor->outfile << "Can not open parameter file " << filename
<< " for inputing information of black holes" << endl;
MPI_Abort(MPI_COMM_WORLD, 1);
}
for (int i = 1; inf.good(); i++)
{
inf.getline(pline, LEN);
str = pline;
int status = misc::parse_parts(str, sgrp, skey, sval, sind);
if (status == -1)
{
if (ErrorMonitor->outfile)
ErrorMonitor->outfile << "error reading parameter file " << filename << " in line " << i << endl;
MPI_Abort(MPI_COMM_WORLD, 1);
}
else if (status == 0)
continue;
if (sgrp == "BSSN" && skey == "BH_num")
{
BH_NM = atoi(sval.c_str());
break;
}
}
inf.close();
}
Porg_here = new double[3 * BH_NM];
Pmom_here = new double[3 * BH_NM];
Spin_here = new double[3 * BH_NM];
Mass_here = new double[BH_NM];
// read parameter from file
{
const int LEN = 256;
char pline[LEN];
string str, sgrp, skey, sval;
int sind;
ifstream inf(filename, ifstream::in);
if (!inf.good() && myrank == 0)
{
if (ErrorMonitor->outfile)
ErrorMonitor->outfile << "Can not open parameter file " << filename
<< " for inputing information of black holes" << endl;
MPI_Abort(MPI_COMM_WORLD, 1);
}
for (int i = 1; inf.good(); i++)
{
inf.getline(pline, LEN);
str = pline;
int status = misc::parse_parts(str, sgrp, skey, sval, sind);
if (status == -1)
{
if (ErrorMonitor->outfile)
ErrorMonitor->outfile << "error reading parameter file " << filename << " in line " << i << endl;
MPI_Abort(MPI_COMM_WORLD, 1);
}
else if (status == 0)
continue;
if (sgrp == "BSSN" && sind < BH_NM)
{
if (skey == "Mass")
Mass_here[sind] = atof(sval.c_str());
else if (skey == "Porgx")
Porg_here[sind * 3] = atof(sval.c_str());
else if (skey == "Porgy")
Porg_here[sind * 3 + 1] = atof(sval.c_str());
else if (skey == "Porgz")
Porg_here[sind * 3 + 2] = atof(sval.c_str());
else if (skey == "Spinx")
Spin_here[sind * 3] = atof(sval.c_str());
else if (skey == "Spiny")
Spin_here[sind * 3 + 1] = atof(sval.c_str());
else if (skey == "Spinz")
Spin_here[sind * 3 + 2] = atof(sval.c_str());
else if (skey == "Pmomx")
Pmom_here[sind * 3] = atof(sval.c_str());
else if (skey == "Pmomy")
Pmom_here[sind * 3 + 1] = atof(sval.c_str());
else if (skey == "Pmomz")
Pmom_here[sind * 3 + 2] = atof(sval.c_str());
}
}
inf.close();
}
// set initial data
for (int lev = 0; lev < GH->levels; lev++)
{
MyList<Patch> *Pp = GH->PatL[lev];
while (Pp)
{
MyList<Block> *BL = Pp->data->blb;
while (BL)
{
Block *cg = BL->data;
if (myrank == cg->rank)
{
// Use Cao's analytic formulas to compute initial data
f_get_initial_nbhs(cg->shape, cg->X[0], cg->X[1], cg->X[2],
cg->fgfs[phi0->sgfn], cg->fgfs[trK0->sgfn],
cg->fgfs[gxx0->sgfn], cg->fgfs[gxy0->sgfn], cg->fgfs[gxz0->sgfn],
cg->fgfs[gyy0->sgfn], cg->fgfs[gyz0->sgfn], cg->fgfs[gzz0->sgfn],
cg->fgfs[Axx0->sgfn], cg->fgfs[Axy0->sgfn], cg->fgfs[Axz0->sgfn],
cg->fgfs[Ayy0->sgfn], cg->fgfs[Ayz0->sgfn], cg->fgfs[Azz0->sgfn],
cg->fgfs[Gmx0->sgfn], cg->fgfs[Gmy0->sgfn], cg->fgfs[Gmz0->sgfn],
cg->fgfs[Lap0->sgfn],
cg->fgfs[Sfx0->sgfn], cg->fgfs[Sfy0->sgfn], cg->fgfs[Sfz0->sgfn],
cg->fgfs[dtSfx0->sgfn], cg->fgfs[dtSfy0->sgfn], cg->fgfs[dtSfz0->sgfn],
Mass_here, Porg_here, Pmom_here, Spin_here, BH_NM);
}
if (BL == Pp->data->ble)
break;
BL = BL->next;
}
Pp = Pp->next;
}
}
// dump read_in initial data
for (int lev = 0; lev < GH->levels; lev++)
Parallel::Dump_Data(GH->PatL[lev], StateList, 0, PhysTime, dT);
#ifdef WithShell
// ShellPatch part
MyList<ss_patch> *Pp = SH->PatL;
while (Pp)
{
MyList<Block> *BL = Pp->data->blb;
while (BL)
{
Block *cg = BL->data;
if (myrank == cg->rank)
{
f_get_initial_nbhs_sh(cg->shape,
cg->fgfs[Pp->data->fngfs + ShellPatch::gx],
cg->fgfs[Pp->data->fngfs + ShellPatch::gy],
cg->fgfs[Pp->data->fngfs + ShellPatch::gz],
cg->fgfs[phi0->sgfn], cg->fgfs[trK0->sgfn],
cg->fgfs[gxx0->sgfn], cg->fgfs[gxy0->sgfn], cg->fgfs[gxz0->sgfn],
cg->fgfs[gyy0->sgfn], cg->fgfs[gyz0->sgfn], cg->fgfs[gzz0->sgfn],
cg->fgfs[Axx0->sgfn], cg->fgfs[Axy0->sgfn], cg->fgfs[Axz0->sgfn],
cg->fgfs[Ayy0->sgfn], cg->fgfs[Ayz0->sgfn], cg->fgfs[Azz0->sgfn],
cg->fgfs[Gmx0->sgfn], cg->fgfs[Gmy0->sgfn], cg->fgfs[Gmz0->sgfn],
cg->fgfs[Lap0->sgfn],
cg->fgfs[Sfx0->sgfn], cg->fgfs[Sfy0->sgfn], cg->fgfs[Sfz0->sgfn],
cg->fgfs[dtSfx0->sgfn], cg->fgfs[dtSfy0->sgfn], cg->fgfs[dtSfz0->sgfn],
Mass_here, Porg_here, Pmom_here, Spin_here, BH_NM);
}
if (BL == Pp->data->ble)
break;
BL = BL->next;
}
Pp = Pp->next;
}
// dump read_in initial data
SH->Dump_Data(StateList, 0, PhysTime, dT);
#endif
delete[] Porg_here;
delete[] Mass_here;
delete[] Pmom_here;
delete[] Spin_here;
// SH->Synch(GH->PatL[0],StateList,Symmetry);
// exit(0);
}
}
//================================================================================================
//================================================================================================
// This member function computes Kerr-Schild initial data via an analytic method
//================================================================================================
void bssn_class::Setup_KerrSchild()
{
if (!checkrun)
{
if (myrank == 0)
{
cout << endl;
cout << " Setup initial data with Kerr-Schild formula. " << endl;
cout << endl;
}
// set initial data
for (int lev = 0; lev < GH->levels; lev++)
{
MyList<Patch> *Pp = GH->PatL[lev];
while (Pp)
{
MyList<Block> *BL = Pp->data->blb;
while (BL)
{
Block *cg = BL->data;
if (myrank == cg->rank)
{
f_get_initial_kerrschild(cg->shape, cg->X[0], cg->X[1], cg->X[2],
cg->fgfs[phi0->sgfn], cg->fgfs[trK0->sgfn],
cg->fgfs[gxx0->sgfn], cg->fgfs[gxy0->sgfn], cg->fgfs[gxz0->sgfn],
cg->fgfs[gyy0->sgfn], cg->fgfs[gyz0->sgfn], cg->fgfs[gzz0->sgfn],
cg->fgfs[Axx0->sgfn], cg->fgfs[Axy0->sgfn], cg->fgfs[Axz0->sgfn],
cg->fgfs[Ayy0->sgfn], cg->fgfs[Ayz0->sgfn], cg->fgfs[Azz0->sgfn],
cg->fgfs[Gmx0->sgfn], cg->fgfs[Gmy0->sgfn], cg->fgfs[Gmz0->sgfn],
cg->fgfs[Lap0->sgfn],
cg->fgfs[Sfx0->sgfn], cg->fgfs[Sfy0->sgfn], cg->fgfs[Sfz0->sgfn],
cg->fgfs[dtSfx0->sgfn], cg->fgfs[dtSfy0->sgfn], cg->fgfs[dtSfz0->sgfn]);
}
if (BL == Pp->data->ble)
break;
BL = BL->next;
}
Pp = Pp->next;
}
}
#ifdef WithShell
// ShellPatch part
MyList<ss_patch> *Pp = SH->PatL;
while (Pp)
{
int lev = 0, fngfs = Pp->data->fngfs;
MyList<Block> *BL = Pp->data->blb;
while (BL)
{
Block *cg = BL->data;
if (myrank == cg->rank)
{
f_get_initial_kerrschild_ss(cg->shape,
cg->fgfs[Pp->data->fngfs + ShellPatch::gx],
cg->fgfs[Pp->data->fngfs + ShellPatch::gy],
cg->fgfs[Pp->data->fngfs + ShellPatch::gz],
cg->fgfs[phi0->sgfn], cg->fgfs[trK0->sgfn],
cg->fgfs[gxx0->sgfn], cg->fgfs[gxy0->sgfn], cg->fgfs[gxz0->sgfn],
cg->fgfs[gyy0->sgfn], cg->fgfs[gyz0->sgfn], cg->fgfs[gzz0->sgfn],
cg->fgfs[Axx0->sgfn], cg->fgfs[Axy0->sgfn], cg->fgfs[Axz0->sgfn],
cg->fgfs[Ayy0->sgfn], cg->fgfs[Ayz0->sgfn], cg->fgfs[Azz0->sgfn],
cg->fgfs[Gmx0->sgfn], cg->fgfs[Gmy0->sgfn], cg->fgfs[Gmz0->sgfn],
cg->fgfs[Lap0->sgfn],
cg->fgfs[Sfx0->sgfn], cg->fgfs[Sfy0->sgfn], cg->fgfs[Sfz0->sgfn],
cg->fgfs[dtSfx0->sgfn], cg->fgfs[dtSfy0->sgfn], cg->fgfs[dtSfz0->sgfn]);
/*
f_fderivs_shc(cg->shape,cg->fgfs[phi0->sgfn],
cg->fgfs[Sfx_rhs->sgfn],
cg->fgfs[Sfy_rhs->sgfn],
cg->fgfs[Sfz_rhs->sgfn],
cg->X[0],cg->X[1],cg->X[2],
phi0->SoA[0],phi0->SoA[1],phi0->SoA[2],
Symmetry,lev,Pp->data->sst,
cg->fgfs[fngfs+ShellPatch::drhodx],
cg->fgfs[fngfs+ShellPatch::drhody],
cg->fgfs[fngfs+ShellPatch::drhodz],
cg->fgfs[fngfs+ShellPatch::dsigmadx],
cg->fgfs[fngfs+ShellPatch::dsigmady],
cg->fgfs[fngfs+ShellPatch::dsigmadz],
cg->fgfs[fngfs+ShellPatch::dRdx],
cg->fgfs[fngfs+ShellPatch::dRdy],
cg->fgfs[fngfs+ShellPatch::dRdz]);
f_fdderivs_shc(cg->shape,cg->fgfs[phi0->sgfn],
cg->fgfs[Axx_rhs->sgfn],cg->fgfs[Axy_rhs->sgfn],cg->fgfs[Axz_rhs->sgfn],
cg->fgfs[Ayy_rhs->sgfn],cg->fgfs[Ayz_rhs->sgfn],cg->fgfs[Azz_rhs->sgfn],
cg->X[0],cg->X[1],cg->X[2],
phi0->SoA[0],phi0->SoA[1],phi0->SoA[2],
Symmetry,lev,Pp->data->sst,
cg->fgfs[fngfs+ShellPatch::drhodx],
cg->fgfs[fngfs+ShellPatch::drhody],
cg->fgfs[fngfs+ShellPatch::drhodz],
cg->fgfs[fngfs+ShellPatch::dsigmadx],
cg->fgfs[fngfs+ShellPatch::dsigmady],
cg->fgfs[fngfs+ShellPatch::dsigmadz],
cg->fgfs[fngfs+ShellPatch::dRdx],
cg->fgfs[fngfs+ShellPatch::dRdy],
cg->fgfs[fngfs+ShellPatch::dRdz],
cg->fgfs[fngfs+ShellPatch::drhodxx],
cg->fgfs[fngfs+ShellPatch::drhodxy],
cg->fgfs[fngfs+ShellPatch::drhodxz],
cg->fgfs[fngfs+ShellPatch::drhodyy],
cg->fgfs[fngfs+ShellPatch::drhodyz],
cg->fgfs[fngfs+ShellPatch::drhodzz],
cg->fgfs[fngfs+ShellPatch::dsigmadxx],
cg->fgfs[fngfs+ShellPatch::dsigmadxy],
cg->fgfs[fngfs+ShellPatch::dsigmadxz],
cg->fgfs[fngfs+ShellPatch::dsigmadyy],
cg->fgfs[fngfs+ShellPatch::dsigmadyz],
cg->fgfs[fngfs+ShellPatch::dsigmadzz],
cg->fgfs[fngfs+ShellPatch::dRdxx],
cg->fgfs[fngfs+ShellPatch::dRdxy],
cg->fgfs[fngfs+ShellPatch::dRdxz],
cg->fgfs[fngfs+ShellPatch::dRdyy],
cg->fgfs[fngfs+ShellPatch::dRdyz],
cg->fgfs[fngfs+ShellPatch::dRdzz]);
*/
}
if (BL == Pp->data->ble)
break;
BL = BL->next;
}
Pp = Pp->next;
}
#endif
// dump read_in initial data
// SH->Synch(GH->PatL[0],StateList,Symmetry);
// for(int lev=0;lev<GH->levels;lev++) Parallel::Dump_Data(GH->PatL[lev],StateList,0,PhysTime,dT);
// SH->Dump_Data(StateList,0,PhysTime,dT);
// exit(0);
/*
{
MyList<var> * DG_List=new MyList<var>(Sfx_rhs);
DG_List->insert(Sfy_rhs);
DG_List->insert(Sfz_rhs);
DG_List->insert(Axx_rhs);
DG_List->insert(Axy_rhs);
DG_List->insert(Axz_rhs);
DG_List->insert(Ayy_rhs);
DG_List->insert(Ayz_rhs);
DG_List->insert(Azz_rhs);
SH->Synch(DG_List,Symmetry);
SH->Dump_Data(DG_List,0,PhysTime,dT);
DG_List->clearList();
exit(0);
}
*/
}
}
//================================================================================================
//================================================================================================
// This member function reads initial data produced by Pablo Galaviz's Olliptic program
//================================================================================================
// Read initial data solved by Pablo's Olliptic Phys.Rev.D 82 024005 (2010)
//|----------------------------------------------------------------------------
// read ASCII file with the style of Pablo
//|----------------------------------------------------------------------------
bool bssn_class::read_Pablo_file(int *ext, double *datain, char *filename)
{
if (myrank == 0)
{
cout << endl;
cout << " Setup initial data with Pablo_file. " << endl;
cout << endl;
}
int nx = ext[0], ny = ext[1], nz = ext[2];
int i, j, k;
double x, y, z;
//|--->open in put file
ifstream infile;
infile.open(filename);
if (!infile)
{
cout << "bssn_class: read_Pablo_file can't open " << filename << " for input." << endl;
return false;
}
for (k = 0; k < nz; k++)
for (j = 0; j < ny; j++)
for (i = 0; i < nx; i++)
{
infile >> x >> y >> z >> datain[i + j * nx + k * nx * ny];
}
infile.close();
return true;
}
//================================================================================================
//================================================================================================
// This member function writes initial data file in the style of Pablo Galaviz's Olliptic program
//================================================================================================
//|----------------------------------------------------------------------------
// write ASCII file with the style of Pablo
//|----------------------------------------------------------------------------
void bssn_class::write_Pablo_file(int *ext, double xmin, double xmax, double ymin, double ymax, double zmin, double zmax,
char *filename)
{
int nx = ext[0], ny = ext[1], nz = ext[2];
int i, j, k;
double *X, *Y, *Z;
X = new double[nx];
Y = new double[ny];
Z = new double[nz];
double dX, dY, dZ;
#ifdef Vertex
#ifdef Cell
#error Both Cell and Vertex are defined
#endif
dX = (xmax - xmin) / (nx - 1);
for (i = 0; i < nx; i++)
X[i] = xmin + i * dX;
dY = (ymax - ymin) / (ny - 1);
for (j = 0; j < ny; j++)
Y[j] = ymin + j * dY;
dZ = (zmax - zmin) / (nz - 1);
for (k = 0; k < nz; k++)
Z[k] = zmin + k * dZ;
#else
#ifdef Cell
dX = (xmax - xmin) / nx;
for (i = 0; i < nx; i++)
X[i] = xmin + (i + 0.5) * dX;
dY = (ymax - ymin) / ny;
for (j = 0; j < ny; j++)
Y[j] = ymin + (j + 0.5) * dY;
dZ = (zmax - zmin) / nz;
for (k = 0; k < nz; k++)
Z[k] = zmin + (k + 0.5) * dZ;
#else
#error Not define Vertex nor Cell
#endif
#endif
//|--->open out put file
ofstream outfile;
outfile.open(filename);
if (!outfile)
{
cout << "bssn_class: write_Pablo_file can't open " << filename << " for output." << endl;
MPI_Abort(MPI_COMM_WORLD, 1);
}
outfile.setf(ios::scientific, ios::floatfield);
outfile.precision(16);
for (k = 0; k < nz; k++)
for (j = 0; j < ny; j++)
for (i = 0; i < nx; i++)
{
outfile << X[i] << " " << Y[j] << " " << Z[k] << " "
<< 0 << endl;
}
outfile.close();
delete[] X;
delete[] Y;
delete[] Z;
}
//================================================================================================
//================================================================================================
// Read initial data solved by Ansorg, PRD 70, 064011 (2004)
void bssn_class::Read_Ansorg()
{
if (!checkrun)
{
if (myrank == 0)
{
cout << endl;
cout << " Read initial data from Ansorg's solver,"
<< " please be sure the input parameters for black holes are puncture parameters!! " << endl;
cout << endl;
}
char filename[50];
{
map<string, string>::iterator iter = parameters::str_par.find("inputpar");
if (iter != parameters::str_par.end())
{
strcpy(filename, (iter->second).c_str());
}
else
{
cout << "Error inputpar" << endl;
exit(0);
}
}
int BH_NM;
double *Porg_here, *Pmom_here, *Spin_here, *Mass_here;
// read parameter from file
{
const int LEN = 256;
char pline[LEN];
string str, sgrp, skey, sval;
int sind;
ifstream inf(filename, ifstream::in);
if (!inf.good() && myrank == 0)
{
if (ErrorMonitor->outfile)
ErrorMonitor->outfile << "Can not open parameter file " << filename
<< " for inputing information of black holes" << endl;
MPI_Abort(MPI_COMM_WORLD, 1);
}
for (int i = 1; inf.good(); i++)
{
inf.getline(pline, LEN);
str = pline;
int status = misc::parse_parts(str, sgrp, skey, sval, sind);
if (status == -1)
{
if (ErrorMonitor->outfile)
ErrorMonitor->outfile << "error reading parameter file " << filename << " in line " << i << endl;
MPI_Abort(MPI_COMM_WORLD, 1);
}
else if (status == 0)
continue;
if (sgrp == "BSSN" && skey == "BH_num")
{
BH_NM = atoi(sval.c_str());
break;
}
}
inf.close();
}
Porg_here = new double[3 * BH_NM];
Pmom_here = new double[3 * BH_NM];
Spin_here = new double[3 * BH_NM];
Mass_here = new double[BH_NM];
// read parameter from file
{
const int LEN = 256;
char pline[LEN];
string str, sgrp, skey, sval;
int sind;
ifstream inf(filename, ifstream::in);
if (!inf.good() && myrank == 0)
{
if (ErrorMonitor->outfile)
ErrorMonitor->outfile << "Can not open parameter file " << filename
<< " for inputing information of black holes" << endl;
MPI_Abort(MPI_COMM_WORLD, 1);
}
for (int i = 1; inf.good(); i++)
{
inf.getline(pline, LEN);
str = pline;
int status = misc::parse_parts(str, sgrp, skey, sval, sind);
if (status == -1)
{
if (ErrorMonitor->outfile)
ErrorMonitor->outfile << "error reading parameter file " << filename << " in line " << i << endl;
MPI_Abort(MPI_COMM_WORLD, 1);
}
else if (status == 0)
continue;
if (sgrp == "BSSN" && sind < BH_NM)
{
if (skey == "Mass")
Mass_here[sind] = atof(sval.c_str());
else if (skey == "Porgx")
Porg_here[sind * 3] = atof(sval.c_str());
else if (skey == "Porgy")
Porg_here[sind * 3 + 1] = atof(sval.c_str());
else if (skey == "Porgz")
Porg_here[sind * 3 + 2] = atof(sval.c_str());
else if (skey == "Spinx")
Spin_here[sind * 3] = atof(sval.c_str());
else if (skey == "Spiny")
Spin_here[sind * 3 + 1] = atof(sval.c_str());
else if (skey == "Spinz")
Spin_here[sind * 3 + 2] = atof(sval.c_str());
else if (skey == "Pmomx")
Pmom_here[sind * 3] = atof(sval.c_str());
else if (skey == "Pmomy")
Pmom_here[sind * 3 + 1] = atof(sval.c_str());
else if (skey == "Pmomz")
Pmom_here[sind * 3 + 2] = atof(sval.c_str());
}
}
inf.close();
}
int order = 6;
Ansorg read_ansorg("Ansorg.psid", order);
// set initial data
for (int lev = 0; lev < GH->levels; lev++)
{
MyList<Patch> *Pp = GH->PatL[lev];
while (Pp)
{
MyList<Block> *BL = Pp->data->blb;
while (BL)
{
Block *cg = BL->data;
if (myrank == cg->rank)
{
for (int k = 0; k < cg->shape[2]; k++)
for (int j = 0; j < cg->shape[1]; j++)
for (int i = 0; i < cg->shape[0]; i++)
cg->fgfs[phi0->sgfn][i + j * cg->shape[0] + k * cg->shape[0] * cg->shape[1]] =
read_ansorg.ps_u_at_xyz(cg->X[0][i], cg->X[1][j], cg->X[2][k]);
f_get_ansorg_nbhs(cg->shape, cg->X[0], cg->X[1], cg->X[2],
cg->fgfs[phi0->sgfn], cg->fgfs[trK0->sgfn],
cg->fgfs[gxx0->sgfn], cg->fgfs[gxy0->sgfn], cg->fgfs[gxz0->sgfn],
cg->fgfs[gyy0->sgfn], cg->fgfs[gyz0->sgfn], cg->fgfs[gzz0->sgfn],
cg->fgfs[Axx0->sgfn], cg->fgfs[Axy0->sgfn], cg->fgfs[Axz0->sgfn],
cg->fgfs[Ayy0->sgfn], cg->fgfs[Ayz0->sgfn], cg->fgfs[Azz0->sgfn],
cg->fgfs[Gmx0->sgfn], cg->fgfs[Gmy0->sgfn], cg->fgfs[Gmz0->sgfn],
cg->fgfs[Lap0->sgfn],
cg->fgfs[Sfx0->sgfn], cg->fgfs[Sfy0->sgfn], cg->fgfs[Sfz0->sgfn],
cg->fgfs[dtSfx0->sgfn], cg->fgfs[dtSfy0->sgfn], cg->fgfs[dtSfz0->sgfn],
Mass_here, Porg_here, Pmom_here, Spin_here, BH_NM);
}
if (BL == Pp->data->ble)
break;
BL = BL->next;
}
Pp = Pp->next;
}
}
#ifdef WithShell
// ShellPatch part
MyList<ss_patch> *Pp = SH->PatL;
while (Pp)
{
MyList<Block> *BL = Pp->data->blb;
while (BL)
{
Block *cg = BL->data;
if (myrank == cg->rank)
{
for (int k = 0; k < cg->shape[2]; k++)
for (int j = 0; j < cg->shape[1]; j++)
for (int i = 0; i < cg->shape[0]; i++)
cg->fgfs[phi0->sgfn][i + j * cg->shape[0] + k * cg->shape[0] * cg->shape[1]] =
read_ansorg.ps_u_at_xyz(cg->fgfs[Pp->data->fngfs + ShellPatch::gx][i + j * cg->shape[0] + k * cg->shape[0] * cg->shape[1]],
cg->fgfs[Pp->data->fngfs + ShellPatch::gy][i + j * cg->shape[0] + k * cg->shape[0] * cg->shape[1]],
cg->fgfs[Pp->data->fngfs + ShellPatch::gz][i + j * cg->shape[0] + k * cg->shape[0] * cg->shape[1]]);
f_get_ansorg_nbhs_ss(cg->shape,
cg->fgfs[Pp->data->fngfs + ShellPatch::gx],
cg->fgfs[Pp->data->fngfs + ShellPatch::gy],
cg->fgfs[Pp->data->fngfs + ShellPatch::gz],
cg->fgfs[phi0->sgfn], cg->fgfs[trK0->sgfn],
cg->fgfs[gxx0->sgfn], cg->fgfs[gxy0->sgfn], cg->fgfs[gxz0->sgfn],
cg->fgfs[gyy0->sgfn], cg->fgfs[gyz0->sgfn], cg->fgfs[gzz0->sgfn],
cg->fgfs[Axx0->sgfn], cg->fgfs[Axy0->sgfn], cg->fgfs[Axz0->sgfn],
cg->fgfs[Ayy0->sgfn], cg->fgfs[Ayz0->sgfn], cg->fgfs[Azz0->sgfn],
cg->fgfs[Gmx0->sgfn], cg->fgfs[Gmy0->sgfn], cg->fgfs[Gmz0->sgfn],
cg->fgfs[Lap0->sgfn],
cg->fgfs[Sfx0->sgfn], cg->fgfs[Sfy0->sgfn], cg->fgfs[Sfz0->sgfn],
cg->fgfs[dtSfx0->sgfn], cg->fgfs[dtSfy0->sgfn], cg->fgfs[dtSfz0->sgfn],
Mass_here, Porg_here, Pmom_here, Spin_here, BH_NM);
#if 0
// for check fderivs_sh
f_fderivs_sh(cg->shape,cg->fgfs[Ayz0->sgfn],
cg->fgfs[Sfx0->sgfn],cg->fgfs[Sfy0->sgfn],cg->fgfs[Sfz0->sgfn],
cg->X[0],cg->X[1],cg->X[2],
Ayz0->SoA[0],Ayz0->SoA[1],Ayz0->SoA[2],
Symmetry,Pp->data->sst,Pp->data->sst);
#endif
#if 0
// for check fderivs_shc
int fngfs = Pp->data->fngfs;
f_fderivs_shc(cg->shape,cg->fgfs[Ayz0->sgfn],
cg->fgfs[Sfx0->sgfn],cg->fgfs[Sfy0->sgfn],cg->fgfs[Sfz0->sgfn],
cg->X[0],cg->X[1],cg->X[2],
Ayz0->SoA[0],Ayz0->SoA[1],Ayz0->SoA[2],
Symmetry,Pp->data->sst,Pp->data->sst,
cg->fgfs[fngfs+ShellPatch::drhodx],
cg->fgfs[fngfs+ShellPatch::drhody],
cg->fgfs[fngfs+ShellPatch::drhodz],
cg->fgfs[fngfs+ShellPatch::dsigmadx],
cg->fgfs[fngfs+ShellPatch::dsigmady],
cg->fgfs[fngfs+ShellPatch::dsigmadz],
cg->fgfs[fngfs+ShellPatch::dRdx],
cg->fgfs[fngfs+ShellPatch::dRdy],
cg->fgfs[fngfs+ShellPatch::dRdz]);
#endif
}
if (BL == Pp->data->ble)
break;
BL = BL->next;
}
Pp = Pp->next;
}
#endif
delete[] Porg_here;
delete[] Mass_here;
delete[] Pmom_here;
delete[] Spin_here;
Compute_Constraint();
// dump read_in initial data
for (int lev = 0; lev < GH->levels; lev++)
Parallel::Dump_Data(GH->PatL[lev], DumpList, 0, PhysTime, dT);
#ifdef WithShell
SH->Dump_Data(DumpList, 0, PhysTime, dT);
#endif
// if(myrank==0) MPI_Abort(MPI_COMM_WORLD,1);
}
}
//================================================================================================
//================================================================================================
// This member function sets up the time evolution for the entire process
//================================================================================================
void bssn_class::Evolve(int Steps)
{
clock_t prev_clock, curr_clock;
double LastDump = 0.0, LastCheck = 0.0, Last2dDump = 0.0;
LastAnas = 0;
#if 0
//initial checkpoint for special uasge
{
CheckPoint->write_Black_Hole_position(BH_num_input,BH_num,Porg0,Porgbr,Mass);
CheckPoint->writecheck_cgh(PhysTime,GH);
#ifdef WithShell
CheckPoint->writecheck_sh(PhysTime,SH);
#endif
CheckPoint->write_bssn(LastDump,Last2dDump,LastAnas);
misc::tillherecheck("complete initialization preparation"); // we need synchronization here
if(myrank==0) MPI_Abort(MPI_COMM_WORLD,1);
}
#endif
// for step 0 constraint interpolation
Interp_Constraint(true);
#ifdef With_AHF
// setup apparent horizon finder direct of thornburg
{
HN_num = BH_num;
for (int ia = 0; ia < BH_num; ia++)
for (int ib = ia + 1; ib < BH_num; ib++)
HN_num++;
AHFinderDirect::AHFinderDirect_setup(AHList, GaugeList,
this,
Symmetry, HN_num, &PhysTime);
lastahdumpid = new int[HN_num];
findeveryl = new int[HN_num];
xc = new double[HN_num];
yc = new double[HN_num];
zc = new double[HN_num];
xr = new double[HN_num];
yr = new double[HN_num];
zr = new double[HN_num];
dTT = new double[HN_num];
trigger = new bool[HN_num];
dumpid = new int[HN_num];
for (int ihn = 0; ihn < HN_num; ihn++)
{
lastahdumpid[ihn] = 0;
findeveryl[ihn] = AHfindevery;
}
}
#endif
if (checkrun)
CheckPoint->read_bssn(LastDump, Last2dDump, LastAnas);
double dT_mon = dT * pow(0.5, Mymax(0, trfls));
/*
#ifdef With_AHF
//initial apparent horizon finding
{
double gam;
double massmin=Mass[0];
for(int ihn=1;ihn<BH_num;ihn++) massmin=Mymin(massmin,Mass[ihn]);
for(int ihn=0;ihn<BH_num;ihn++)
{
xc[ihn] = Porg0[ihn][0];
yc[ihn] = Porg0[ihn][1];
zc[ihn] = Porg0[ihn][2];
gam = fabs(Pmom[ihn*3])/(Mass[ihn]);
gam = sqrt(1-gam*gam);
xr[ihn] = Mass[ihn]*gam;
gam = fabs(Pmom[ihn*3+1])/(Mass[ihn]);
gam = sqrt(1-gam*gam);
yr[ihn] = Mass[ihn]*gam;
gam = fabs(Pmom[ihn*3+2])/(Mass[ihn]);
gam = sqrt(1-gam*gam);
zr[ihn] = Mass[ihn]*gam;
findeveryl[ihn] = findeveryl[ihn]*(Mymax(int(Mass[ihn]/massmin),1));
}
int ihn = BH_num;
for(int ia=0;ia<BH_num;ia++)
for(int ib=ia+1;ib<BH_num;ib++) {
xc[ihn] = (Porg0[ia][0] + Porg0[ib][0])/2;
yc[ihn] = (Porg0[ia][1] + Porg0[ib][1])/2;
zc[ihn] = (Porg0[ia][2] + Porg0[ib][2])/2;
xr[ihn] = yr[ihn] = zr[ihn] = (Mass[ia])+(Mass[ib]);
findeveryl[ihn] = findeveryl[ihn]*(int(xr[ihn]/Mass[0]));
ihn++;
}
AHFinderDirect::AHFinderDirect_enforcefind(HN_num,xc,yc,zc,xr,yr,zr);
// note rhs has been used as temp storage space
delete[] xc; delete[] yc; delete[] zc; delete[] xr; delete[] yr; delete[] zr;
}
#endif
*/
perf bssn_perf;
size_t current_min, current_avg, current_max, peak_min, peak_avg, peak_max;
for (int lev = 0; lev < GH->levels; lev++)
GH->Lt[lev] = PhysTime;
GH->settrfls(trfls);
for (int ncount = 1; ncount < Steps + 1; ncount++)
{
// special for large mass ratio consideration
// if(fabs(Porg0[0][0]-Porg0[1][0])+fabs(Porg0[0][1]-Porg0[1][1])+fabs(Porg0[0][2]-Porg0[1][2])<1e-6)
// { GH->levels=GH->movls; }
if (myrank == 0)
curr_clock = clock();
#if (PSTR == 0)
RecursiveStep(0);
#elif (PSTR == 1 || PSTR == 2 || PSTR == 3)
// data analysis part
// Warning NOTE: the variables1 are used as temp storege room
AnalysisStuff(a_lev, dT_mon);
ParallelStep();
#endif
// misc::tillherecheck("before Constraint_Out");
Constraint_Out(); // this will affect the Dump_List
LastDump += dT_mon;
Last2dDump += dT_mon;
LastCheck += dT_mon;
// When LastDump >= DumpTime, output corresponding binary data
if (LastDump >= DumpTime)
{
// misc::tillherecheck("before Dump_Data");
for (int lev = 0; lev < GH->levels; lev++)
Parallel::Dump_Data(GH->PatL[lev], DumpList, 0, PhysTime, dT_mon);
#ifdef WithShell
SH->Dump_Data(DumpList, 0, PhysTime, dT_mon);
#endif
LastDump = 0;
if (myrank == 0)
{
cout << " Dump done. " << endl;
}
}
// When Last2dDump >= d2DumpTime, output corresponding 2D data
if (Last2dDump >= d2DumpTime)
{
// misc::tillherecheck("before 2dDump_Data");
for (int lev = 0; lev < GH->levels; lev++)
Parallel::d2Dump_Data(GH->PatL[lev], DumpList, 0, PhysTime, dT_mon);
Last2dDump = 0;
if (myrank == 0)
{
cout << " 2d Dump done. " << endl;
}
}
if (myrank == 0)
{
prev_clock = curr_clock;
curr_clock = clock();
cout << endl;
cout << " Timestep # " << ncount << ": integrating to time: " << PhysTime << " "
<< " Computer used " << (double)(curr_clock - prev_clock) / ((double)CLOCKS_PER_SEC)
<< " seconds! " << endl;
// cout << endl;
}
if (PhysTime >= TotalTime)
break;
#if (REGLEV == 1)
GH->Regrid(Symmetry, BH_num, Porgbr, Porg0,
SynchList_cor, OldStateList, StateList, SynchList_pre,
fgt(PhysTime - dT_mon, StartTime, dT_mon / 2), ErrorMonitor);
for (int il = 0; il < GH->levels; il++) { sync_cache_pre[il].invalidate(); sync_cache_cor[il].invalidate(); sync_cache_rp_coarse[il].invalidate(); sync_cache_rp_fine[il].invalidate(); }
#endif
#if (REGLEV == 0 && (PSTR == 1 || PSTR == 2))
// GH->Regrid_fake(Symmetry,BH_num,Porgbr,Porg0,
// SynchList_cor,OldStateList,StateList,SynchList_pre,
// fgt(PhysTime-dT_mon,StartTime,dT_mon/2),ErrorMonitor);
#endif
// Retrieve memory usage information used during computation; master process prints it
bssn_perf.MemoryUsage(&current_min, &current_avg, &current_max,
&peak_min, &peak_avg, &peak_max, nprocs);
if (myrank == 0)
{
printf(" Memory usage: current %0.4lg/%0.4lg/%0.4lgMB, "
"peak %0.4lg/%0.4lg/%0.4lgMB\n",
(double)current_min / (1024.0 * 1024.0),
(double)current_avg / (1024.0 * 1024.0),
(double)current_max / (1024.0 * 1024.0),
(double)peak_min / (1024.0 * 1024.0),
(double)peak_avg / (1024.0 * 1024.0),
(double)peak_max / (1024.0 * 1024.0));
cout << endl;
}
// Output puncture positions at each step
if (myrank == 0)
{
for (int i_count=0; i_count<BH_num; i_count++)
{
cout << " puncture position: no."
<< setw(2) << setfill(' ') << i_count
<< " = (" << Porg0[i_count][0] << " "
<< Porg0[i_count][1] << " "
<< Porg0[i_count][2] << ")"
<< endl;
}
cout << endl;
cout << " If you think the physical evolution time is enough for this simulation, please input 'stop' in the terminal to stop the MPI processes in the next evolution step ! " << endl;
// cout << endl;
}
////////////////////////////////////////////////////////////
// If an "abort" command is detected on stdin, terminate MPI processes
////////////////////////////////////////////////////////////
bool shouldAbort = false;
// Only rank 0 checks stdin
if (myrank == 0) {
if (check_Stdin_Abort()) {
shouldAbort = true;
}
}
// Broadcast abort flag to all ranks
int abortFlag = shouldAbort ? 1 : 0;
MPI_Bcast(&abortFlag, 1, MPI_INT, 0, MPI_COMM_WORLD);
if (abortFlag && myrank == 0) {
cout << endl;
cout << " Aborting the MPI Process due to 'abort' command !! " << endl;
cout << endl;
MPI_Abort(MPI_COMM_WORLD, 1); // terminate MPI processes
// MPI_Finalize();
}
////////////////////////////////////////////////////////////
// When LastCheck >= CheckTime, perform runtime checks and output status data
if (LastCheck >= CheckTime)
{
LastCheck = 0;
CheckPoint->write_Black_Hole_position(BH_num_input, BH_num, Porg0, Porgbr, Mass);
CheckPoint->writecheck_cgh(PhysTime, GH);
#ifdef WithShell
CheckPoint->writecheck_sh(PhysTime, SH);
#endif
CheckPoint->write_bssn(LastDump, Last2dDump, LastAnas);
}
}
/*
#ifdef With_AHF
// final apparent horizon finding
{
double gam;
for(int ihn=0;ihn<BH_num;ihn++)
{
xc[ihn] = Porg0[ihn][0];
yc[ihn] = Porg0[ihn][1];
zc[ihn] = Porg0[ihn][2];
gam = fabs(Pmom[ihn*3] )/(Mass[ihn]);
gam = sqrt(1-gam*gam);
xr[ihn] = Mass[ihn]*gam;
gam = fabs(Pmom[ihn*3+1])/(Mass[ihn]);
gam = sqrt(1-gam*gam);
yr[ihn] = Mass[ihn]*gam;
gam = fabs(Pmom[ihn*3+2])/(Mass[ihn]);
gam = sqrt(1-gam*gam);
zr[ihn] = Mass[ihn]*gam;
}
int ihn = BH_num;
for(int ia=0;ia<BH_num;ia++)
for(int ib=ia+1;ib<BH_num;ib++)
{
xc[ihn] = (Porg0[ia][0] + Porg0[ib][0])/2;
yc[ihn] = (Porg0[ia][1] + Porg0[ib][1])/2;
zc[ihn] = (Porg0[ia][2] + Porg0[ib][2])/2;
xr[ihn] = yr[ihn] = zr[ihn] = (Mass[ia])+(Mass[ib]);
ihn++;
}
AHFinderDirect::AHFinderDirect_enforcefind(HN_num,xc,yc,zc,xr,yr,zr);
// note rhs has been used as temp storage space
delete[] xc; delete[] yc; delete[] zc; delete[] xr; delete[] yr; delete[] zr;
}
#endif
*/
}
//================================================================================================
//================================================================================================
// This member function implements recursive time-stepping across AMR levels
// This variant is for the case PSTR == 0
//================================================================================================
#if (PSTR == 0)
void bssn_class::RecursiveStep(int lev)
{
double dT_lev = dT * pow(0.5, Mymax(lev, trfls));
int NoIterations = 1, YN;
if (lev <= trfls)
NoIterations = 1;
else
NoIterations = 2;
for (int i = 0; i < NoIterations; i++)
{
// if(myrank==0) cout<<"level now = "<<lev<<" NoIteration = "<<i<<endl;
YN = (i == NoIterations - 1) ? 1 : 0; // 1: same time level for coarse level and fine level
Step(lev, YN);
#if (AGM == 2)
if (GH->levels == 1)
{
Enforce_algcon(lev, 0);
}
#endif
GH->Lt[lev] += dT_lev;
if (lev < GH->levels - 1)
{
int lf = lev + 1;
RecursiveStep(lf);
}
else
PhysTime += dT * pow(0.5, lev);
#if (AGM == 2)
if (lev > 0)
{
Enforce_algcon(lev, 0);
if (YN == 1)
Enforce_algcon(lev - 1, 0);
}
#endif
#if (RPS == 1)
// mesh refinement boundary part
//
// till here the PhysTime has updated dT_lev
// if(myrank==0) cout<<"level now = "<<lev<<", "<<fgt(PhysTime-dT_lev,StartTime,dT_lev/2)<<endl;
RestrictProlong(lev, YN, fgt(PhysTime - dT_lev, StartTime, dT_lev / 2), StateList, OldStateList, SynchList_cor);
// RestrictProlong(lev,YN,false,StateList,OldStateList,SynchList_cor);
#ifdef WithShell
if (lev == 0)
{
clock_t prev_clock, curr_clock;
if (myrank == 0)
curr_clock = clock();
SH->CS_Inter(StateList, Symmetry);
if (myrank == 0)
{
prev_clock = curr_clock;
curr_clock = clock();
cout << " CS_Inter used " << (double)(curr_clock - prev_clock) / ((double)CLOCKS_PER_SEC) << " seconds! " << endl;
}
}
#endif
#endif
// Parallel::Dump_Data(GH->PatL[lev],StateList,0,PhysTime,dT_lev);
}
#if 0
if(lev>0) Parallel::Restrict_after(GH->PatL[lev-1],GH->PatL[lev],StateList,StateList,Symmetry);
#endif
#if (REGLEV == 0)
GH->Regrid_Onelevel(lev, Symmetry, BH_num, Porgbr, Porg0,
SynchList_cor, OldStateList, StateList, SynchList_pre,
fgt(PhysTime - dT_lev, StartTime, dT_lev / 2), ErrorMonitor);
for (int il = 0; il < GH->levels; il++) { sync_cache_pre[il].invalidate(); sync_cache_cor[il].invalidate(); sync_cache_rp_coarse[il].invalidate(); sync_cache_rp_fine[il].invalidate(); }
#endif
}
//================================================================================================
//================================================================================================
// This member function implements recursive time-stepping across AMR levels
// This variant handles the cases PSTR == 1 and PSTR == 2
//================================================================================================
#elif (PSTR == 1 || PSTR == 2)
void bssn_class::RecursiveStep(int lev)
{
double dT_lev = dT * pow(0.5, Mymax(lev, trfls));
int NoIterations = 1, YN;
if (lev <= trfls)
NoIterations = 1;
else
NoIterations = 2;
for (int i = 0; i < NoIterations; i++)
{
// if(myrank==0) cout<<"level now = "<<lev<<" NoIteration = "<<i<<endl;
YN = (i == NoIterations - 1) ? 1 : 0; // 1: same time level for coarse level and fine level
if (GH->mylev == lev)
{
Step(lev, YN);
#if (AGM == 2)
if (GH->levels == 1)
{
Enforce_algcon(lev, 0);
}
else if (lev > 0)
{
Enforce_algcon(lev, 0);
if (YN == 1)
Enforce_algcon(lev - 1, 0);
}
#endif
}
if (GH->mylev == lev || GH->mylev == lev - 1)
{
#if (RPS == 1)
// mesh refinement boundary part
//
// till here the PhysTime has updated dT_lev
RestrictProlong(lev, YN, fgt(PhysTime - dT_lev, StartTime, dT_lev / 2), StateList, OldStateList, SynchList_cor);
#endif
}
GH->Lt[lev] += dT_lev;
PhysTime += dT * pow(0.5, lev);
}
if (lev > 0)
{
int lc = lev - 1;
RecursiveStep(lc);
}
}
//================================================================================================
//================================================================================================
// ParallelStep performs time evolution across AMR levels (parallelized)
//================================================================================================
#if 1
void bssn_class::ParallelStep()
{
int YN, NoIterations = 1;
if (GH->mylev <= trfls)
NoIterations = 1;
else
NoIterations = int(pow(2.0, GH->mylev - trfls));
double dT_lev = dT * pow(0.5, Mymax(GH->mylev, trfls));
for (int i = 0; i < NoIterations; i++)
{
YN = (i % 2) ? 1 : 0; // 1: same time level for coarse level and fine level
Step(GH->mylev, YN);
#if (RPS == 1)
// mesh refinement boundary part
//
// till here the PhysTime has updated dT_lev
if (GH->mylev < GH->levels - 1)
{
RestrictProlong(GH->mylev + 1, 0, fgt(PhysTime - dT_lev / 2, StartTime, dT_lev / 4), StateList, OldStateList, SynchList_cor);
RestrictProlong(GH->mylev + 1, 1, fgt(PhysTime - dT_lev / 2, StartTime, dT_lev / 4), StateList, OldStateList, SynchList_cor);
}
if (GH->mylev > 0)
RestrictProlong(GH->mylev, YN, fgt(PhysTime - dT_lev, StartTime, dT_lev / 2), StateList, OldStateList, SynchList_cor);
#endif
#ifdef WithShell
SHStep();
#if (RPS == 1)
{
clock_t prev_clock, curr_clock;
if (myrank == 0)
curr_clock = clock();
SH->CS_Inter(StateList, Symmetry);
if (myrank == 0)
{
prev_clock = curr_clock;
curr_clock = clock();
cout << " CS_Inter used " << (double)(curr_clock - prev_clock) / ((double)CLOCKS_PER_SEC) << " seconds! " << endl;
}
}
#endif
#endif
PhysTime += dT_lev;
}
// stringstream a_stream;
// a_stream.setf(ios::left);
double *tporg, *tporgo;
tporg = new double[3 * BH_num];
tporgo = new double[3 * BH_num];
{
int lev = GH->mylev;
MPI_Status status;
// receive
if (lev < GH->levels - 1)
{
if (myrank == GH->start_rank[lev])
{
MPI_Recv(tporgo, 3 * BH_num, MPI_DOUBLE, GH->start_rank[lev + 1], 1, MPI_COMM_WORLD, &status);
// cout<<tporgo[0]<<","<<tporgo[1]<<","<<tporgo[2]<<endl;
}
else
for (int i = 0; i < 3 * BH_num; i++)
tporgo[i] = 0;
MPI_Allreduce(tporgo, tporg, 3 * BH_num, MPI_DOUBLE, MPI_SUM, GH->Commlev[lev]);
for (int i = 0; i < BH_num; i++)
for (int j = 0; j < 3; j++)
Porg0[i][j] = tporg[3 * i + j];
// if(myrank==GH->start_rank[lev]) cout<<Porg0[0][0]<<","<<Porg0[0][1]<<","<<Porg0[0][2]<<endl;
}
// send
if (lev > 0 && myrank == GH->start_rank[lev])
{
for (int i = 0; i < BH_num; i++)
for (int j = 0; j < 3; j++)
tporg[3 * i + j] = Porg0[i][j];
MPI_Send(tporg, 3 * BH_num, MPI_DOUBLE, GH->start_rank[lev - 1], 1, MPI_COMM_WORLD);
}
// a_stream.clear();
// a_stream.str("");
// a_stream<<lev<<": after PBH Sync";
// misc::tillherecheck(GH->Commlev[lev],GH->start_rank[lev],a_stream.str());
}
delete[] tporg;
delete[] tporgo;
#if (REGLEV == 0)
GH->Regrid_Onelevel(GH->mylev, Symmetry, BH_num, Porgbr, Porg0,
SynchList_cor, OldStateList, StateList, SynchList_pre,
fgt(PhysTime - dT_lev, StartTime, dT_lev / 2), ErrorMonitor);
for (int il = 0; il < GH->levels; il++) { sync_cache_pre[il].invalidate(); sync_cache_cor[il].invalidate(); sync_cache_rp_coarse[il].invalidate(); sync_cache_rp_fine[il].invalidate(); }
#endif
}
//================================================================================================
//================================================================================================
// ParallelStep performs time evolution across AMR levels (parallelized)
// This is an alternate implementation
//================================================================================================
#else
void bssn_class::ParallelStep()
{
// stringstream a_stream;
// a_stream.setf(ios::left);
double *tporg, *tporgo;
tporg = new double[3 * BH_num];
tporgo = new double[3 * BH_num];
int lev = GH->mylev;
double dT_lev = dT * pow(0.5, Mymax(lev, trfls));
double dT_levp1 = dT * pow(0.5, Mymax(lev + 1, trfls));
double dT_levm1 = dT * pow(0.5, Mymax(lev - 1, trfls));
int NoIterations = 1, YN;
if (lev <= trfls)
NoIterations = 1;
else
NoIterations = int(pow(2.0, lev - trfls));
for (int i = 0; i < NoIterations; i++)
{
// if(myrank==GH->start_rank[lev]) cout<<"level now = "<<lev<<" NoIteration = "<<i<<endl;
if (NoIterations == 1)
YN = 1;
else if (i % 2 == 0)
YN = 0;
else
YN = 1; // 1: same time level for coarse level and fine level
// a_stream<<lev<<": before calling Step";
// misc::tillherecheck(GH->Commlev[lev],GH->start_rank[lev],a_stream.str());
Step(lev, YN);
// a_stream.clear();
// a_stream.str("");
// a_stream<<lev<<": after calling Step";
// misc::tillherecheck(GH->Commlev[lev],GH->start_rank[lev],a_stream.str());
#if (AGM == 2)
if (GH->levels == 1)
{
Enforce_algcon(lev, 0);
}
#endif
GH->Lt[lev] += dT_lev;
PhysTime += dT_lev;
#if (AGM == 2)
if (lev > 0)
{
Enforce_algcon(lev, 0);
if (YN == 1)
Enforce_algcon(lev - 1, 0);
}
#endif
#if (RPS == 1)
// mesh refinement boundary part
//
// till here the PhysTime has updated dT_lev
// a_stream.clear();
// a_stream.str("");
// a_stream<<lev<<": before RestrictProlong";
// misc::tillherecheck(GH->Commlev[lev],GH->start_rank[lev],a_stream.str());
if (lev < GH->levels - 1)
{
if (lev + 1 <= trfls)
{
// RestrictProlong_aux(lev,1,fgt(PhysTime-dT_lev,StartTime,dT_levp1/2),StateList,OldStateList,SynchList_cor);
RestrictProlong(lev + 1, 1, fgt(PhysTime - dT_lev, StartTime, dT_levp1 / 2), StateList, OldStateList, SynchList_cor);
}
else
{
// if(myrank==GH->start_rank[lev]) cout<<GH->mylev<<", "<<YN<<endl;
// misc::tillherecheck(GH->Commlev[lev],GH->start_rank[lev],"between RestrictProlong");
// RestrictProlong_aux(lev,0,fgt(PhysTime-dT_lev,StartTime,dT_levp1/2),StateList,OldStateList,SynchList_cor);
// RestrictProlong_aux(lev,1,fgt(PhysTime-dT_levp1,StartTime,dT_levp1/2),StateList,OldStateList,SynchList_cor);
RestrictProlong(lev + 1, 0, fgt(PhysTime - dT_lev, StartTime, dT_levp1 / 2), StateList, OldStateList, SynchList_cor);
RestrictProlong(lev + 1, 1, fgt(PhysTime - dT_levp1, StartTime, dT_levp1 / 2), StateList, OldStateList, SynchList_cor);
}
}
// if(myrank==GH->start_rank[lev]) cout<<GH->mylev<<", "<<YN<<endl;
// a_stream.clear();
// a_stream.str("");
// a_stream<<lev<<": between RestrictProlong";
// misc::tillherecheck(GH->Commlev[lev],GH->start_rank[lev],a_stream.str());
RestrictProlong(lev, YN, fgt(PhysTime - dT_lev, StartTime, dT_lev / 2), StateList, OldStateList, SynchList_cor);
// RestrictProlong(lev,YN,false,StateList,OldStateList,SynchList_cor);
// if(myrank==GH->start_rank[lev]) cout<<GH->mylev<<endl;
// a_stream.clear();
// a_stream.str("");
// a_stream<<lev<<": after RestrictProlong";
// misc::tillherecheck(GH->Commlev[lev],GH->start_rank[lev],a_stream.str());
#endif
// Parallel::Dump_Data(GH->PatL[lev],StateList,0,PhysTime,dT_lev);
{
MPI_Status status;
// receive
if (lev < GH->levels - 1)
{
if (myrank == GH->start_rank[lev])
{
MPI_Recv(tporgo, 3 * BH_num, MPI_DOUBLE, GH->start_rank[lev + 1], 1, MPI_COMM_WORLD, &status);
// cout<<tporgo[0]<<","<<tporgo[1]<<","<<tporgo[2]<<endl;
}
else
for (int i = 0; i < 3 * BH_num; i++)
tporgo[i] = 0;
MPI_Allreduce(tporgo, tporg, 3 * BH_num, MPI_DOUBLE, MPI_SUM, GH->Commlev[lev]);
for (int i = 0; i < BH_num; i++)
for (int j = 0; j < 3; j++)
Porg0[i][j] = tporg[3 * i + j];
// if(myrank==GH->start_rank[lev]) cout<<Porg0[0][0]<<","<<Porg0[0][1]<<","<<Porg0[0][2]<<endl;
}
// send
if (lev > 0 && YN == 1 && myrank == GH->start_rank[lev])
{
for (int i = 0; i < BH_num; i++)
for (int j = 0; j < 3; j++)
tporg[3 * i + j] = Porg0[i][j];
MPI_Send(tporg, 3 * BH_num, MPI_DOUBLE, GH->start_rank[lev - 1], 1, MPI_COMM_WORLD);
}
// a_stream.clear();
// a_stream.str("");
// a_stream<<lev<<": after PBH Sync";
// misc::tillherecheck(GH->Commlev[lev],GH->start_rank[lev],a_stream.str());
}
#if (REGLEV == 0)
// for higher level
if (lev < GH->levels - 1)
{
if (lev + 1 >= GH->movls)
{
// GH->Regrid_Onelevel_aux(lev,Symmetry,BH_num,Porgbr,Porg0,
GH->Regrid_Onelevel(lev + 1, Symmetry, BH_num, Porgbr, Porg0,
SynchList_cor, OldStateList, StateList, SynchList_pre,
fgt(PhysTime - dT_levp1, StartTime, dT_levp1 / 2), ErrorMonitor);
for (int il = 0; il < GH->levels; il++) { sync_cache_pre[il].invalidate(); sync_cache_cor[il].invalidate(); sync_cache_rp_coarse[il].invalidate(); sync_cache_rp_fine[il].invalidate(); }
// a_stream.clear();
// a_stream.str("");
// a_stream<<lev<<": after calling GH->Regrid_Onelevel_aux for higher level";
// misc::tillherecheck(GH->Commlev[lev],GH->start_rank[lev],a_stream.str());
}
}
// for this level
if (YN == 1)
{
GH->Regrid_Onelevel(lev, Symmetry, BH_num, Porgbr, Porg0,
SynchList_cor, OldStateList, StateList, SynchList_pre,
fgt(PhysTime - dT_lev, StartTime, dT_lev / 2), ErrorMonitor);
for (int il = 0; il < GH->levels; il++) { sync_cache_pre[il].invalidate(); sync_cache_cor[il].invalidate(); sync_cache_rp_coarse[il].invalidate(); sync_cache_rp_fine[il].invalidate(); }
// a_stream.clear();
// a_stream.str("");
// a_stream<<lev<<": after calling GH->Regrid_Onelevel";
// misc::tillherecheck(GH->Commlev[lev],GH->start_rank[lev],a_stream.str());
}
// for lower level
if (lev - 1 >= GH->movls)
{
if (lev - 1 <= trfls)
{
if (YN == 1)
{
// GH->Regrid_Onelevel_aux(lev-2,Symmetry,BH_num,Porgbr,Porg0,
GH->Regrid_Onelevel(lev - 1, Symmetry, BH_num, Porgbr, Porg0,
SynchList_cor, OldStateList, StateList, SynchList_pre,
fgt(PhysTime - dT_lev, StartTime, dT_levm1 / 2), ErrorMonitor);
for (int il = 0; il < GH->levels; il++) { sync_cache_pre[il].invalidate(); sync_cache_cor[il].invalidate(); sync_cache_rp_coarse[il].invalidate(); sync_cache_rp_fine[il].invalidate(); }
// a_stream.clear();
// a_stream.str("");
// a_stream<<lev<<": after calling GH->Regrid_Onelevel_aux for lower level";
// misc::tillherecheck(GH->Commlev[lev],GH->start_rank[lev],a_stream.str());
}
}
else
{
if (i % 4 == 3)
{
// GH->Regrid_Onelevel_aux(lev-2,Symmetry,BH_num,Porgbr,Porg0,
GH->Regrid_Onelevel(lev - 1, Symmetry, BH_num, Porgbr, Porg0,
SynchList_cor, OldStateList, StateList, SynchList_pre,
fgt(PhysTime - dT_lev, StartTime, dT_levm1 / 2), ErrorMonitor);
for (int il = 0; il < GH->levels; il++) { sync_cache_pre[il].invalidate(); sync_cache_cor[il].invalidate(); sync_cache_rp_coarse[il].invalidate(); sync_cache_rp_fine[il].invalidate(); }
// a_stream.clear();
// a_stream.str("");
// a_stream<<lev<<": after calling GH->Regrid_Onelevel_aux for lower level";
// misc::tillherecheck(GH->Commlev[lev],GH->start_rank[lev],a_stream.str());
}
}
}
#endif
}
#ifdef WithShell
SHStep();
// a_stream.clear();
// a_stream.str("");
// a_stream<<lev<<": after calling SHStep";
// misc::tillherecheck(GH->Commlev[lev],GH->start_rank[lev],a_stream.str());
#if (RPS == 1)
{
clock_t prev_clock, curr_clock;
if (myrank == 0)
curr_clock = clock();
SH->CS_Inter(StateList, Symmetry);
if (myrank == 0)
{
prev_clock = curr_clock;
curr_clock = clock();
cout << " CS_Inter used " << (double)(curr_clock - prev_clock) / ((double)CLOCKS_PER_SEC)
<< " seconds! " << endl;
}
// a_stream.clear();
// a_stream.str("");
// a_stream<<lev<<": after calling shell cartisian sync";
// misc::tillherecheck(GH->Commlev[lev],GH->start_rank[lev],a_stream.str());
}
#endif
#endif
#if 0
if(lev>0) Parallel::Restrict_after(GH->PatL[lev-1],GH->PatL[lev],StateList,StateList,Symmetry);
#endif
delete[] tporg;
delete[] tporgo;
}
#endif
//================================================================================================
//================================================================================================
// ParallelStep performs time evolution across AMR levels (parallelized)
// This is another implementation, for the case PSTR == 3
//================================================================================================
#elif (PSTR == 3)
#warning "remember do not use Shell"
void bssn_class::ParallelStep()
{
// stringstream a_stream;
// a_stream.setf(ios::left);
double *tporg, *tporgo;
tporg = new double[3 * BH_num];
tporgo = new double[3 * BH_num];
int lev = GH->mylev;
double dT_lev = dT * pow(0.5, Mymax(GH->levels - 1, trfls));
if (lev == 1)
{
lev = GH->levels - 1;
for (int i = 0; i < misc::MYpow2(lev); i++)
{
Step(lev, i % 2);
PhysTime += dT_lev;
// if(myrank==nprocs-1) cout<<"OOO level now = "<<lev<<", "<<i%2<<", "<<fgt(PhysTime-dT_lev,StartTime,dT_lev/2)<<endl;
RestrictProlong(lev, i % 2, fgt(PhysTime - dT_lev, StartTime, dT_lev / 2), StateList, OldStateList, SynchList_cor);
}
}
else
{
lev = GH->levels - 2;
for (int i = 1; i < misc::MYpow2(lev + 1); i++)
{
RecursiveStep(lev, i);
PhysTime += dT_lev;
if (i % 2 == 0)
{
// if(myrank==0) cout<<"level now = "<<lev+1<<", "<<fgt(PhysTime-dT_lev,StartTime,dT_lev/2)<<endl;
RestrictProlong(lev + 1, 1, fgt(PhysTime - dT_lev, StartTime, dT_lev / 2), StateList, OldStateList, SynchList_cor);
}
}
PhysTime += dT_lev;
// if(myrank==0) cout<<"level now = "<<lev+1<<", "<<fgt(PhysTime-dT_lev,StartTime,dT_lev/2)<<endl;
RestrictProlong(lev + 1, 1, fgt(PhysTime - dT_lev, StartTime, dT_lev / 2), StateList, OldStateList, SynchList_cor);
}
lev = GH->mylev;
if (lev == -1)
lev = 0;
else
lev = GH->levels - 1;
{
MPI_Status status;
// receive
if (lev == 0)
{
if (myrank == GH->start_rank[lev])
{
MPI_Recv(tporgo, 3 * BH_num, MPI_DOUBLE, GH->start_rank[GH->levels - 1], 1, MPI_COMM_WORLD, &status);
// cout<<tporgo[0]<<","<<tporgo[1]<<","<<tporgo[2]<<endl;
}
else
for (int i = 0; i < 3 * BH_num; i++)
tporgo[i] = 0;
MPI_Allreduce(tporgo, tporg, 3 * BH_num, MPI_DOUBLE, MPI_SUM, GH->Commlev[lev]);
for (int i = 0; i < BH_num; i++)
for (int j = 0; j < 3; j++)
Porg0[i][j] = tporg[3 * i + j];
// if(myrank==GH->start_rank[lev]) cout<<Porg0[0][0]<<","<<Porg0[0][1]<<","<<Porg0[0][2]<<endl;
}
// send
else if (myrank == GH->start_rank[lev])
{
for (int i = 0; i < BH_num; i++)
for (int j = 0; j < 3; j++)
tporg[3 * i + j] = Porg0[i][j];
MPI_Send(tporg, 3 * BH_num, MPI_DOUBLE, GH->start_rank[0], 1, MPI_COMM_WORLD);
}
}
delete[] tporg;
delete[] tporgo;
}
//================================================================================================
//================================================================================================
// This member function implements recursive time-stepping across AMR levels
//================================================================================================
void bssn_class::RecursiveStep(int lev, int num) // in all 2^(lev+1)-1 steps
{
if (trfls > 0)
cout << "error: bssn_class::RecursiveStep does not support trfls > 0 yet" << endl;
if (num / 2 * 2 == num)
RecursiveStep(lev - 1, num / 2);
else
{
Step(lev, 0);
double dT_lev = dT * pow(0.5, Mymax(lev + 1, trfls));
if (myrank == 0)
cout << "level now = " << lev + 1 << ", " << (num - 1) % 2 << ", "
<< fgt(PhysTime - dT_lev, StartTime, dT_lev / 2) << endl;
RestrictProlong(lev + 1, (num - 1) % 2, fgt(PhysTime - dT_lev, StartTime, dT_lev / 2), StateList, OldStateList, SynchList_cor);
}
}
#endif
//================================================================================================
//================================================================================================
// This member function configures a single time-step evolution for each grid level.
// Applicable for the case PSTR == 0
//================================================================================================
#if (PSTR == 0)
#if 1
void bssn_class::Step(int lev, int YN)
{
setpbh(BH_num, Porg0, Mass, BH_num_input);
double dT_lev = dT * pow(0.5, Mymax(lev, trfls));
// new code 2013-2-15, zjcao
#if (MAPBH == 1)
// for black hole position
if (BH_num > 0 && lev == GH->levels - 1)
{
compute_Porg_rhs(Porg0, Porg_rhs, Sfx0, Sfy0, Sfz0, lev);
for (int ithBH = 0; ithBH < BH_num; ithBH++)
{
for (int ith = 0; ith < 3; ith++)
Porg1[ithBH][ith] = Porg0[ithBH][ith] + Porg_rhs[ithBH][ith] * dT_lev;
if (Symmetry > 0)
Porg1[ithBH][2] = fabs(Porg1[ithBH][2]);
if (Symmetry == 2)
{
Porg1[ithBH][0] = fabs(Porg1[ithBH][0]);
Porg1[ithBH][1] = fabs(Porg1[ithBH][1]);
}
if (!finite(Porg1[ithBH][0]) || !finite(Porg1[ithBH][1]) || !finite(Porg1[ithBH][2]))
{
if (ErrorMonitor->outfile)
ErrorMonitor->outfile << "predictor step finds NaN for BH's position from ("
<< Porg0[ithBH][0] << "," << Porg0[ithBH][1] << "," << Porg0[ithBH][2] << ")" << endl;
MyList<var> *DG_List = new MyList<var>(Sfx0);
DG_List->insert(Sfx0);
DG_List->insert(Sfy0);
DG_List->insert(Sfz0);
Parallel::Dump_Data(GH->PatL[lev], DG_List, 0, PhysTime, dT_lev);
DG_List->clearList();
}
}
}
// data analysis part
// Warning NOTE: the variables1 are used as temp storege room
if (lev == a_lev)
{
AnalysisStuff(lev, dT_lev);
}
#endif
#ifdef With_AHF
AH_Step_Find(lev, dT_lev);
#endif
bool BB = fgt(PhysTime, StartTime, dT_lev / 2);
double ndeps = numepss;
if (lev < GH->movls)
ndeps = numepsb;
double TRK4 = PhysTime;
int iter_count = 0; // count RK4 substeps
int pre = 0, cor = 1;
int ERROR = 0;
MyList<ss_patch> *sPp;
// Predictor
MyList<Patch> *Pp = GH->PatL[lev];
while (Pp)
{
MyList<Block> *BP = Pp->data->blb;
while (BP)
{
Block *cg = BP->data;
if (myrank == cg->rank)
{
#if (AGM == 0)
f_enforce_ga(cg->shape,
cg->fgfs[gxx0->sgfn], cg->fgfs[gxy0->sgfn], cg->fgfs[gxz0->sgfn],
cg->fgfs[gyy0->sgfn], cg->fgfs[gyz0->sgfn], cg->fgfs[gzz0->sgfn],
cg->fgfs[Axx0->sgfn], cg->fgfs[Axy0->sgfn], cg->fgfs[Axz0->sgfn],
cg->fgfs[Ayy0->sgfn], cg->fgfs[Ayz0->sgfn], cg->fgfs[Azz0->sgfn]);
#endif
if (f_compute_rhs_bssn(cg->shape, TRK4, cg->X[0], cg->X[1], cg->X[2],
cg->fgfs[phi0->sgfn], cg->fgfs[trK0->sgfn],
cg->fgfs[gxx0->sgfn], cg->fgfs[gxy0->sgfn], cg->fgfs[gxz0->sgfn],
cg->fgfs[gyy0->sgfn], cg->fgfs[gyz0->sgfn], cg->fgfs[gzz0->sgfn],
cg->fgfs[Axx0->sgfn], cg->fgfs[Axy0->sgfn], cg->fgfs[Axz0->sgfn],
cg->fgfs[Ayy0->sgfn], cg->fgfs[Ayz0->sgfn], cg->fgfs[Azz0->sgfn],
cg->fgfs[Gmx0->sgfn], cg->fgfs[Gmy0->sgfn], cg->fgfs[Gmz0->sgfn],
cg->fgfs[Lap0->sgfn],
cg->fgfs[Sfx0->sgfn], cg->fgfs[Sfy0->sgfn], cg->fgfs[Sfz0->sgfn],
cg->fgfs[dtSfx0->sgfn], cg->fgfs[dtSfy0->sgfn], cg->fgfs[dtSfz0->sgfn],
cg->fgfs[phi_rhs->sgfn], cg->fgfs[trK_rhs->sgfn],
cg->fgfs[gxx_rhs->sgfn], cg->fgfs[gxy_rhs->sgfn], cg->fgfs[gxz_rhs->sgfn],
cg->fgfs[gyy_rhs->sgfn], cg->fgfs[gyz_rhs->sgfn], cg->fgfs[gzz_rhs->sgfn],
cg->fgfs[Axx_rhs->sgfn], cg->fgfs[Axy_rhs->sgfn], cg->fgfs[Axz_rhs->sgfn],
cg->fgfs[Ayy_rhs->sgfn], cg->fgfs[Ayz_rhs->sgfn], cg->fgfs[Azz_rhs->sgfn],
cg->fgfs[Gmx_rhs->sgfn], cg->fgfs[Gmy_rhs->sgfn], cg->fgfs[Gmz_rhs->sgfn],
cg->fgfs[Lap_rhs->sgfn],
cg->fgfs[Sfx_rhs->sgfn], cg->fgfs[Sfy_rhs->sgfn], cg->fgfs[Sfz_rhs->sgfn],
cg->fgfs[dtSfx_rhs->sgfn], cg->fgfs[dtSfy_rhs->sgfn], cg->fgfs[dtSfz_rhs->sgfn],
cg->fgfs[rho->sgfn], cg->fgfs[Sx->sgfn], cg->fgfs[Sy->sgfn], cg->fgfs[Sz->sgfn],
cg->fgfs[Sxx->sgfn], cg->fgfs[Sxy->sgfn], cg->fgfs[Sxz->sgfn],
cg->fgfs[Syy->sgfn], cg->fgfs[Syz->sgfn], cg->fgfs[Szz->sgfn],
cg->fgfs[Gamxxx->sgfn], cg->fgfs[Gamxxy->sgfn], cg->fgfs[Gamxxz->sgfn],
cg->fgfs[Gamxyy->sgfn], cg->fgfs[Gamxyz->sgfn], cg->fgfs[Gamxzz->sgfn],
cg->fgfs[Gamyxx->sgfn], cg->fgfs[Gamyxy->sgfn], cg->fgfs[Gamyxz->sgfn],
cg->fgfs[Gamyyy->sgfn], cg->fgfs[Gamyyz->sgfn], cg->fgfs[Gamyzz->sgfn],
cg->fgfs[Gamzxx->sgfn], cg->fgfs[Gamzxy->sgfn], cg->fgfs[Gamzxz->sgfn],
cg->fgfs[Gamzyy->sgfn], cg->fgfs[Gamzyz->sgfn], cg->fgfs[Gamzzz->sgfn],
cg->fgfs[Rxx->sgfn], cg->fgfs[Rxy->sgfn], cg->fgfs[Rxz->sgfn],
cg->fgfs[Ryy->sgfn], cg->fgfs[Ryz->sgfn], cg->fgfs[Rzz->sgfn],
cg->fgfs[Cons_Ham->sgfn],
cg->fgfs[Cons_Px->sgfn], cg->fgfs[Cons_Py->sgfn], cg->fgfs[Cons_Pz->sgfn],
cg->fgfs[Cons_Gx->sgfn], cg->fgfs[Cons_Gy->sgfn], cg->fgfs[Cons_Gz->sgfn],
Symmetry, lev, ndeps, pre))
{
cout << "find NaN in domain: ("
<< cg->bbox[0] << ":" << cg->bbox[3] << ","
<< cg->bbox[1] << ":" << cg->bbox[4] << ","
<< cg->bbox[2] << ":" << cg->bbox[5] << ")" << endl;
ERROR = 1;
}
// rk4 substep and boundary
{
MyList<var> *varl0 = StateList, *varl = SynchList_pre, *varlrhs = RHSList; // we do not check the correspondence here
while (varl0)
{
#if (SommerType == 0)
#ifndef WithShell
if (lev == 0) // sommerfeld indeed
f_sommerfeld_routbam(cg->shape, cg->X[0], cg->X[1], cg->X[2],
Pp->data->bbox[0], Pp->data->bbox[1], Pp->data->bbox[2],
Pp->data->bbox[3], Pp->data->bbox[4], Pp->data->bbox[5],
cg->fgfs[varlrhs->data->sgfn],
cg->fgfs[varl0->data->sgfn],
varl0->data->propspeed, varl0->data->SoA,
Symmetry);
#endif
#endif
f_rungekutta4_rout(cg->shape, dT_lev,
cg->fgfs[varl0->data->sgfn],
cg->fgfs[varl->data->sgfn],
cg->fgfs[varlrhs->data->sgfn],
iter_count);
#ifndef WithShell
if (lev > 0) // fix BD point
#endif
f_sommerfeld_rout(cg->shape, cg->X[0], cg->X[1], cg->X[2],
Pp->data->bbox[0], Pp->data->bbox[1], Pp->data->bbox[2],
Pp->data->bbox[3], Pp->data->bbox[4], Pp->data->bbox[5],
dT_lev,
cg->fgfs[phi0->sgfn],
cg->fgfs[Lap0->sgfn],
cg->fgfs[varl0->data->sgfn], cg->fgfs[varl->data->sgfn],
varl0->data->SoA,
Symmetry, cor);
#if (SommerType == 1)
#warning "shell part still bam type"
if (lev == 0) // Shibata type sommerfeld
f_sommerfeld_rout(cg->shape, cg->X[0], cg->X[1], cg->X[2],
Pp->data->bbox[0], Pp->data->bbox[1], Pp->data->bbox[2],
Pp->data->bbox[3], Pp->data->bbox[4], Pp->data->bbox[5],
dT_lev,
cg->fgfs[phi0->sgfn],
cg->fgfs[Lap0->sgfn],
cg->fgfs[varl0->data->sgfn], cg->fgfs[varl->data->sgfn],
varl0->data->SoA,
Symmetry, pre);
#endif
varl0 = varl0->next;
varl = varl->next;
varlrhs = varlrhs->next;
}
}
f_lowerboundset(cg->shape, cg->fgfs[phi->sgfn], chitiny);
}
if (BP == Pp->data->ble)
break;
BP = BP->next;
}
Pp = Pp->next;
}
// NOTE: error check deferred to after Shell Patch computation to reduce MPI_Allreduce calls
#ifdef WithShell
// evolve Shell Patches
if (lev == 0)
{
sPp = SH->PatL;
while (sPp)
{
MyList<Block> *BP = sPp->data->blb;
int fngfs = sPp->data->fngfs;
while (BP)
{
Block *cg = BP->data;
if (myrank == cg->rank)
{
#if (AGM == 0)
f_enforce_ga(cg->shape,
cg->fgfs[gxx0->sgfn], cg->fgfs[gxy0->sgfn], cg->fgfs[gxz0->sgfn],
cg->fgfs[gyy0->sgfn], cg->fgfs[gyz0->sgfn], cg->fgfs[gzz0->sgfn],
cg->fgfs[Axx0->sgfn], cg->fgfs[Axy0->sgfn], cg->fgfs[Axz0->sgfn],
cg->fgfs[Ayy0->sgfn], cg->fgfs[Ayz0->sgfn], cg->fgfs[Azz0->sgfn]);
#endif
if (f_compute_rhs_bssn_ss(cg->shape, TRK4, cg->X[0], cg->X[1], cg->X[2],
cg->fgfs[fngfs + ShellPatch::gx],
cg->fgfs[fngfs + ShellPatch::gy],
cg->fgfs[fngfs + ShellPatch::gz],
cg->fgfs[fngfs + ShellPatch::drhodx],
cg->fgfs[fngfs + ShellPatch::drhody],
cg->fgfs[fngfs + ShellPatch::drhodz],
cg->fgfs[fngfs + ShellPatch::dsigmadx],
cg->fgfs[fngfs + ShellPatch::dsigmady],
cg->fgfs[fngfs + ShellPatch::dsigmadz],
cg->fgfs[fngfs + ShellPatch::dRdx],
cg->fgfs[fngfs + ShellPatch::dRdy],
cg->fgfs[fngfs + ShellPatch::dRdz],
cg->fgfs[fngfs + ShellPatch::drhodxx],
cg->fgfs[fngfs + ShellPatch::drhodxy],
cg->fgfs[fngfs + ShellPatch::drhodxz],
cg->fgfs[fngfs + ShellPatch::drhodyy],
cg->fgfs[fngfs + ShellPatch::drhodyz],
cg->fgfs[fngfs + ShellPatch::drhodzz],
cg->fgfs[fngfs + ShellPatch::dsigmadxx],
cg->fgfs[fngfs + ShellPatch::dsigmadxy],
cg->fgfs[fngfs + ShellPatch::dsigmadxz],
cg->fgfs[fngfs + ShellPatch::dsigmadyy],
cg->fgfs[fngfs + ShellPatch::dsigmadyz],
cg->fgfs[fngfs + ShellPatch::dsigmadzz],
cg->fgfs[fngfs + ShellPatch::dRdxx],
cg->fgfs[fngfs + ShellPatch::dRdxy],
cg->fgfs[fngfs + ShellPatch::dRdxz],
cg->fgfs[fngfs + ShellPatch::dRdyy],
cg->fgfs[fngfs + ShellPatch::dRdyz],
cg->fgfs[fngfs + ShellPatch::dRdzz],
cg->fgfs[phi0->sgfn], cg->fgfs[trK0->sgfn],
cg->fgfs[gxx0->sgfn], cg->fgfs[gxy0->sgfn], cg->fgfs[gxz0->sgfn],
cg->fgfs[gyy0->sgfn], cg->fgfs[gyz0->sgfn], cg->fgfs[gzz0->sgfn],
cg->fgfs[Axx0->sgfn], cg->fgfs[Axy0->sgfn], cg->fgfs[Axz0->sgfn],
cg->fgfs[Ayy0->sgfn], cg->fgfs[Ayz0->sgfn], cg->fgfs[Azz0->sgfn],
cg->fgfs[Gmx0->sgfn], cg->fgfs[Gmy0->sgfn], cg->fgfs[Gmz0->sgfn],
cg->fgfs[Lap0->sgfn],
cg->fgfs[Sfx0->sgfn], cg->fgfs[Sfy0->sgfn], cg->fgfs[Sfz0->sgfn],
cg->fgfs[dtSfx0->sgfn], cg->fgfs[dtSfy0->sgfn], cg->fgfs[dtSfz0->sgfn],
cg->fgfs[phi_rhs->sgfn], cg->fgfs[trK_rhs->sgfn],
cg->fgfs[gxx_rhs->sgfn], cg->fgfs[gxy_rhs->sgfn], cg->fgfs[gxz_rhs->sgfn],
cg->fgfs[gyy_rhs->sgfn], cg->fgfs[gyz_rhs->sgfn], cg->fgfs[gzz_rhs->sgfn],
cg->fgfs[Axx_rhs->sgfn], cg->fgfs[Axy_rhs->sgfn], cg->fgfs[Axz_rhs->sgfn],
cg->fgfs[Ayy_rhs->sgfn], cg->fgfs[Ayz_rhs->sgfn], cg->fgfs[Azz_rhs->sgfn],
cg->fgfs[Gmx_rhs->sgfn], cg->fgfs[Gmy_rhs->sgfn], cg->fgfs[Gmz_rhs->sgfn],
cg->fgfs[Lap_rhs->sgfn],
cg->fgfs[Sfx_rhs->sgfn], cg->fgfs[Sfy_rhs->sgfn], cg->fgfs[Sfz_rhs->sgfn],
cg->fgfs[dtSfx_rhs->sgfn], cg->fgfs[dtSfy_rhs->sgfn], cg->fgfs[dtSfz_rhs->sgfn],
cg->fgfs[rho->sgfn], cg->fgfs[Sx->sgfn], cg->fgfs[Sy->sgfn], cg->fgfs[Sz->sgfn],
cg->fgfs[Sxx->sgfn], cg->fgfs[Sxy->sgfn], cg->fgfs[Sxz->sgfn],
cg->fgfs[Syy->sgfn], cg->fgfs[Syz->sgfn], cg->fgfs[Szz->sgfn],
cg->fgfs[Gamxxx->sgfn], cg->fgfs[Gamxxy->sgfn], cg->fgfs[Gamxxz->sgfn],
cg->fgfs[Gamxyy->sgfn], cg->fgfs[Gamxyz->sgfn], cg->fgfs[Gamxzz->sgfn],
cg->fgfs[Gamyxx->sgfn], cg->fgfs[Gamyxy->sgfn], cg->fgfs[Gamyxz->sgfn],
cg->fgfs[Gamyyy->sgfn], cg->fgfs[Gamyyz->sgfn], cg->fgfs[Gamyzz->sgfn],
cg->fgfs[Gamzxx->sgfn], cg->fgfs[Gamzxy->sgfn], cg->fgfs[Gamzxz->sgfn],
cg->fgfs[Gamzyy->sgfn], cg->fgfs[Gamzyz->sgfn], cg->fgfs[Gamzzz->sgfn],
cg->fgfs[Rxx->sgfn], cg->fgfs[Rxy->sgfn], cg->fgfs[Rxz->sgfn],
cg->fgfs[Ryy->sgfn], cg->fgfs[Ryz->sgfn], cg->fgfs[Rzz->sgfn],
cg->fgfs[Cons_Ham->sgfn],
cg->fgfs[Cons_Px->sgfn], cg->fgfs[Cons_Py->sgfn], cg->fgfs[Cons_Pz->sgfn],
cg->fgfs[Cons_Gx->sgfn], cg->fgfs[Cons_Gy->sgfn], cg->fgfs[Cons_Gz->sgfn],
Symmetry, lev, numepsh, sPp->data->sst, pre))
{
cout << "find NaN in Shell domain: sst = " << sPp->data->sst << ", ("
<< cg->bbox[0] << ":" << cg->bbox[3] << ","
<< cg->bbox[1] << ":" << cg->bbox[4] << ","
<< cg->bbox[2] << ":" << cg->bbox[5] << ")" << endl;
ERROR = 1;
}
// rk4 substep and boundary
{
MyList<var> *varl0 = StateList, *varl = SynchList_pre, *varlrhs = RHSList;
// we do not check the correspondence here
while (varl0)
{
// sommerfeld indeed for outter boudary while fix BD for inner boundary
f_sommerfeld_routbam_ss(cg->shape, cg->X[0], cg->X[1], cg->X[2],
sPp->data->bbox[0], sPp->data->bbox[1], sPp->data->bbox[2],
sPp->data->bbox[3], sPp->data->bbox[4], sPp->data->bbox[5],
cg->fgfs[varlrhs->data->sgfn],
cg->fgfs[varl0->data->sgfn],
varl0->data->propspeed, varl0->data->SoA,
Symmetry);
f_rungekutta4_rout(cg->shape, dT_lev,
cg->fgfs[varl0->data->sgfn],
cg->fgfs[varl->data->sgfn],
cg->fgfs[varlrhs->data->sgfn],
iter_count);
varl0 = varl0->next;
varl = varl->next;
varlrhs = varlrhs->next;
}
}
f_lowerboundset(cg->shape, cg->fgfs[phi->sgfn], chitiny);
}
if (BP == sPp->data->ble)
break;
BP = BP->next;
}
sPp = sPp->next;
}
#if 0
// check rhs
{
SH->Dump_Data(RHSList,0,PhysTime,dT_lev);
if(myrank == 0)
{
cout<<"check rhs"<<endl;
MPI_Abort(MPI_COMM_WORLD,1);
}
}
#endif
}
// Non-blocking error reduction overlapped with Sync to hide Allreduce latency
MPI_Request err_req;
{
int erh = ERROR;
MPI_Iallreduce(&erh, &ERROR, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD, &err_req);
}
#endif
Parallel::AsyncSyncState async_pre;
Parallel::Sync_start(GH->PatL[lev], SynchList_pre, Symmetry, sync_cache_pre[lev], async_pre);
#ifdef WithShell
if (lev == 0)
{
clock_t prev_clock, curr_clock;
if (myrank == 0)
curr_clock = clock();
SH->Synch(SynchList_pre, Symmetry);
if (myrank == 0)
{
prev_clock = curr_clock;
curr_clock = clock();
cout << " Shell stuff synchronization used "
<< (double)(curr_clock - prev_clock) / ((double)CLOCKS_PER_SEC)
<< " seconds! " << endl;
}
}
#endif
Parallel::Sync_finish(sync_cache_pre[lev], async_pre, SynchList_pre, Symmetry);
#ifdef WithShell
// Complete non-blocking error reduction and check
MPI_Wait(&err_req, MPI_STATUS_IGNORE);
if (ERROR)
{
Parallel::Dump_Data(GH->PatL[lev], StateList, 0, PhysTime, dT_lev);
SH->Dump_Data(StateList, 0, PhysTime, dT_lev);
if (myrank == 0)
{
if (ErrorMonitor->outfile)
ErrorMonitor->outfile << "find NaN in state variables at t = " << PhysTime << ", lev = " << lev << endl;
MPI_Abort(MPI_COMM_WORLD, 1);
}
}
#endif
#if (MAPBH == 0)
// for black hole position
if (BH_num > 0 && lev == GH->levels - 1)
{
compute_Porg_rhs(Porg0, Porg_rhs, Sfx0, Sfy0, Sfz0, lev);
for (int ithBH = 0; ithBH < BH_num; ithBH++)
{
f_rungekutta4_scalar(dT_lev, Porg0[ithBH][0], Porg[ithBH][0], Porg_rhs[ithBH][0], iter_count);
f_rungekutta4_scalar(dT_lev, Porg0[ithBH][1], Porg[ithBH][1], Porg_rhs[ithBH][1], iter_count);
f_rungekutta4_scalar(dT_lev, Porg0[ithBH][2], Porg[ithBH][2], Porg_rhs[ithBH][2], iter_count);
if (Symmetry > 0)
Porg[ithBH][2] = fabs(Porg[ithBH][2]);
if (Symmetry == 2)
{
Porg[ithBH][0] = fabs(Porg[ithBH][0]);
Porg[ithBH][1] = fabs(Porg[ithBH][1]);
}
if (!finite(Porg[ithBH][0]) || !finite(Porg[ithBH][1]) || !finite(Porg[ithBH][2]))
{
if (ErrorMonitor->outfile)
ErrorMonitor->outfile << "predictor step finds NaN for BH's position from ("
<< Porg0[ithBH][0] << "," << Porg0[ithBH][1] << "," << Porg0[ithBH][2] << ")" << endl;
MyList<var> *DG_List = new MyList<var>(Sfx0);
DG_List->insert(Sfx0);
DG_List->insert(Sfy0);
DG_List->insert(Sfz0);
Parallel::Dump_Data(GH->PatL[lev], DG_List, 0, PhysTime, dT_lev);
DG_List->clearList();
}
}
}
// data analysis part
// Warning NOTE: the variables1 are used as temp storege room
if (lev == a_lev)
{
AnalysisStuff(lev, dT_lev);
}
#endif
// corrector
for (iter_count = 1; iter_count < 4; iter_count++)
{
// for RK4: t0, t0+dt/2, t0+dt/2, t0+dt;
if (iter_count == 1 || iter_count == 3)
TRK4 += dT_lev / 2;
Pp = GH->PatL[lev];
while (Pp)
{
MyList<Block> *BP = Pp->data->blb;
while (BP)
{
Block *cg = BP->data;
if (myrank == cg->rank)
{
#if (AGM == 0)
f_enforce_ga(cg->shape,
cg->fgfs[gxx->sgfn], cg->fgfs[gxy->sgfn], cg->fgfs[gxz->sgfn],
cg->fgfs[gyy->sgfn], cg->fgfs[gyz->sgfn], cg->fgfs[gzz->sgfn],
cg->fgfs[Axx->sgfn], cg->fgfs[Axy->sgfn], cg->fgfs[Axz->sgfn],
cg->fgfs[Ayy->sgfn], cg->fgfs[Ayz->sgfn], cg->fgfs[Azz->sgfn]);
#elif (AGM == 1)
if (iter_count == 3)
f_enforce_ga(cg->shape,
cg->fgfs[gxx->sgfn], cg->fgfs[gxy->sgfn], cg->fgfs[gxz->sgfn],
cg->fgfs[gyy->sgfn], cg->fgfs[gyz->sgfn], cg->fgfs[gzz->sgfn],
cg->fgfs[Axx->sgfn], cg->fgfs[Axy->sgfn], cg->fgfs[Axz->sgfn],
cg->fgfs[Ayy->sgfn], cg->fgfs[Ayz->sgfn], cg->fgfs[Azz->sgfn]);
#endif
if (f_compute_rhs_bssn(cg->shape, TRK4, cg->X[0], cg->X[1], cg->X[2],
cg->fgfs[phi->sgfn], cg->fgfs[trK->sgfn],
cg->fgfs[gxx->sgfn], cg->fgfs[gxy->sgfn], cg->fgfs[gxz->sgfn],
cg->fgfs[gyy->sgfn], cg->fgfs[gyz->sgfn], cg->fgfs[gzz->sgfn],
cg->fgfs[Axx->sgfn], cg->fgfs[Axy->sgfn], cg->fgfs[Axz->sgfn],
cg->fgfs[Ayy->sgfn], cg->fgfs[Ayz->sgfn], cg->fgfs[Azz->sgfn],
cg->fgfs[Gmx->sgfn], cg->fgfs[Gmy->sgfn], cg->fgfs[Gmz->sgfn],
cg->fgfs[Lap->sgfn],
cg->fgfs[Sfx->sgfn], cg->fgfs[Sfy->sgfn], cg->fgfs[Sfz->sgfn],
cg->fgfs[dtSfx->sgfn], cg->fgfs[dtSfy->sgfn], cg->fgfs[dtSfz->sgfn],
cg->fgfs[phi1->sgfn], cg->fgfs[trK1->sgfn],
cg->fgfs[gxx1->sgfn], cg->fgfs[gxy1->sgfn], cg->fgfs[gxz1->sgfn],
cg->fgfs[gyy1->sgfn], cg->fgfs[gyz1->sgfn], cg->fgfs[gzz1->sgfn],
cg->fgfs[Axx1->sgfn], cg->fgfs[Axy1->sgfn], cg->fgfs[Axz1->sgfn],
cg->fgfs[Ayy1->sgfn], cg->fgfs[Ayz1->sgfn], cg->fgfs[Azz1->sgfn],
cg->fgfs[Gmx1->sgfn], cg->fgfs[Gmy1->sgfn], cg->fgfs[Gmz1->sgfn],
cg->fgfs[Lap1->sgfn],
cg->fgfs[Sfx1->sgfn], cg->fgfs[Sfy1->sgfn], cg->fgfs[Sfz1->sgfn],
cg->fgfs[dtSfx1->sgfn], cg->fgfs[dtSfy1->sgfn], cg->fgfs[dtSfz1->sgfn],
cg->fgfs[rho->sgfn], cg->fgfs[Sx->sgfn], cg->fgfs[Sy->sgfn], cg->fgfs[Sz->sgfn],
cg->fgfs[Sxx->sgfn], cg->fgfs[Sxy->sgfn], cg->fgfs[Sxz->sgfn],
cg->fgfs[Syy->sgfn], cg->fgfs[Syz->sgfn], cg->fgfs[Szz->sgfn],
cg->fgfs[Gamxxx->sgfn], cg->fgfs[Gamxxy->sgfn], cg->fgfs[Gamxxz->sgfn],
cg->fgfs[Gamxyy->sgfn], cg->fgfs[Gamxyz->sgfn], cg->fgfs[Gamxzz->sgfn],
cg->fgfs[Gamyxx->sgfn], cg->fgfs[Gamyxy->sgfn], cg->fgfs[Gamyxz->sgfn],
cg->fgfs[Gamyyy->sgfn], cg->fgfs[Gamyyz->sgfn], cg->fgfs[Gamyzz->sgfn],
cg->fgfs[Gamzxx->sgfn], cg->fgfs[Gamzxy->sgfn], cg->fgfs[Gamzxz->sgfn],
cg->fgfs[Gamzyy->sgfn], cg->fgfs[Gamzyz->sgfn], cg->fgfs[Gamzzz->sgfn],
cg->fgfs[Rxx->sgfn], cg->fgfs[Rxy->sgfn], cg->fgfs[Rxz->sgfn],
cg->fgfs[Ryy->sgfn], cg->fgfs[Ryz->sgfn], cg->fgfs[Rzz->sgfn],
cg->fgfs[Cons_Ham->sgfn],
cg->fgfs[Cons_Px->sgfn], cg->fgfs[Cons_Py->sgfn], cg->fgfs[Cons_Pz->sgfn],
cg->fgfs[Cons_Gx->sgfn], cg->fgfs[Cons_Gy->sgfn], cg->fgfs[Cons_Gz->sgfn],
Symmetry, lev, ndeps, cor))
{
cout << "find NaN in domain: ("
<< cg->bbox[0] << ":" << cg->bbox[3] << ","
<< cg->bbox[1] << ":" << cg->bbox[4] << ","
<< cg->bbox[2] << ":" << cg->bbox[5] << ")" << endl;
ERROR = 1;
}
// rk4 substep and boundary
{
MyList<var> *varl0 = StateList, *varl = SynchList_pre, *varl1 = SynchList_cor, *varlrhs = RHSList; // we do not check the correspondence here
while (varl0)
{
#if (SommerType == 0)
#ifndef WithShell
if (lev == 0) // sommerfeld indeed
f_sommerfeld_routbam(cg->shape, cg->X[0], cg->X[1], cg->X[2],
Pp->data->bbox[0], Pp->data->bbox[1], Pp->data->bbox[2],
Pp->data->bbox[3], Pp->data->bbox[4], Pp->data->bbox[5],
cg->fgfs[varl1->data->sgfn],
cg->fgfs[varl->data->sgfn], varl0->data->propspeed, varl0->data->SoA,
Symmetry);
#endif
#endif
f_rungekutta4_rout(cg->shape, dT_lev,
cg->fgfs[varl0->data->sgfn],
cg->fgfs[varl1->data->sgfn],
cg->fgfs[varlrhs->data->sgfn],
iter_count);
#ifndef WithShell
if (lev > 0) // fix BD point
#endif
f_sommerfeld_rout(cg->shape, cg->X[0], cg->X[1], cg->X[2],
Pp->data->bbox[0], Pp->data->bbox[1], Pp->data->bbox[2],
Pp->data->bbox[3], Pp->data->bbox[4], Pp->data->bbox[5],
dT_lev,
cg->fgfs[phi0->sgfn],
cg->fgfs[Lap0->sgfn],
cg->fgfs[varl0->data->sgfn], cg->fgfs[varl1->data->sgfn],
varl0->data->SoA,
Symmetry, cor);
#if (SommerType == 1)
if (lev == 1) // shibata type sommerfeld
f_sommerfeld_rout(cg->shape, cg->X[0], cg->X[1], cg->X[2],
Pp->data->bbox[0], Pp->data->bbox[1], Pp->data->bbox[2],
Pp->data->bbox[3], Pp->data->bbox[4], Pp->data->bbox[5],
dT_lev,
cg->fgfs[phi0->sgfn],
cg->fgfs[Lap0->sgfn],
cg->fgfs[varl->data->sgfn], cg->fgfs[varl1->data->sgfn],
varl0->data->SoA,
Symmetry, cor);
#endif
varl0 = varl0->next;
varl = varl->next;
varl1 = varl1->next;
varlrhs = varlrhs->next;
}
}
f_lowerboundset(cg->shape, cg->fgfs[phi1->sgfn], chitiny);
}
if (BP == Pp->data->ble)
break;
BP = BP->next;
}
Pp = Pp->next;
}
// NOTE: error check deferred to after Shell Patch computation to reduce MPI_Allreduce calls
#ifdef WithShell
// evolve Shell Patches
if (lev == 0)
{
sPp = SH->PatL;
while (sPp)
{
MyList<Block> *BP = sPp->data->blb;
int fngfs = sPp->data->fngfs;
while (BP)
{
Block *cg = BP->data;
if (myrank == cg->rank)
{
#if (AGM == 0)
f_enforce_ga(cg->shape,
cg->fgfs[gxx->sgfn], cg->fgfs[gxy->sgfn], cg->fgfs[gxz->sgfn],
cg->fgfs[gyy->sgfn], cg->fgfs[gyz->sgfn], cg->fgfs[gzz->sgfn],
cg->fgfs[Axx->sgfn], cg->fgfs[Axy->sgfn], cg->fgfs[Axz->sgfn],
cg->fgfs[Ayy->sgfn], cg->fgfs[Ayz->sgfn], cg->fgfs[Azz->sgfn]);
#elif (AGM == 1)
if (iter_count == 3)
f_enforce_ga(cg->shape,
cg->fgfs[gxx->sgfn], cg->fgfs[gxy->sgfn], cg->fgfs[gxz->sgfn],
cg->fgfs[gyy->sgfn], cg->fgfs[gyz->sgfn], cg->fgfs[gzz->sgfn],
cg->fgfs[Axx->sgfn], cg->fgfs[Axy->sgfn], cg->fgfs[Axz->sgfn],
cg->fgfs[Ayy->sgfn], cg->fgfs[Ayz->sgfn], cg->fgfs[Azz->sgfn]);
#endif
if (f_compute_rhs_bssn_ss(cg->shape, TRK4, cg->X[0], cg->X[1], cg->X[2],
cg->fgfs[fngfs + ShellPatch::gx],
cg->fgfs[fngfs + ShellPatch::gy],
cg->fgfs[fngfs + ShellPatch::gz],
cg->fgfs[fngfs + ShellPatch::drhodx],
cg->fgfs[fngfs + ShellPatch::drhody],
cg->fgfs[fngfs + ShellPatch::drhodz],
cg->fgfs[fngfs + ShellPatch::dsigmadx],
cg->fgfs[fngfs + ShellPatch::dsigmady],
cg->fgfs[fngfs + ShellPatch::dsigmadz],
cg->fgfs[fngfs + ShellPatch::dRdx],
cg->fgfs[fngfs + ShellPatch::dRdy],
cg->fgfs[fngfs + ShellPatch::dRdz],
cg->fgfs[fngfs + ShellPatch::drhodxx],
cg->fgfs[fngfs + ShellPatch::drhodxy],
cg->fgfs[fngfs + ShellPatch::drhodxz],
cg->fgfs[fngfs + ShellPatch::drhodyy],
cg->fgfs[fngfs + ShellPatch::drhodyz],
cg->fgfs[fngfs + ShellPatch::drhodzz],
cg->fgfs[fngfs + ShellPatch::dsigmadxx],
cg->fgfs[fngfs + ShellPatch::dsigmadxy],
cg->fgfs[fngfs + ShellPatch::dsigmadxz],
cg->fgfs[fngfs + ShellPatch::dsigmadyy],
cg->fgfs[fngfs + ShellPatch::dsigmadyz],
cg->fgfs[fngfs + ShellPatch::dsigmadzz],
cg->fgfs[fngfs + ShellPatch::dRdxx],
cg->fgfs[fngfs + ShellPatch::dRdxy],
cg->fgfs[fngfs + ShellPatch::dRdxz],
cg->fgfs[fngfs + ShellPatch::dRdyy],
cg->fgfs[fngfs + ShellPatch::dRdyz],
cg->fgfs[fngfs + ShellPatch::dRdzz],
cg->fgfs[phi->sgfn], cg->fgfs[trK->sgfn],
cg->fgfs[gxx->sgfn], cg->fgfs[gxy->sgfn], cg->fgfs[gxz->sgfn],
cg->fgfs[gyy->sgfn], cg->fgfs[gyz->sgfn], cg->fgfs[gzz->sgfn],
cg->fgfs[Axx->sgfn], cg->fgfs[Axy->sgfn], cg->fgfs[Axz->sgfn],
cg->fgfs[Ayy->sgfn], cg->fgfs[Ayz->sgfn], cg->fgfs[Azz->sgfn],
cg->fgfs[Gmx->sgfn], cg->fgfs[Gmy->sgfn], cg->fgfs[Gmz->sgfn],
cg->fgfs[Lap->sgfn],
cg->fgfs[Sfx->sgfn], cg->fgfs[Sfy->sgfn], cg->fgfs[Sfz->sgfn],
cg->fgfs[dtSfx->sgfn], cg->fgfs[dtSfy->sgfn], cg->fgfs[dtSfz->sgfn],
cg->fgfs[phi1->sgfn], cg->fgfs[trK1->sgfn],
cg->fgfs[gxx1->sgfn], cg->fgfs[gxy1->sgfn], cg->fgfs[gxz1->sgfn],
cg->fgfs[gyy1->sgfn], cg->fgfs[gyz1->sgfn], cg->fgfs[gzz1->sgfn],
cg->fgfs[Axx1->sgfn], cg->fgfs[Axy1->sgfn], cg->fgfs[Axz1->sgfn],
cg->fgfs[Ayy1->sgfn], cg->fgfs[Ayz1->sgfn], cg->fgfs[Azz1->sgfn],
cg->fgfs[Gmx1->sgfn], cg->fgfs[Gmy1->sgfn], cg->fgfs[Gmz1->sgfn],
cg->fgfs[Lap1->sgfn],
cg->fgfs[Sfx1->sgfn], cg->fgfs[Sfy1->sgfn], cg->fgfs[Sfz1->sgfn],
cg->fgfs[dtSfx1->sgfn], cg->fgfs[dtSfy1->sgfn], cg->fgfs[dtSfz1->sgfn],
cg->fgfs[rho->sgfn],
cg->fgfs[Sx->sgfn], cg->fgfs[Sy->sgfn], cg->fgfs[Sz->sgfn],
cg->fgfs[Sxx->sgfn], cg->fgfs[Sxy->sgfn], cg->fgfs[Sxz->sgfn],
cg->fgfs[Syy->sgfn], cg->fgfs[Syz->sgfn], cg->fgfs[Szz->sgfn],
cg->fgfs[Gamxxx->sgfn], cg->fgfs[Gamxxy->sgfn], cg->fgfs[Gamxxz->sgfn],
cg->fgfs[Gamxyy->sgfn], cg->fgfs[Gamxyz->sgfn], cg->fgfs[Gamxzz->sgfn],
cg->fgfs[Gamyxx->sgfn], cg->fgfs[Gamyxy->sgfn], cg->fgfs[Gamyxz->sgfn],
cg->fgfs[Gamyyy->sgfn], cg->fgfs[Gamyyz->sgfn], cg->fgfs[Gamyzz->sgfn],
cg->fgfs[Gamzxx->sgfn], cg->fgfs[Gamzxy->sgfn], cg->fgfs[Gamzxz->sgfn],
cg->fgfs[Gamzyy->sgfn], cg->fgfs[Gamzyz->sgfn], cg->fgfs[Gamzzz->sgfn],
cg->fgfs[Rxx->sgfn], cg->fgfs[Rxy->sgfn], cg->fgfs[Rxz->sgfn],
cg->fgfs[Ryy->sgfn], cg->fgfs[Ryz->sgfn], cg->fgfs[Rzz->sgfn],
cg->fgfs[Cons_Ham->sgfn],
cg->fgfs[Cons_Px->sgfn], cg->fgfs[Cons_Py->sgfn], cg->fgfs[Cons_Pz->sgfn],
cg->fgfs[Cons_Gx->sgfn], cg->fgfs[Cons_Gy->sgfn], cg->fgfs[Cons_Gz->sgfn],
Symmetry, lev, numepsh, sPp->data->sst, cor))
{
cout << "find NaN in Shell domain: sst = " << sPp->data->sst << ", ("
<< cg->bbox[0] << ":" << cg->bbox[3] << ","
<< cg->bbox[1] << ":" << cg->bbox[4] << ","
<< cg->bbox[2] << ":" << cg->bbox[5] << ")" << endl;
ERROR = 1;
}
// rk4 substep and boundary
{
MyList<var> *varl0 = StateList, *varl = SynchList_pre, *varl1 = SynchList_cor, *varlrhs = RHSList;
// we do not check the correspondence here
while (varl0)
{
// sommerfeld indeed for outter boudary while fix BD for inner boundary
f_sommerfeld_routbam_ss(cg->shape, cg->X[0], cg->X[1], cg->X[2],
sPp->data->bbox[0], sPp->data->bbox[1], sPp->data->bbox[2],
sPp->data->bbox[3], sPp->data->bbox[4], sPp->data->bbox[5],
cg->fgfs[varl1->data->sgfn],
cg->fgfs[varl->data->sgfn],
varl0->data->propspeed, varl0->data->SoA,
Symmetry);
f_rungekutta4_rout(cg->shape, dT_lev,
cg->fgfs[varl0->data->sgfn],
cg->fgfs[varl1->data->sgfn],
cg->fgfs[varlrhs->data->sgfn],
iter_count);
varl0 = varl0->next;
varl = varl->next;
varl1 = varl1->next;
varlrhs = varlrhs->next;
}
}
f_lowerboundset(cg->shape, cg->fgfs[phi1->sgfn], chitiny);
}
if (BP == sPp->data->ble)
break;
BP = BP->next;
}
sPp = sPp->next;
}
}
// Non-blocking error reduction overlapped with Sync to hide Allreduce latency
MPI_Request err_req_cor;
{
int erh = ERROR;
MPI_Iallreduce(&erh, &ERROR, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD, &err_req_cor);
}
#endif
Parallel::AsyncSyncState async_cor;
Parallel::Sync_start(GH->PatL[lev], SynchList_cor, Symmetry, sync_cache_cor[lev], async_cor);
#ifdef WithShell
if (lev == 0)
{
clock_t prev_clock, curr_clock;
if (myrank == 0)
curr_clock = clock();
SH->Synch(SynchList_cor, Symmetry);
if (myrank == 0)
{
prev_clock = curr_clock;
curr_clock = clock();
cout << " Shell stuff synchronization used "
<< (double)(curr_clock - prev_clock) / ((double)CLOCKS_PER_SEC)
<< " seconds! " << endl;
}
}
#endif
Parallel::Sync_finish(sync_cache_cor[lev], async_cor, SynchList_cor, Symmetry);
#ifdef WithShell
// Complete non-blocking error reduction and check
MPI_Wait(&err_req_cor, MPI_STATUS_IGNORE);
if (ERROR)
{
Parallel::Dump_Data(GH->PatL[lev], SynchList_pre, 0, PhysTime, dT_lev);
SH->Dump_Data(SynchList_pre, 0, PhysTime, dT_lev);
if (myrank == 0)
{
if (ErrorMonitor->outfile)
ErrorMonitor->outfile << "find NaN in RK4 substep#" << iter_count
<< " variables at t = " << PhysTime
<< ", lev = " << lev << endl;
MPI_Abort(MPI_COMM_WORLD, 1);
}
}
#endif
#if (MAPBH == 0)
// for black hole position
if (BH_num > 0 && lev == GH->levels - 1)
{
compute_Porg_rhs(Porg, Porg1, Sfx, Sfy, Sfz, lev);
for (int ithBH = 0; ithBH < BH_num; ithBH++)
{
f_rungekutta4_scalar(dT_lev, Porg0[ithBH][0], Porg1[ithBH][0], Porg_rhs[ithBH][0], iter_count);
f_rungekutta4_scalar(dT_lev, Porg0[ithBH][1], Porg1[ithBH][1], Porg_rhs[ithBH][1], iter_count);
f_rungekutta4_scalar(dT_lev, Porg0[ithBH][2], Porg1[ithBH][2], Porg_rhs[ithBH][2], iter_count);
if (Symmetry > 0)
Porg1[ithBH][2] = fabs(Porg1[ithBH][2]);
if (Symmetry == 2)
{
Porg1[ithBH][0] = fabs(Porg1[ithBH][0]);
Porg1[ithBH][1] = fabs(Porg1[ithBH][1]);
}
if (!finite(Porg1[ithBH][0]) || !finite(Porg1[ithBH][1]) || !finite(Porg1[ithBH][2]))
{
if (ErrorMonitor->outfile)
ErrorMonitor->outfile << iter_count << " corrector step finds NaN for BH's position from ("
<< Porg[ithBH][0] << "," << Porg[ithBH][1] << "," << Porg[ithBH][2]
<< ")" << endl;
MyList<var> *DG_List = new MyList<var>(Sfx0);
DG_List->insert(Sfx0);
DG_List->insert(Sfy0);
DG_List->insert(Sfz0);
Parallel::Dump_Data(GH->PatL[lev], DG_List, 0, PhysTime, dT_lev);
DG_List->clearList();
}
}
}
#endif
// swap time level
if (iter_count < 3)
{
Pp = GH->PatL[lev];
while (Pp)
{
MyList<Block> *BP = Pp->data->blb;
while (BP)
{
Block *cg = BP->data;
cg->swapList(SynchList_pre, SynchList_cor, myrank);
if (BP == Pp->data->ble)
break;
BP = BP->next;
}
Pp = Pp->next;
}
#ifdef WithShell
if (lev == 0)
{
sPp = SH->PatL;
while (sPp)
{
MyList<Block> *BP = sPp->data->blb;
while (BP)
{
Block *cg = BP->data;
cg->swapList(SynchList_pre, SynchList_cor, myrank);
if (BP == sPp->data->ble)
break;
BP = BP->next;
}
sPp = sPp->next;
}
}
#endif
#if (MAPBH == 0)
// for black hole position
if (BH_num > 0 && lev == GH->levels - 1)
{
for (int ithBH = 0; ithBH < BH_num; ithBH++)
{
Porg[ithBH][0] = Porg1[ithBH][0];
Porg[ithBH][1] = Porg1[ithBH][1];
Porg[ithBH][2] = Porg1[ithBH][2];
}
}
#endif
}
}
#if (RPS == 0)
// mesh refinement boundary part
RestrictProlong(lev, YN, BB);
#ifdef WithShell
if (lev == 0)
{
clock_t prev_clock, curr_clock;
if (myrank == 0)
curr_clock = clock();
SH->CS_Inter(SynchList_cor, Symmetry);
if (myrank == 0)
{
prev_clock = curr_clock;
curr_clock = clock();
cout << " CS_Inter used " << (double)(curr_clock - prev_clock) / ((double)CLOCKS_PER_SEC)
<< " seconds! " << endl;
}
}
#endif
#endif
// note the data structure before update
// SynchList_cor 1 -----------
//
// StateList 0 -----------
//
// OldStateList old -----------
// update
Pp = GH->PatL[lev];
while (Pp)
{
MyList<Block> *BP = Pp->data->blb;
while (BP)
{
Block *cg = BP->data;
cg->swapList(StateList, SynchList_cor, myrank);
cg->swapList(OldStateList, SynchList_cor, myrank);
if (BP == Pp->data->ble)
break;
BP = BP->next;
}
Pp = Pp->next;
}
#ifdef WithShell
if (lev == 0)
{
sPp = SH->PatL;
while (sPp)
{
MyList<Block> *BP = sPp->data->blb;
while (BP)
{
Block *cg = BP->data;
cg->swapList(StateList, SynchList_cor, myrank);
cg->swapList(OldStateList, SynchList_cor, myrank);
if (BP == sPp->data->ble)
break;
BP = BP->next;
}
sPp = sPp->next;
}
#if 0
// check StateList
{
SH->Dump_Data(StateList,0,PhysTime,dT_lev);
if(myrank == 0)
{
cout<<"check StateList"<<endl;
MPI_Abort(MPI_COMM_WORLD,1);
}
}
#endif
}
#endif
// for black hole position
if (BH_num > 0 && lev == GH->levels - 1)
{
for (int ithBH = 0; ithBH < BH_num; ithBH++)
{
Porg0[ithBH][0] = Porg1[ithBH][0];
Porg0[ithBH][1] = Porg1[ithBH][1];
Porg0[ithBH][2] = Porg1[ithBH][2];
}
}
}
//================================================================================================
//================================================================================================
// This member function implements single-step time evolution for each AMR level (alternate)
//================================================================================================
// ICN for bam comparison
#else
void bssn_class::Step(int lev, int YN)
{
double dT_lev = dT * pow(0.5, Mymax(lev, trfls));
#ifdef With_AHF
AH_Step_Find(lev, dT_lev);
#endif
bool BB = fgt(PhysTime, StartTime, dT_lev / 2);
double ndeps = numepss;
if (lev < GH->movls)
ndeps = numepsb;
double TRK4 = PhysTime;
int iter_count = 0; // count RK4 substeps
int pre = 0, cor = 1;
int ERROR = 0;
MyList<ss_patch> *sPp;
// Predictor
MyList<Patch> *Pp = GH->PatL[lev];
while (Pp)
{
MyList<Block> *BP = Pp->data->blb;
while (BP)
{
Block *cg = BP->data;
if (myrank == cg->rank)
{
#if (AGM == 0)
f_enforce_ga(cg->shape,
cg->fgfs[gxx0->sgfn], cg->fgfs[gxy0->sgfn], cg->fgfs[gxz0->sgfn],
cg->fgfs[gyy0->sgfn], cg->fgfs[gyz0->sgfn], cg->fgfs[gzz0->sgfn],
cg->fgfs[Axx0->sgfn], cg->fgfs[Axy0->sgfn], cg->fgfs[Axz0->sgfn],
cg->fgfs[Ayy0->sgfn], cg->fgfs[Ayz0->sgfn], cg->fgfs[Azz0->sgfn]);
#endif
if (f_compute_rhs_bssn(cg->shape, TRK4, cg->X[0], cg->X[1], cg->X[2],
cg->fgfs[phi0->sgfn], cg->fgfs[trK0->sgfn],
cg->fgfs[gxx0->sgfn], cg->fgfs[gxy0->sgfn], cg->fgfs[gxz0->sgfn],
cg->fgfs[gyy0->sgfn], cg->fgfs[gyz0->sgfn], cg->fgfs[gzz0->sgfn],
cg->fgfs[Axx0->sgfn], cg->fgfs[Axy0->sgfn], cg->fgfs[Axz0->sgfn],
cg->fgfs[Ayy0->sgfn], cg->fgfs[Ayz0->sgfn], cg->fgfs[Azz0->sgfn],
cg->fgfs[Gmx0->sgfn], cg->fgfs[Gmy0->sgfn], cg->fgfs[Gmz0->sgfn],
cg->fgfs[Lap0->sgfn],
cg->fgfs[Sfx0->sgfn], cg->fgfs[Sfy0->sgfn], cg->fgfs[Sfz0->sgfn],
cg->fgfs[dtSfx0->sgfn], cg->fgfs[dtSfy0->sgfn], cg->fgfs[dtSfz0->sgfn],
cg->fgfs[phi_rhs->sgfn], cg->fgfs[trK_rhs->sgfn],
cg->fgfs[gxx_rhs->sgfn], cg->fgfs[gxy_rhs->sgfn], cg->fgfs[gxz_rhs->sgfn],
cg->fgfs[gyy_rhs->sgfn], cg->fgfs[gyz_rhs->sgfn], cg->fgfs[gzz_rhs->sgfn],
cg->fgfs[Axx_rhs->sgfn], cg->fgfs[Axy_rhs->sgfn], cg->fgfs[Axz_rhs->sgfn],
cg->fgfs[Ayy_rhs->sgfn], cg->fgfs[Ayz_rhs->sgfn], cg->fgfs[Azz_rhs->sgfn],
cg->fgfs[Gmx_rhs->sgfn], cg->fgfs[Gmy_rhs->sgfn], cg->fgfs[Gmz_rhs->sgfn],
cg->fgfs[Lap_rhs->sgfn],
cg->fgfs[Sfx_rhs->sgfn], cg->fgfs[Sfy_rhs->sgfn], cg->fgfs[Sfz_rhs->sgfn],
cg->fgfs[dtSfx_rhs->sgfn], cg->fgfs[dtSfy_rhs->sgfn], cg->fgfs[dtSfz_rhs->sgfn],
cg->fgfs[rho->sgfn], cg->fgfs[Sx->sgfn], cg->fgfs[Sy->sgfn], cg->fgfs[Sz->sgfn],
cg->fgfs[Sxx->sgfn], cg->fgfs[Sxy->sgfn], cg->fgfs[Sxz->sgfn],
cg->fgfs[Syy->sgfn], cg->fgfs[Syz->sgfn], cg->fgfs[Szz->sgfn],
cg->fgfs[Gamxxx->sgfn], cg->fgfs[Gamxxy->sgfn], cg->fgfs[Gamxxz->sgfn],
cg->fgfs[Gamxyy->sgfn], cg->fgfs[Gamxyz->sgfn], cg->fgfs[Gamxzz->sgfn],
cg->fgfs[Gamyxx->sgfn], cg->fgfs[Gamyxy->sgfn], cg->fgfs[Gamyxz->sgfn],
cg->fgfs[Gamyyy->sgfn], cg->fgfs[Gamyyz->sgfn], cg->fgfs[Gamyzz->sgfn],
cg->fgfs[Gamzxx->sgfn], cg->fgfs[Gamzxy->sgfn], cg->fgfs[Gamzxz->sgfn],
cg->fgfs[Gamzyy->sgfn], cg->fgfs[Gamzyz->sgfn], cg->fgfs[Gamzzz->sgfn],
cg->fgfs[Rxx->sgfn], cg->fgfs[Rxy->sgfn], cg->fgfs[Rxz->sgfn],
cg->fgfs[Ryy->sgfn], cg->fgfs[Ryz->sgfn], cg->fgfs[Rzz->sgfn],
cg->fgfs[Cons_Ham->sgfn],
cg->fgfs[Cons_Px->sgfn], cg->fgfs[Cons_Py->sgfn], cg->fgfs[Cons_Pz->sgfn],
cg->fgfs[Cons_Gx->sgfn], cg->fgfs[Cons_Gy->sgfn], cg->fgfs[Cons_Gz->sgfn],
Symmetry, lev, ndeps, pre))
{
cout << "find NaN in domain: ("
<< cg->bbox[0] << ":" << cg->bbox[3] << ","
<< cg->bbox[1] << ":" << cg->bbox[4] << ","
<< cg->bbox[2] << ":" << cg->bbox[5] << ")" << endl;
ERROR = 1;
}
// rk4 substep and boundary
{
MyList<var> *varl0 = StateList, *varl = SynchList_pre, *varlrhs = RHSList;
// we do not check the correspondence here
while (varl0)
{
#ifndef WithShell
if (lev == 0) // sommerfeld indeed
f_sommerfeld_routbam(cg->shape, cg->X[0], cg->X[1], cg->X[2],
Pp->data->bbox[0], Pp->data->bbox[1], Pp->data->bbox[2],
Pp->data->bbox[3], Pp->data->bbox[4], Pp->data->bbox[5],
cg->fgfs[varlrhs->data->sgfn],
cg->fgfs[varl0->data->sgfn], varl0->data->propspeed, varl0->data->SoA,
Symmetry);
#endif
f_icn_rout(cg->shape, dT_lev,
cg->fgfs[varl0->data->sgfn],
cg->fgfs[varl->data->sgfn],
cg->fgfs[varlrhs->data->sgfn],
iter_count);
#ifndef WithShell
if (lev > 0) // fix BD point
#endif
f_sommerfeld_rout(cg->shape, cg->X[0], cg->X[1], cg->X[2],
Pp->data->bbox[0], Pp->data->bbox[1], Pp->data->bbox[2],
Pp->data->bbox[3], Pp->data->bbox[4], Pp->data->bbox[5],
dT_lev,
cg->fgfs[phi0->sgfn],
cg->fgfs[Lap0->sgfn],
cg->fgfs[varl0->data->sgfn], cg->fgfs[varl->data->sgfn],
varl0->data->SoA,
Symmetry, cor);
varl0 = varl0->next;
varl = varl->next;
varlrhs = varlrhs->next;
}
}
f_lowerboundset(cg->shape, cg->fgfs[phi->sgfn], chitiny);
}
if (BP == Pp->data->ble)
break;
BP = BP->next;
}
Pp = Pp->next;
}
// NOTE: error check deferred to after Shell Patch computation to reduce MPI_Allreduce calls
#ifdef WithShell
// evolve Shell Patches
if (lev == 0)
{
sPp = SH->PatL;
while (sPp)
{
MyList<Block> *BP = sPp->data->blb;
int fngfs = sPp->data->fngfs;
while (BP)
{
Block *cg = BP->data;
if (myrank == cg->rank)
{
#if (AGM == 0)
f_enforce_ga(cg->shape,
cg->fgfs[gxx0->sgfn], cg->fgfs[gxy0->sgfn], cg->fgfs[gxz0->sgfn],
cg->fgfs[gyy0->sgfn], cg->fgfs[gyz0->sgfn], cg->fgfs[gzz0->sgfn],
cg->fgfs[Axx0->sgfn], cg->fgfs[Axy0->sgfn], cg->fgfs[Axz0->sgfn],
cg->fgfs[Ayy0->sgfn], cg->fgfs[Ayz0->sgfn], cg->fgfs[Azz0->sgfn]);
#endif
if (f_compute_rhs_bssn_ss(cg->shape, TRK4, cg->X[0], cg->X[1], cg->X[2],
cg->fgfs[fngfs + ShellPatch::gx],
cg->fgfs[fngfs + ShellPatch::gy],
cg->fgfs[fngfs + ShellPatch::gz],
cg->fgfs[fngfs + ShellPatch::drhodx],
cg->fgfs[fngfs + ShellPatch::drhody],
cg->fgfs[fngfs + ShellPatch::drhodz],
cg->fgfs[fngfs + ShellPatch::dsigmadx],
cg->fgfs[fngfs + ShellPatch::dsigmady],
cg->fgfs[fngfs + ShellPatch::dsigmadz],
cg->fgfs[fngfs + ShellPatch::dRdx],
cg->fgfs[fngfs + ShellPatch::dRdy],
cg->fgfs[fngfs + ShellPatch::dRdz],
cg->fgfs[fngfs + ShellPatch::drhodxx],
cg->fgfs[fngfs + ShellPatch::drhodxy],
cg->fgfs[fngfs + ShellPatch::drhodxz],
cg->fgfs[fngfs + ShellPatch::drhodyy],
cg->fgfs[fngfs + ShellPatch::drhodyz],
cg->fgfs[fngfs + ShellPatch::drhodzz],
cg->fgfs[fngfs + ShellPatch::dsigmadxx],
cg->fgfs[fngfs + ShellPatch::dsigmadxy],
cg->fgfs[fngfs + ShellPatch::dsigmadxz],
cg->fgfs[fngfs + ShellPatch::dsigmadyy],
cg->fgfs[fngfs + ShellPatch::dsigmadyz],
cg->fgfs[fngfs + ShellPatch::dsigmadzz],
cg->fgfs[fngfs + ShellPatch::dRdxx],
cg->fgfs[fngfs + ShellPatch::dRdxy],
cg->fgfs[fngfs + ShellPatch::dRdxz],
cg->fgfs[fngfs + ShellPatch::dRdyy],
cg->fgfs[fngfs + ShellPatch::dRdyz],
cg->fgfs[fngfs + ShellPatch::dRdzz],
cg->fgfs[phi0->sgfn], cg->fgfs[trK0->sgfn],
cg->fgfs[gxx0->sgfn], cg->fgfs[gxy0->sgfn], cg->fgfs[gxz0->sgfn],
cg->fgfs[gyy0->sgfn], cg->fgfs[gyz0->sgfn], cg->fgfs[gzz0->sgfn],
cg->fgfs[Axx0->sgfn], cg->fgfs[Axy0->sgfn], cg->fgfs[Axz0->sgfn],
cg->fgfs[Ayy0->sgfn], cg->fgfs[Ayz0->sgfn], cg->fgfs[Azz0->sgfn],
cg->fgfs[Gmx0->sgfn], cg->fgfs[Gmy0->sgfn], cg->fgfs[Gmz0->sgfn],
cg->fgfs[Lap0->sgfn],
cg->fgfs[Sfx0->sgfn], cg->fgfs[Sfy0->sgfn], cg->fgfs[Sfz0->sgfn],
cg->fgfs[dtSfx0->sgfn], cg->fgfs[dtSfy0->sgfn], cg->fgfs[dtSfz0->sgfn],
cg->fgfs[phi_rhs->sgfn], cg->fgfs[trK_rhs->sgfn],
cg->fgfs[gxx_rhs->sgfn], cg->fgfs[gxy_rhs->sgfn], cg->fgfs[gxz_rhs->sgfn],
cg->fgfs[gyy_rhs->sgfn], cg->fgfs[gyz_rhs->sgfn], cg->fgfs[gzz_rhs->sgfn],
cg->fgfs[Axx_rhs->sgfn], cg->fgfs[Axy_rhs->sgfn], cg->fgfs[Axz_rhs->sgfn],
cg->fgfs[Ayy_rhs->sgfn], cg->fgfs[Ayz_rhs->sgfn], cg->fgfs[Azz_rhs->sgfn],
cg->fgfs[Gmx_rhs->sgfn], cg->fgfs[Gmy_rhs->sgfn], cg->fgfs[Gmz_rhs->sgfn],
cg->fgfs[Lap_rhs->sgfn],
cg->fgfs[Sfx_rhs->sgfn], cg->fgfs[Sfy_rhs->sgfn], cg->fgfs[Sfz_rhs->sgfn],
cg->fgfs[dtSfx_rhs->sgfn], cg->fgfs[dtSfy_rhs->sgfn], cg->fgfs[dtSfz_rhs->sgfn],
cg->fgfs[rho->sgfn], cg->fgfs[Sx->sgfn], cg->fgfs[Sy->sgfn], cg->fgfs[Sz->sgfn],
cg->fgfs[Sxx->sgfn], cg->fgfs[Sxy->sgfn], cg->fgfs[Sxz->sgfn],
cg->fgfs[Syy->sgfn], cg->fgfs[Syz->sgfn], cg->fgfs[Szz->sgfn],
cg->fgfs[Gamxxx->sgfn], cg->fgfs[Gamxxy->sgfn], cg->fgfs[Gamxxz->sgfn],
cg->fgfs[Gamxyy->sgfn], cg->fgfs[Gamxyz->sgfn], cg->fgfs[Gamxzz->sgfn],
cg->fgfs[Gamyxx->sgfn], cg->fgfs[Gamyxy->sgfn], cg->fgfs[Gamyxz->sgfn],
cg->fgfs[Gamyyy->sgfn], cg->fgfs[Gamyyz->sgfn], cg->fgfs[Gamyzz->sgfn],
cg->fgfs[Gamzxx->sgfn], cg->fgfs[Gamzxy->sgfn], cg->fgfs[Gamzxz->sgfn],
cg->fgfs[Gamzyy->sgfn], cg->fgfs[Gamzyz->sgfn], cg->fgfs[Gamzzz->sgfn],
cg->fgfs[Rxx->sgfn], cg->fgfs[Rxy->sgfn], cg->fgfs[Rxz->sgfn],
cg->fgfs[Ryy->sgfn], cg->fgfs[Ryz->sgfn], cg->fgfs[Rzz->sgfn],
cg->fgfs[Cons_Ham->sgfn],
cg->fgfs[Cons_Px->sgfn], cg->fgfs[Cons_Py->sgfn], cg->fgfs[Cons_Pz->sgfn],
cg->fgfs[Cons_Gx->sgfn], cg->fgfs[Cons_Gy->sgfn], cg->fgfs[Cons_Gz->sgfn],
Symmetry, lev, numepsh, sPp->data->sst, pre))
{
cout << "find NaN in Shell domain: sst = " << sPp->data->sst << ", ("
<< cg->bbox[0] << ":" << cg->bbox[3] << ","
<< cg->bbox[1] << ":" << cg->bbox[4] << ","
<< cg->bbox[2] << ":" << cg->bbox[5] << ")" << endl;
ERROR = 1;
}
// rk4 substep and boundary
{
MyList<var> *varl0 = StateList, *varl = SynchList_pre, *varlrhs = RHSList; // we do not check the correspondence here
while (varl0)
{
// sommerfeld indeed for outter boudary while fix BD for inner boundary
f_sommerfeld_routbam_ss(cg->shape, cg->X[0], cg->X[1], cg->X[2],
sPp->data->bbox[0], sPp->data->bbox[1], sPp->data->bbox[2],
sPp->data->bbox[3], sPp->data->bbox[4], sPp->data->bbox[5],
cg->fgfs[varlrhs->data->sgfn],
cg->fgfs[varl0->data->sgfn],
varl0->data->propspeed, varl0->data->SoA,
Symmetry);
f_icn_rout(cg->shape, dT_lev,
cg->fgfs[varl0->data->sgfn],
cg->fgfs[varl->data->sgfn],
cg->fgfs[varlrhs->data->sgfn],
iter_count);
varl0 = varl0->next;
varl = varl->next;
varlrhs = varlrhs->next;
}
}
f_lowerboundset(cg->shape, cg->fgfs[phi->sgfn], chitiny);
}
if (BP == sPp->data->ble)
break;
BP = BP->next;
}
sPp = sPp->next;
}
#if 0
// check rhs
{
SH->Dump_Data(RHSList,0,PhysTime,dT_lev);
if(myrank == 0)
{
cout<<"check rhs"<<endl;
MPI_Abort(MPI_COMM_WORLD,1);
}
}
#endif
}
// Non-blocking error reduction overlapped with Sync to hide Allreduce latency
MPI_Request err_req;
{
int erh = ERROR;
MPI_Iallreduce(&erh, &ERROR, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD, &err_req);
}
#endif
Parallel::AsyncSyncState async_pre;
Parallel::Sync_start(GH->PatL[lev], SynchList_pre, Symmetry, sync_cache_pre[lev], async_pre);
#ifdef WithShell
if (lev == 0)
{
clock_t prev_clock, curr_clock;
if (myrank == 0)
curr_clock = clock();
SH->Synch(SynchList_pre, Symmetry);
if (myrank == 0)
{
prev_clock = curr_clock;
curr_clock = clock();
cout << " Shell stuff synchronization used "
<< (double)(curr_clock - prev_clock) / ((double)CLOCKS_PER_SEC)
<< " seconds! " << endl;
}
}
#endif
Parallel::Sync_finish(sync_cache_pre[lev], async_pre, SynchList_pre, Symmetry);
#ifdef WithShell
// Complete non-blocking error reduction and check
MPI_Wait(&err_req, MPI_STATUS_IGNORE);
if (ERROR)
{
Parallel::Dump_Data(GH->PatL[lev], StateList, 0, PhysTime, dT_lev);
SH->Dump_Data(StateList, 0, PhysTime, dT_lev);
if (myrank == 0)
{
if (ErrorMonitor->outfile)
ErrorMonitor->outfile << "find NaN in state variables at t = " << PhysTime
<< ", lev = " << lev << endl;
MPI_Abort(MPI_COMM_WORLD, 1);
}
}
#endif
// for black hole position
if (BH_num > 0 && lev == GH->levels - 1)
{
compute_Porg_rhs(Porg0, Porg_rhs, Sfx0, Sfy0, Sfz0, lev);
for (int ithBH = 0; ithBH < BH_num; ithBH++)
{
f_icn_scalar(dT_lev, Porg0[ithBH][0], Porg[ithBH][0], Porg_rhs[ithBH][0], iter_count);
f_icn_scalar(dT_lev, Porg0[ithBH][1], Porg[ithBH][1], Porg_rhs[ithBH][1], iter_count);
f_icn_scalar(dT_lev, Porg0[ithBH][2], Porg[ithBH][2], Porg_rhs[ithBH][2], iter_count);
if (Symmetry > 0)
Porg[ithBH][2] = fabs(Porg[ithBH][2]);
if (Symmetry == 2)
{
Porg[ithBH][0] = fabs(Porg[ithBH][0]);
Porg[ithBH][1] = fabs(Porg[ithBH][1]);
}
if (!finite(Porg[ithBH][0]) || !finite(Porg[ithBH][1]) || !finite(Porg[ithBH][2]))
{
if (ErrorMonitor->outfile)
ErrorMonitor->outfile << "predictor step finds NaN for BH's position from ("
<< Porg0[ithBH][0] << "," << Porg0[ithBH][1] << "," << Porg0[ithBH][2]
<< ")" << endl;
MyList<var> *DG_List = new MyList<var>(Sfx0);
DG_List->insert(Sfx0);
DG_List->insert(Sfy0);
DG_List->insert(Sfz0);
Parallel::Dump_Data(GH->PatL[lev], DG_List, 0, PhysTime, dT_lev);
DG_List->clearList();
}
}
}
// data analysis part
// Warning NOTE: the variables1 are used as temp storege room
if (lev == a_lev)
{
AnalysisStuff(lev, dT_lev);
}
// corrector
for (iter_count = 1; iter_count < 3; iter_count++)
{
Pp = GH->PatL[lev];
while (Pp)
{
MyList<Block> *BP = Pp->data->blb;
while (BP)
{
Block *cg = BP->data;
if (myrank == cg->rank)
{
#if (AGM == 0)
f_enforce_ga(cg->shape,
cg->fgfs[gxx->sgfn], cg->fgfs[gxy->sgfn], cg->fgfs[gxz->sgfn],
cg->fgfs[gyy->sgfn], cg->fgfs[gyz->sgfn], cg->fgfs[gzz->sgfn],
cg->fgfs[Axx->sgfn], cg->fgfs[Axy->sgfn], cg->fgfs[Axz->sgfn],
cg->fgfs[Ayy->sgfn], cg->fgfs[Ayz->sgfn], cg->fgfs[Azz->sgfn]);
#elif (AGM == 1)
if (iter_count == 3)
f_enforce_ga(cg->shape,
cg->fgfs[gxx->sgfn], cg->fgfs[gxy->sgfn], cg->fgfs[gxz->sgfn],
cg->fgfs[gyy->sgfn], cg->fgfs[gyz->sgfn], cg->fgfs[gzz->sgfn],
cg->fgfs[Axx->sgfn], cg->fgfs[Axy->sgfn], cg->fgfs[Axz->sgfn],
cg->fgfs[Ayy->sgfn], cg->fgfs[Ayz->sgfn], cg->fgfs[Azz->sgfn]);
#endif
if (f_compute_rhs_bssn(cg->shape, TRK4, cg->X[0], cg->X[1], cg->X[2],
cg->fgfs[phi->sgfn], cg->fgfs[trK->sgfn],
cg->fgfs[gxx->sgfn], cg->fgfs[gxy->sgfn], cg->fgfs[gxz->sgfn],
cg->fgfs[gyy->sgfn], cg->fgfs[gyz->sgfn], cg->fgfs[gzz->sgfn],
cg->fgfs[Axx->sgfn], cg->fgfs[Axy->sgfn], cg->fgfs[Axz->sgfn],
cg->fgfs[Ayy->sgfn], cg->fgfs[Ayz->sgfn], cg->fgfs[Azz->sgfn],
cg->fgfs[Gmx->sgfn], cg->fgfs[Gmy->sgfn], cg->fgfs[Gmz->sgfn],
cg->fgfs[Lap->sgfn],
cg->fgfs[Sfx->sgfn], cg->fgfs[Sfy->sgfn], cg->fgfs[Sfz->sgfn],
cg->fgfs[dtSfx->sgfn], cg->fgfs[dtSfy->sgfn], cg->fgfs[dtSfz->sgfn],
cg->fgfs[phi1->sgfn], cg->fgfs[trK1->sgfn],
cg->fgfs[gxx1->sgfn], cg->fgfs[gxy1->sgfn], cg->fgfs[gxz1->sgfn],
cg->fgfs[gyy1->sgfn], cg->fgfs[gyz1->sgfn], cg->fgfs[gzz1->sgfn],
cg->fgfs[Axx1->sgfn], cg->fgfs[Axy1->sgfn], cg->fgfs[Axz1->sgfn],
cg->fgfs[Ayy1->sgfn], cg->fgfs[Ayz1->sgfn], cg->fgfs[Azz1->sgfn],
cg->fgfs[Gmx1->sgfn], cg->fgfs[Gmy1->sgfn], cg->fgfs[Gmz1->sgfn],
cg->fgfs[Lap1->sgfn],
cg->fgfs[Sfx1->sgfn], cg->fgfs[Sfy1->sgfn], cg->fgfs[Sfz1->sgfn],
cg->fgfs[dtSfx1->sgfn], cg->fgfs[dtSfy1->sgfn], cg->fgfs[dtSfz1->sgfn],
cg->fgfs[rho->sgfn],
cg->fgfs[Sx->sgfn], cg->fgfs[Sy->sgfn], cg->fgfs[Sz->sgfn],
cg->fgfs[Sxx->sgfn], cg->fgfs[Sxy->sgfn], cg->fgfs[Sxz->sgfn],
cg->fgfs[Syy->sgfn], cg->fgfs[Syz->sgfn], cg->fgfs[Szz->sgfn],
cg->fgfs[Gamxxx->sgfn], cg->fgfs[Gamxxy->sgfn], cg->fgfs[Gamxxz->sgfn],
cg->fgfs[Gamxyy->sgfn], cg->fgfs[Gamxyz->sgfn], cg->fgfs[Gamxzz->sgfn],
cg->fgfs[Gamyxx->sgfn], cg->fgfs[Gamyxy->sgfn], cg->fgfs[Gamyxz->sgfn],
cg->fgfs[Gamyyy->sgfn], cg->fgfs[Gamyyz->sgfn], cg->fgfs[Gamyzz->sgfn],
cg->fgfs[Gamzxx->sgfn], cg->fgfs[Gamzxy->sgfn], cg->fgfs[Gamzxz->sgfn],
cg->fgfs[Gamzyy->sgfn], cg->fgfs[Gamzyz->sgfn], cg->fgfs[Gamzzz->sgfn],
cg->fgfs[Rxx->sgfn], cg->fgfs[Rxy->sgfn], cg->fgfs[Rxz->sgfn],
cg->fgfs[Ryy->sgfn], cg->fgfs[Ryz->sgfn], cg->fgfs[Rzz->sgfn],
cg->fgfs[Cons_Ham->sgfn],
cg->fgfs[Cons_Px->sgfn], cg->fgfs[Cons_Py->sgfn], cg->fgfs[Cons_Pz->sgfn],
cg->fgfs[Cons_Gx->sgfn], cg->fgfs[Cons_Gy->sgfn], cg->fgfs[Cons_Gz->sgfn],
Symmetry, lev, ndeps, cor))
{
cout << "find NaN in domain: ("
<< cg->bbox[0] << ":" << cg->bbox[3] << ","
<< cg->bbox[1] << ":" << cg->bbox[4] << ","
<< cg->bbox[2] << ":" << cg->bbox[5] << ")" << endl;
ERROR = 1;
}
// rk4 substep and boundary
{
MyList<var> *varl0 = StateList, *varl = SynchList_pre, *varl1 = SynchList_cor, *varlrhs = RHSList;
// we do not check the correspondence here
while (varl0)
{
#ifndef WithShell
if (lev == 0) // sommerfeld indeed
f_sommerfeld_routbam(cg->shape, cg->X[0], cg->X[1], cg->X[2],
Pp->data->bbox[0], Pp->data->bbox[1], Pp->data->bbox[2],
Pp->data->bbox[3], Pp->data->bbox[4], Pp->data->bbox[5],
cg->fgfs[varl1->data->sgfn],
cg->fgfs[varl->data->sgfn],
varl0->data->propspeed, varl0->data->SoA,
Symmetry);
#endif
f_icn_rout(cg->shape, dT_lev,
cg->fgfs[varl0->data->sgfn],
cg->fgfs[varl1->data->sgfn],
cg->fgfs[varlrhs->data->sgfn],
iter_count);
#ifndef WithShell
if (lev > 0) // fix BD point
#endif
f_sommerfeld_rout(cg->shape, cg->X[0], cg->X[1], cg->X[2],
Pp->data->bbox[0], Pp->data->bbox[1], Pp->data->bbox[2],
Pp->data->bbox[3], Pp->data->bbox[4], Pp->data->bbox[5],
dT_lev,
cg->fgfs[phi0->sgfn],
cg->fgfs[Lap0->sgfn],
cg->fgfs[varl0->data->sgfn], cg->fgfs[varl1->data->sgfn],
varl0->data->SoA,
Symmetry, cor);
varl0 = varl0->next;
varl = varl->next;
varl1 = varl1->next;
varlrhs = varlrhs->next;
}
}
f_lowerboundset(cg->shape, cg->fgfs[phi1->sgfn], chitiny);
}
if (BP == Pp->data->ble)
break;
BP = BP->next;
}
Pp = Pp->next;
}
// NOTE: error check deferred to after Shell Patch computation to reduce MPI_Allreduce calls
#ifdef WithShell
// evolve Shell Patches
if (lev == 0)
{
sPp = SH->PatL;
while (sPp)
{
MyList<Block> *BP = sPp->data->blb;
int fngfs = sPp->data->fngfs;
while (BP)
{
Block *cg = BP->data;
if (myrank == cg->rank)
{
#if (AGM == 0)
f_enforce_ga(cg->shape,
cg->fgfs[gxx->sgfn], cg->fgfs[gxy->sgfn], cg->fgfs[gxz->sgfn],
cg->fgfs[gyy->sgfn], cg->fgfs[gyz->sgfn], cg->fgfs[gzz->sgfn],
cg->fgfs[Axx->sgfn], cg->fgfs[Axy->sgfn], cg->fgfs[Axz->sgfn],
cg->fgfs[Ayy->sgfn], cg->fgfs[Ayz->sgfn], cg->fgfs[Azz->sgfn]);
#elif (AGM == 1)
if (iter_count == 3)
f_enforce_ga(cg->shape,
cg->fgfs[gxx->sgfn], cg->fgfs[gxy->sgfn], cg->fgfs[gxz->sgfn],
cg->fgfs[gyy->sgfn], cg->fgfs[gyz->sgfn], cg->fgfs[gzz->sgfn],
cg->fgfs[Axx->sgfn], cg->fgfs[Axy->sgfn], cg->fgfs[Axz->sgfn],
cg->fgfs[Ayy->sgfn], cg->fgfs[Ayz->sgfn], cg->fgfs[Azz->sgfn]);
#endif
if (f_compute_rhs_bssn_ss(cg->shape, TRK4, cg->X[0], cg->X[1], cg->X[2],
cg->fgfs[fngfs + ShellPatch::gx],
cg->fgfs[fngfs + ShellPatch::gy],
cg->fgfs[fngfs + ShellPatch::gz],
cg->fgfs[fngfs + ShellPatch::drhodx],
cg->fgfs[fngfs + ShellPatch::drhody],
cg->fgfs[fngfs + ShellPatch::drhodz],
cg->fgfs[fngfs + ShellPatch::dsigmadx],
cg->fgfs[fngfs + ShellPatch::dsigmady],
cg->fgfs[fngfs + ShellPatch::dsigmadz],
cg->fgfs[fngfs + ShellPatch::dRdx],
cg->fgfs[fngfs + ShellPatch::dRdy],
cg->fgfs[fngfs + ShellPatch::dRdz],
cg->fgfs[fngfs + ShellPatch::drhodxx],
cg->fgfs[fngfs + ShellPatch::drhodxy],
cg->fgfs[fngfs + ShellPatch::drhodxz],
cg->fgfs[fngfs + ShellPatch::drhodyy],
cg->fgfs[fngfs + ShellPatch::drhodyz],
cg->fgfs[fngfs + ShellPatch::drhodzz],
cg->fgfs[fngfs + ShellPatch::dsigmadxx],
cg->fgfs[fngfs + ShellPatch::dsigmadxy],
cg->fgfs[fngfs + ShellPatch::dsigmadxz],
cg->fgfs[fngfs + ShellPatch::dsigmadyy],
cg->fgfs[fngfs + ShellPatch::dsigmadyz],
cg->fgfs[fngfs + ShellPatch::dsigmadzz],
cg->fgfs[fngfs + ShellPatch::dRdxx],
cg->fgfs[fngfs + ShellPatch::dRdxy],
cg->fgfs[fngfs + ShellPatch::dRdxz],
cg->fgfs[fngfs + ShellPatch::dRdyy],
cg->fgfs[fngfs + ShellPatch::dRdyz],
cg->fgfs[fngfs + ShellPatch::dRdzz],
cg->fgfs[phi->sgfn], cg->fgfs[trK->sgfn],
cg->fgfs[gxx->sgfn], cg->fgfs[gxy->sgfn], cg->fgfs[gxz->sgfn],
cg->fgfs[gyy->sgfn], cg->fgfs[gyz->sgfn], cg->fgfs[gzz->sgfn],
cg->fgfs[Axx->sgfn], cg->fgfs[Axy->sgfn], cg->fgfs[Axz->sgfn],
cg->fgfs[Ayy->sgfn], cg->fgfs[Ayz->sgfn], cg->fgfs[Azz->sgfn],
cg->fgfs[Gmx->sgfn], cg->fgfs[Gmy->sgfn], cg->fgfs[Gmz->sgfn],
cg->fgfs[Lap->sgfn],
cg->fgfs[Sfx->sgfn], cg->fgfs[Sfy->sgfn], cg->fgfs[Sfz->sgfn],
cg->fgfs[dtSfx->sgfn], cg->fgfs[dtSfy->sgfn], cg->fgfs[dtSfz->sgfn],
cg->fgfs[phi1->sgfn], cg->fgfs[trK1->sgfn],
cg->fgfs[gxx1->sgfn], cg->fgfs[gxy1->sgfn], cg->fgfs[gxz1->sgfn],
cg->fgfs[gyy1->sgfn], cg->fgfs[gyz1->sgfn], cg->fgfs[gzz1->sgfn],
cg->fgfs[Axx1->sgfn], cg->fgfs[Axy1->sgfn], cg->fgfs[Axz1->sgfn],
cg->fgfs[Ayy1->sgfn], cg->fgfs[Ayz1->sgfn], cg->fgfs[Azz1->sgfn],
cg->fgfs[Gmx1->sgfn], cg->fgfs[Gmy1->sgfn], cg->fgfs[Gmz1->sgfn],
cg->fgfs[Lap1->sgfn],
cg->fgfs[Sfx1->sgfn], cg->fgfs[Sfy1->sgfn], cg->fgfs[Sfz1->sgfn],
cg->fgfs[dtSfx1->sgfn], cg->fgfs[dtSfy1->sgfn], cg->fgfs[dtSfz1->sgfn],
cg->fgfs[rho->sgfn],
cg->fgfs[Sx->sgfn], cg->fgfs[Sy->sgfn], cg->fgfs[Sz->sgfn],
cg->fgfs[Sxx->sgfn], cg->fgfs[Sxy->sgfn], cg->fgfs[Sxz->sgfn],
cg->fgfs[Syy->sgfn], cg->fgfs[Syz->sgfn], cg->fgfs[Szz->sgfn],
cg->fgfs[Gamxxx->sgfn], cg->fgfs[Gamxxy->sgfn], cg->fgfs[Gamxxz->sgfn],
cg->fgfs[Gamxyy->sgfn], cg->fgfs[Gamxyz->sgfn], cg->fgfs[Gamxzz->sgfn],
cg->fgfs[Gamyxx->sgfn], cg->fgfs[Gamyxy->sgfn], cg->fgfs[Gamyxz->sgfn],
cg->fgfs[Gamyyy->sgfn], cg->fgfs[Gamyyz->sgfn], cg->fgfs[Gamyzz->sgfn],
cg->fgfs[Gamzxx->sgfn], cg->fgfs[Gamzxy->sgfn], cg->fgfs[Gamzxz->sgfn],
cg->fgfs[Gamzyy->sgfn], cg->fgfs[Gamzyz->sgfn], cg->fgfs[Gamzzz->sgfn],
cg->fgfs[Rxx->sgfn], cg->fgfs[Rxy->sgfn], cg->fgfs[Rxz->sgfn],
cg->fgfs[Ryy->sgfn], cg->fgfs[Ryz->sgfn], cg->fgfs[Rzz->sgfn],
cg->fgfs[Cons_Ham->sgfn],
cg->fgfs[Cons_Px->sgfn], cg->fgfs[Cons_Py->sgfn], cg->fgfs[Cons_Pz->sgfn],
cg->fgfs[Cons_Gx->sgfn], cg->fgfs[Cons_Gy->sgfn], cg->fgfs[Cons_Gz->sgfn],
Symmetry, lev, numepsh, sPp->data->sst, cor))
{
cout << "find NaN in Shell domain: sst = " << sPp->data->sst << ", ("
<< cg->bbox[0] << ":" << cg->bbox[3] << ","
<< cg->bbox[1] << ":" << cg->bbox[4] << ","
<< cg->bbox[2] << ":" << cg->bbox[5] << ")" << endl;
ERROR = 1;
}
// rk4 substep and boundary
{
MyList<var> *varl0 = StateList, *varl = SynchList_pre, *varl1 = SynchList_cor, *varlrhs = RHSList;
// we do not check the correspondence here
while (varl0)
{
// sommerfeld indeed for outter boudary while fix BD for inner boundary
f_sommerfeld_routbam_ss(cg->shape, cg->X[0], cg->X[1], cg->X[2],
sPp->data->bbox[0], sPp->data->bbox[1], sPp->data->bbox[2],
sPp->data->bbox[3], sPp->data->bbox[4], sPp->data->bbox[5],
cg->fgfs[varl1->data->sgfn],
cg->fgfs[varl->data->sgfn],
varl0->data->propspeed, varl0->data->SoA,
Symmetry);
f_rungekutta4_rout(cg->shape, dT_lev,
cg->fgfs[varl0->data->sgfn],
cg->fgfs[varl1->data->sgfn],
cg->fgfs[varlrhs->data->sgfn],
iter_count);
varl0 = varl0->next;
varl = varl->next;
varl1 = varl1->next;
varlrhs = varlrhs->next;
}
}
f_lowerboundset(cg->shape, cg->fgfs[phi1->sgfn], chitiny);
}
if (BP == sPp->data->ble)
break;
BP = BP->next;
}
sPp = sPp->next;
}
}
// Non-blocking error reduction overlapped with Sync to hide Allreduce latency
MPI_Request err_req_cor;
{
int erh = ERROR;
MPI_Iallreduce(&erh, &ERROR, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD, &err_req_cor);
}
#endif
Parallel::AsyncSyncState async_cor;
Parallel::Sync_start(GH->PatL[lev], SynchList_cor, Symmetry, sync_cache_cor[lev], async_cor);
#ifdef WithShell
if (lev == 0)
{
clock_t prev_clock, curr_clock;
if (myrank == 0)
curr_clock = clock();
SH->Synch(SynchList_cor, Symmetry);
if (myrank == 0)
{
prev_clock = curr_clock;
curr_clock = clock();
cout << " Shell stuff synchronization used "
<< (double)(curr_clock - prev_clock) / ((double)CLOCKS_PER_SEC)
<< " seconds! " << endl;
}
}
#endif
Parallel::Sync_finish(sync_cache_cor[lev], async_cor, SynchList_cor, Symmetry);
#ifdef WithShell
// Complete non-blocking error reduction and check
MPI_Wait(&err_req_cor, MPI_STATUS_IGNORE);
if (ERROR)
{
Parallel::Dump_Data(GH->PatL[lev], SynchList_pre, 0, PhysTime, dT_lev);
SH->Dump_Data(SynchList_pre, 0, PhysTime, dT_lev);
if (myrank == 0)
{
if (ErrorMonitor->outfile)
ErrorMonitor->outfile << "find NaN in RK4 substep#" << iter_count
<< " variables at t = " << PhysTime
<< ", lev = " << lev << endl;
MPI_Abort(MPI_COMM_WORLD, 1);
}
}
#endif
// for black hole position
if (BH_num > 0 && lev == GH->levels - 1)
{
compute_Porg_rhs(Porg, Porg1, Sfx, Sfy, Sfz, lev);
for (int ithBH = 0; ithBH < BH_num; ithBH++)
{
f_icn_scalar(dT_lev, Porg0[ithBH][0], Porg1[ithBH][0], Porg_rhs[ithBH][0], iter_count);
f_icn_scalar(dT_lev, Porg0[ithBH][1], Porg1[ithBH][1], Porg_rhs[ithBH][1], iter_count);
f_icn_scalar(dT_lev, Porg0[ithBH][2], Porg1[ithBH][2], Porg_rhs[ithBH][2], iter_count);
if (Symmetry > 0)
Porg1[ithBH][2] = fabs(Porg1[ithBH][2]);
if (Symmetry == 2)
{
Porg1[ithBH][0] = fabs(Porg1[ithBH][0]);
Porg1[ithBH][1] = fabs(Porg1[ithBH][1]);
}
if (!finite(Porg1[ithBH][0]) || !finite(Porg1[ithBH][1]) || !finite(Porg1[ithBH][2]))
{
if (ErrorMonitor->outfile)
ErrorMonitor->outfile << iter_count << " corrector step finds NaN for BH's position from ("
<< Porg[ithBH][0] << "," << Porg[ithBH][1] << "," << Porg[ithBH][2]
<< ")" << endl;
MyList<var> *DG_List = new MyList<var>(Sfx0);
DG_List->insert(Sfx0);
DG_List->insert(Sfy0);
DG_List->insert(Sfz0);
Parallel::Dump_Data(GH->PatL[lev], DG_List, 0, PhysTime, dT_lev);
DG_List->clearList();
}
}
}
// swap time level
if (iter_count < 3)
{
Pp = GH->PatL[lev];
while (Pp)
{
MyList<Block> *BP = Pp->data->blb;
while (BP)
{
Block *cg = BP->data;
cg->swapList(SynchList_pre, SynchList_cor, myrank);
if (BP == Pp->data->ble)
break;
BP = BP->next;
}
Pp = Pp->next;
}
#ifdef WithShell
if (lev == 0)
{
sPp = SH->PatL;
while (sPp)
{
MyList<Block> *BP = sPp->data->blb;
while (BP)
{
Block *cg = BP->data;
cg->swapList(SynchList_pre, SynchList_cor, myrank);
if (BP == sPp->data->ble)
break;
BP = BP->next;
}
sPp = sPp->next;
}
}
#endif
// for black hole position
if (BH_num > 0 && lev == GH->levels - 1)
{
for (int ithBH = 0; ithBH < BH_num; ithBH++)
{
Porg[ithBH][0] = Porg1[ithBH][0];
Porg[ithBH][1] = Porg1[ithBH][1];
Porg[ithBH][2] = Porg1[ithBH][2];
}
}
}
}
#if (RPS == 0)
// mesh refinement boundary part
RestrictProlong(lev, YN, BB);
#ifdef WithShell
if (lev == 0)
{
clock_t prev_clock, curr_clock;
if (myrank == 0)
curr_clock = clock();
SH->CS_Inter(SynchList_cor, Symmetry);
if (myrank == 0)
{
prev_clock = curr_clock;
curr_clock = clock();
cout << " CS_Inter used " << (double)(curr_clock - prev_clock) / ((double)CLOCKS_PER_SEC)
<< " seconds! " << endl;
}
}
#endif
#endif
// note the data structure before update
// SynchList_cor 1 -----------
//
// StateList 0 -----------
//
// OldStateList old -----------
// update
Pp = GH->PatL[lev];
while (Pp)
{
MyList<Block> *BP = Pp->data->blb;
while (BP)
{
Block *cg = BP->data;
cg->swapList(StateList, SynchList_cor, myrank);
cg->swapList(OldStateList, SynchList_cor, myrank);
if (BP == Pp->data->ble)
break;
BP = BP->next;
}
Pp = Pp->next;
}
#ifdef WithShell
if (lev == 0)
{
sPp = SH->PatL;
while (sPp)
{
MyList<Block> *BP = sPp->data->blb;
while (BP)
{
Block *cg = BP->data;
cg->swapList(StateList, SynchList_cor, myrank);
cg->swapList(OldStateList, SynchList_cor, myrank);
if (BP == sPp->data->ble)
break;
BP = BP->next;
}
sPp = sPp->next;
}
#if 0
// check StateList
{
SH->Dump_Data(StateList,0,PhysTime,dT_lev);
if(myrank == 0)
{
cout<<"check StateList"<<endl;
MPI_Abort(MPI_COMM_WORLD,1);
}
}
#endif
}
#endif
// for black hole position
if (BH_num > 0 && lev == GH->levels - 1)
{
for (int ithBH = 0; ithBH < BH_num; ithBH++)
{
Porg0[ithBH][0] = Porg1[ithBH][0];
Porg0[ithBH][1] = Porg1[ithBH][1];
Porg0[ithBH][2] = Porg1[ithBH][2];
}
}
}
#endif
//================================================================================================
//================================================================================================
// This member function implements single-step time evolution for each AMR level
// Variant for the case PSTR == 0
//================================================================================================
#elif (PSTR == 1 || PSTR == 2 || PSTR == 3)
void bssn_class::Step(int lev, int YN)
{
// misc::tillherecheck(GH->Commlev[lev],GH->start_rank[lev],"start Step");
setpbh(BH_num, Porg0, Mass, BH_num_input);
double dT_lev = dT * pow(0.5, Mymax(lev, trfls));
// new code 2013-2-15, zjcao
#if (MAPBH == 1)
// for black hole position
if (BH_num > 0 && lev == GH->levels - 1)
{
compute_Porg_rhs(Porg0, Porg_rhs, Sfx0, Sfy0, Sfz0, lev);
for (int ithBH = 0; ithBH < BH_num; ithBH++)
{
for (int ith = 0; ith < 3; ith++)
Porg1[ithBH][ith] = Porg0[ithBH][ith] + Porg_rhs[ithBH][ith] * dT_lev;
if (Symmetry > 0)
Porg1[ithBH][2] = fabs(Porg1[ithBH][2]);
if (Symmetry == 2)
{
Porg1[ithBH][0] = fabs(Porg1[ithBH][0]);
Porg1[ithBH][1] = fabs(Porg1[ithBH][1]);
}
if (!finite(Porg1[ithBH][0]) || !finite(Porg1[ithBH][1]) || !finite(Porg1[ithBH][2]))
{
if (ErrorMonitor->outfile)
ErrorMonitor->outfile << "predictor step finds NaN for BH's position from ("
<< Porg0[ithBH][0] << "," << Porg0[ithBH][1] << "," << Porg0[ithBH][2]
<< ")" << endl;
MyList<var> *DG_List = new MyList<var>(Sfx0);
DG_List->insert(Sfx0);
DG_List->insert(Sfy0);
DG_List->insert(Sfz0);
Parallel::Dump_Data(GH->PatL[lev], DG_List, 0, PhysTime, dT_lev);
DG_List->clearList();
}
}
}
#endif
// misc::tillherecheck(GH->Commlev[lev],GH->start_rank[lev],"before Predictor");
#ifdef With_AHF
AH_Step_Find(lev, dT_lev);
#endif
bool BB = fgt(PhysTime, StartTime, dT_lev / 2);
double ndeps = numepss;
if (lev < GH->movls)
ndeps = numepsb;
double TRK4 = PhysTime;
int iter_count = 0; // count RK4 substeps
int pre = 0, cor = 1;
int ERROR = 0;
MyList<ss_patch> *sPp;
// Predictor
MyList<Patch> *Pp = GH->PatL[lev];
while (Pp)
{
MyList<Block> *BP = Pp->data->blb;
while (BP)
{
Block *cg = BP->data;
if (myrank == cg->rank)
{
#if (AGM == 0)
f_enforce_ga(cg->shape,
cg->fgfs[gxx0->sgfn], cg->fgfs[gxy0->sgfn], cg->fgfs[gxz0->sgfn],
cg->fgfs[gyy0->sgfn], cg->fgfs[gyz0->sgfn], cg->fgfs[gzz0->sgfn],
cg->fgfs[Axx0->sgfn], cg->fgfs[Axy0->sgfn], cg->fgfs[Axz0->sgfn],
cg->fgfs[Ayy0->sgfn], cg->fgfs[Ayz0->sgfn], cg->fgfs[Azz0->sgfn]);
#endif
if (f_compute_rhs_bssn(cg->shape, TRK4, cg->X[0], cg->X[1], cg->X[2],
cg->fgfs[phi0->sgfn], cg->fgfs[trK0->sgfn],
cg->fgfs[gxx0->sgfn], cg->fgfs[gxy0->sgfn], cg->fgfs[gxz0->sgfn],
cg->fgfs[gyy0->sgfn], cg->fgfs[gyz0->sgfn], cg->fgfs[gzz0->sgfn],
cg->fgfs[Axx0->sgfn], cg->fgfs[Axy0->sgfn], cg->fgfs[Axz0->sgfn],
cg->fgfs[Ayy0->sgfn], cg->fgfs[Ayz0->sgfn], cg->fgfs[Azz0->sgfn],
cg->fgfs[Gmx0->sgfn], cg->fgfs[Gmy0->sgfn], cg->fgfs[Gmz0->sgfn],
cg->fgfs[Lap0->sgfn],
cg->fgfs[Sfx0->sgfn], cg->fgfs[Sfy0->sgfn], cg->fgfs[Sfz0->sgfn],
cg->fgfs[dtSfx0->sgfn], cg->fgfs[dtSfy0->sgfn], cg->fgfs[dtSfz0->sgfn],
cg->fgfs[phi_rhs->sgfn], cg->fgfs[trK_rhs->sgfn],
cg->fgfs[gxx_rhs->sgfn], cg->fgfs[gxy_rhs->sgfn], cg->fgfs[gxz_rhs->sgfn],
cg->fgfs[gyy_rhs->sgfn], cg->fgfs[gyz_rhs->sgfn], cg->fgfs[gzz_rhs->sgfn],
cg->fgfs[Axx_rhs->sgfn], cg->fgfs[Axy_rhs->sgfn], cg->fgfs[Axz_rhs->sgfn],
cg->fgfs[Ayy_rhs->sgfn], cg->fgfs[Ayz_rhs->sgfn], cg->fgfs[Azz_rhs->sgfn],
cg->fgfs[Gmx_rhs->sgfn], cg->fgfs[Gmy_rhs->sgfn], cg->fgfs[Gmz_rhs->sgfn],
cg->fgfs[Lap_rhs->sgfn],
cg->fgfs[Sfx_rhs->sgfn], cg->fgfs[Sfy_rhs->sgfn], cg->fgfs[Sfz_rhs->sgfn],
cg->fgfs[dtSfx_rhs->sgfn], cg->fgfs[dtSfy_rhs->sgfn], cg->fgfs[dtSfz_rhs->sgfn],
cg->fgfs[rho->sgfn], cg->fgfs[Sx->sgfn], cg->fgfs[Sy->sgfn], cg->fgfs[Sz->sgfn],
cg->fgfs[Sxx->sgfn], cg->fgfs[Sxy->sgfn], cg->fgfs[Sxz->sgfn],
cg->fgfs[Syy->sgfn], cg->fgfs[Syz->sgfn], cg->fgfs[Szz->sgfn],
cg->fgfs[Gamxxx->sgfn], cg->fgfs[Gamxxy->sgfn], cg->fgfs[Gamxxz->sgfn],
cg->fgfs[Gamxyy->sgfn], cg->fgfs[Gamxyz->sgfn], cg->fgfs[Gamxzz->sgfn],
cg->fgfs[Gamyxx->sgfn], cg->fgfs[Gamyxy->sgfn], cg->fgfs[Gamyxz->sgfn],
cg->fgfs[Gamyyy->sgfn], cg->fgfs[Gamyyz->sgfn], cg->fgfs[Gamyzz->sgfn],
cg->fgfs[Gamzxx->sgfn], cg->fgfs[Gamzxy->sgfn], cg->fgfs[Gamzxz->sgfn],
cg->fgfs[Gamzyy->sgfn], cg->fgfs[Gamzyz->sgfn], cg->fgfs[Gamzzz->sgfn],
cg->fgfs[Rxx->sgfn], cg->fgfs[Rxy->sgfn], cg->fgfs[Rxz->sgfn],
cg->fgfs[Ryy->sgfn], cg->fgfs[Ryz->sgfn], cg->fgfs[Rzz->sgfn],
cg->fgfs[Cons_Ham->sgfn],
cg->fgfs[Cons_Px->sgfn], cg->fgfs[Cons_Py->sgfn], cg->fgfs[Cons_Pz->sgfn],
cg->fgfs[Cons_Gx->sgfn], cg->fgfs[Cons_Gy->sgfn], cg->fgfs[Cons_Gz->sgfn],
Symmetry, lev, ndeps, pre))
{
cout << "find NaN in domain: ("
<< cg->bbox[0] << ":" << cg->bbox[3] << ","
<< cg->bbox[1] << ":" << cg->bbox[4] << ","
<< cg->bbox[2] << ":" << cg->bbox[5] << ")" << endl;
ERROR = 1;
}
// rk4 substep and boundary
{
MyList<var> *varl0 = StateList, *varl = SynchList_pre, *varlrhs = RHSList; // we do not check the correspondence here
while (varl0)
{
#if (SommerType == 0)
#ifndef WithShell
if (lev == 0) // sommerfeld indeed
f_sommerfeld_routbam(cg->shape, cg->X[0], cg->X[1], cg->X[2],
Pp->data->bbox[0], Pp->data->bbox[1], Pp->data->bbox[2],
Pp->data->bbox[3], Pp->data->bbox[4], Pp->data->bbox[5],
cg->fgfs[varlrhs->data->sgfn],
cg->fgfs[varl0->data->sgfn],
varl0->data->propspeed, varl0->data->SoA,
Symmetry);
#endif
#endif
f_rungekutta4_rout(cg->shape, dT_lev,
cg->fgfs[varl0->data->sgfn],
cg->fgfs[varl->data->sgfn],
cg->fgfs[varlrhs->data->sgfn],
iter_count);
#ifndef WithShell
if (lev > 0) // fix BD point
#endif
f_sommerfeld_rout(cg->shape, cg->X[0], cg->X[1], cg->X[2],
Pp->data->bbox[0], Pp->data->bbox[1], Pp->data->bbox[2],
Pp->data->bbox[3], Pp->data->bbox[4], Pp->data->bbox[5],
dT_lev,
cg->fgfs[phi0->sgfn],
cg->fgfs[Lap0->sgfn],
cg->fgfs[varl0->data->sgfn], cg->fgfs[varl->data->sgfn],
varl0->data->SoA,
Symmetry, cor);
#if (SommerType == 1)
#warning "shell part still bam type"
if (lev == 0) // Shibata type sommerfeld
f_sommerfeld_rout(cg->shape, cg->X[0], cg->X[1], cg->X[2],
Pp->data->bbox[0], Pp->data->bbox[1], Pp->data->bbox[2],
Pp->data->bbox[3], Pp->data->bbox[4], Pp->data->bbox[5],
dT_lev,
cg->fgfs[phi0->sgfn],
cg->fgfs[Lap0->sgfn],
cg->fgfs[varl0->data->sgfn], cg->fgfs[varl->data->sgfn],
varl0->data->SoA,
Symmetry, pre);
#endif
varl0 = varl0->next;
varl = varl->next;
varlrhs = varlrhs->next;
}
}
f_lowerboundset(cg->shape, cg->fgfs[phi->sgfn], chitiny);
}
if (BP == Pp->data->ble)
break;
BP = BP->next;
}
Pp = Pp->next;
}
// misc::tillherecheck(GH->Commlev[lev],GH->start_rank[lev],"after Predictor rhs calculation");
// Non-blocking error reduction overlapped with Sync to hide Allreduce latency
MPI_Request err_req;
{
int erh = ERROR;
MPI_Iallreduce(&erh, &ERROR, 1, MPI_INT, MPI_SUM, GH->Commlev[lev], &err_req);
}
// misc::tillherecheck(GH->Commlev[lev],GH->start_rank[lev],"before Predictor sync");
Parallel::Sync_cached(GH->PatL[lev], SynchList_pre, Symmetry, sync_cache_pre[lev]);
// Complete non-blocking error reduction and check
MPI_Wait(&err_req, MPI_STATUS_IGNORE);
if (ERROR)
{
Parallel::Dump_Data(GH->PatL[lev], StateList, 0, PhysTime, dT_lev);
if (myrank == 0)
{
if (ErrorMonitor->outfile)
ErrorMonitor->outfile << "find NaN in state variables at t = " << PhysTime << ", lev = " << lev << endl;
MPI_Abort(MPI_COMM_WORLD, 1);
}
}
#if (MAPBH == 0)
// for black hole position
if (BH_num > 0 && lev == GH->levels - 1)
{
compute_Porg_rhs(Porg0, Porg_rhs, Sfx0, Sfy0, Sfz0, lev);
for (int ithBH = 0; ithBH < BH_num; ithBH++)
{
f_rungekutta4_scalar(dT_lev, Porg0[ithBH][0], Porg[ithBH][0], Porg_rhs[ithBH][0], iter_count);
f_rungekutta4_scalar(dT_lev, Porg0[ithBH][1], Porg[ithBH][1], Porg_rhs[ithBH][1], iter_count);
f_rungekutta4_scalar(dT_lev, Porg0[ithBH][2], Porg[ithBH][2], Porg_rhs[ithBH][2], iter_count);
if (Symmetry > 0)
Porg[ithBH][2] = fabs(Porg[ithBH][2]);
if (Symmetry == 2)
{
Porg[ithBH][0] = fabs(Porg[ithBH][0]);
Porg[ithBH][1] = fabs(Porg[ithBH][1]);
}
if (!finite(Porg[ithBH][0]) || !finite(Porg[ithBH][1]) || !finite(Porg[ithBH][2]))
{
if (ErrorMonitor->outfile)
ErrorMonitor->outfile << "predictor step finds NaN for BH's position from ("
<< Porg0[ithBH][0] << "," << Porg0[ithBH][1] << "," << Porg0[ithBH][2] << ")" << endl;
MyList<var> *DG_List = new MyList<var>(Sfx0);
DG_List->insert(Sfx0);
DG_List->insert(Sfy0);
DG_List->insert(Sfz0);
Parallel::Dump_Data(GH->PatL[lev], DG_List, 0, PhysTime, dT_lev);
DG_List->clearList();
}
}
}
#endif
// misc::tillherecheck(GH->Commlev[lev],GH->start_rank[lev],"before Corrector");
// corrector
for (iter_count = 1; iter_count < 4; iter_count++)
{
// misc::tillherecheck(GH->Commlev[lev],GH->start_rank[lev],"head of Corrector");
// for RK4: t0, t0+dt/2, t0+dt/2, t0+dt;
if (iter_count == 1 || iter_count == 3)
TRK4 += dT_lev / 2;
Pp = GH->PatL[lev];
while (Pp)
{
MyList<Block> *BP = Pp->data->blb;
while (BP)
{
Block *cg = BP->data;
if (myrank == cg->rank)
{
#if (AGM == 0)
f_enforce_ga(cg->shape,
cg->fgfs[gxx->sgfn], cg->fgfs[gxy->sgfn], cg->fgfs[gxz->sgfn],
cg->fgfs[gyy->sgfn], cg->fgfs[gyz->sgfn], cg->fgfs[gzz->sgfn],
cg->fgfs[Axx->sgfn], cg->fgfs[Axy->sgfn], cg->fgfs[Axz->sgfn],
cg->fgfs[Ayy->sgfn], cg->fgfs[Ayz->sgfn], cg->fgfs[Azz->sgfn]);
#elif (AGM == 1)
if (iter_count == 3)
f_enforce_ga(cg->shape,
cg->fgfs[gxx->sgfn], cg->fgfs[gxy->sgfn], cg->fgfs[gxz->sgfn],
cg->fgfs[gyy->sgfn], cg->fgfs[gyz->sgfn], cg->fgfs[gzz->sgfn],
cg->fgfs[Axx->sgfn], cg->fgfs[Axy->sgfn], cg->fgfs[Axz->sgfn],
cg->fgfs[Ayy->sgfn], cg->fgfs[Ayz->sgfn], cg->fgfs[Azz->sgfn]);
#endif
if (f_compute_rhs_bssn(cg->shape, TRK4, cg->X[0], cg->X[1], cg->X[2],
cg->fgfs[phi->sgfn], cg->fgfs[trK->sgfn],
cg->fgfs[gxx->sgfn], cg->fgfs[gxy->sgfn], cg->fgfs[gxz->sgfn],
cg->fgfs[gyy->sgfn], cg->fgfs[gyz->sgfn], cg->fgfs[gzz->sgfn],
cg->fgfs[Axx->sgfn], cg->fgfs[Axy->sgfn], cg->fgfs[Axz->sgfn],
cg->fgfs[Ayy->sgfn], cg->fgfs[Ayz->sgfn], cg->fgfs[Azz->sgfn],
cg->fgfs[Gmx->sgfn], cg->fgfs[Gmy->sgfn], cg->fgfs[Gmz->sgfn],
cg->fgfs[Lap->sgfn],
cg->fgfs[Sfx->sgfn], cg->fgfs[Sfy->sgfn], cg->fgfs[Sfz->sgfn],
cg->fgfs[dtSfx->sgfn], cg->fgfs[dtSfy->sgfn], cg->fgfs[dtSfz->sgfn],
cg->fgfs[phi1->sgfn], cg->fgfs[trK1->sgfn],
cg->fgfs[gxx1->sgfn], cg->fgfs[gxy1->sgfn], cg->fgfs[gxz1->sgfn],
cg->fgfs[gyy1->sgfn], cg->fgfs[gyz1->sgfn], cg->fgfs[gzz1->sgfn],
cg->fgfs[Axx1->sgfn], cg->fgfs[Axy1->sgfn], cg->fgfs[Axz1->sgfn],
cg->fgfs[Ayy1->sgfn], cg->fgfs[Ayz1->sgfn], cg->fgfs[Azz1->sgfn],
cg->fgfs[Gmx1->sgfn], cg->fgfs[Gmy1->sgfn], cg->fgfs[Gmz1->sgfn],
cg->fgfs[Lap1->sgfn],
cg->fgfs[Sfx1->sgfn], cg->fgfs[Sfy1->sgfn], cg->fgfs[Sfz1->sgfn],
cg->fgfs[dtSfx1->sgfn], cg->fgfs[dtSfy1->sgfn], cg->fgfs[dtSfz1->sgfn],
cg->fgfs[rho->sgfn],
cg->fgfs[Sx->sgfn], cg->fgfs[Sy->sgfn], cg->fgfs[Sz->sgfn],
cg->fgfs[Sxx->sgfn], cg->fgfs[Sxy->sgfn], cg->fgfs[Sxz->sgfn],
cg->fgfs[Syy->sgfn], cg->fgfs[Syz->sgfn], cg->fgfs[Szz->sgfn],
cg->fgfs[Gamxxx->sgfn], cg->fgfs[Gamxxy->sgfn], cg->fgfs[Gamxxz->sgfn],
cg->fgfs[Gamxyy->sgfn], cg->fgfs[Gamxyz->sgfn], cg->fgfs[Gamxzz->sgfn],
cg->fgfs[Gamyxx->sgfn], cg->fgfs[Gamyxy->sgfn], cg->fgfs[Gamyxz->sgfn],
cg->fgfs[Gamyyy->sgfn], cg->fgfs[Gamyyz->sgfn], cg->fgfs[Gamyzz->sgfn],
cg->fgfs[Gamzxx->sgfn], cg->fgfs[Gamzxy->sgfn], cg->fgfs[Gamzxz->sgfn],
cg->fgfs[Gamzyy->sgfn], cg->fgfs[Gamzyz->sgfn], cg->fgfs[Gamzzz->sgfn],
cg->fgfs[Rxx->sgfn], cg->fgfs[Rxy->sgfn], cg->fgfs[Rxz->sgfn],
cg->fgfs[Ryy->sgfn], cg->fgfs[Ryz->sgfn], cg->fgfs[Rzz->sgfn],
cg->fgfs[Cons_Ham->sgfn],
cg->fgfs[Cons_Px->sgfn], cg->fgfs[Cons_Py->sgfn], cg->fgfs[Cons_Pz->sgfn],
cg->fgfs[Cons_Gx->sgfn], cg->fgfs[Cons_Gy->sgfn], cg->fgfs[Cons_Gz->sgfn],
Symmetry, lev, ndeps, cor))
{
cout << "find NaN in domain: ("
<< cg->bbox[0] << ":" << cg->bbox[3] << ","
<< cg->bbox[1] << ":" << cg->bbox[4] << ","
<< cg->bbox[2] << ":" << cg->bbox[5] << ")" << endl;
ERROR = 1;
}
// rk4 substep and boundary
{
MyList<var> *varl0 = StateList, *varl = SynchList_pre, *varl1 = SynchList_cor, *varlrhs = RHSList;
// we do not check the correspondence here
while (varl0)
{
#if (SommerType == 0)
#ifndef WithShell
if (lev == 0) // sommerfeld indeed
f_sommerfeld_routbam(cg->shape, cg->X[0], cg->X[1], cg->X[2],
Pp->data->bbox[0], Pp->data->bbox[1], Pp->data->bbox[2],
Pp->data->bbox[3], Pp->data->bbox[4], Pp->data->bbox[5],
cg->fgfs[varl1->data->sgfn],
cg->fgfs[varl->data->sgfn],
varl0->data->propspeed, varl0->data->SoA,
Symmetry);
#endif
#endif
f_rungekutta4_rout(cg->shape, dT_lev,
cg->fgfs[varl0->data->sgfn],
cg->fgfs[varl1->data->sgfn],
cg->fgfs[varlrhs->data->sgfn],
iter_count);
#ifndef WithShell
if (lev > 0) // fix BD point
#endif
f_sommerfeld_rout(cg->shape, cg->X[0], cg->X[1], cg->X[2],
Pp->data->bbox[0], Pp->data->bbox[1], Pp->data->bbox[2],
Pp->data->bbox[3], Pp->data->bbox[4], Pp->data->bbox[5],
dT_lev,
cg->fgfs[phi0->sgfn],
cg->fgfs[Lap0->sgfn],
cg->fgfs[varl0->data->sgfn], cg->fgfs[varl1->data->sgfn],
varl0->data->SoA,
Symmetry, cor);
#if (SommerType == 1)
if (lev == 1) // shibata type sommerfeld
f_sommerfeld_rout(cg->shape, cg->X[0], cg->X[1], cg->X[2],
Pp->data->bbox[0], Pp->data->bbox[1], Pp->data->bbox[2],
Pp->data->bbox[3], Pp->data->bbox[4], Pp->data->bbox[5],
dT_lev,
cg->fgfs[phi0->sgfn],
cg->fgfs[Lap0->sgfn],
cg->fgfs[varl->data->sgfn], cg->fgfs[varl1->data->sgfn],
varl0->data->SoA,
Symmetry, cor);
#endif
varl0 = varl0->next;
varl = varl->next;
varl1 = varl1->next;
varlrhs = varlrhs->next;
}
}
f_lowerboundset(cg->shape, cg->fgfs[phi1->sgfn], chitiny);
}
if (BP == Pp->data->ble)
break;
BP = BP->next;
}
Pp = Pp->next;
}
// misc::tillherecheck(GH->Commlev[lev],GH->start_rank[lev],"before Corrector error check");
// Non-blocking error reduction overlapped with Sync to hide Allreduce latency
MPI_Request err_req_cor;
{
int erh = ERROR;
MPI_Iallreduce(&erh, &ERROR, 1, MPI_INT, MPI_SUM, GH->Commlev[lev], &err_req_cor);
}
// misc::tillherecheck(GH->Commlev[lev],GH->start_rank[lev],"before Corrector sync");
Parallel::Sync_cached(GH->PatL[lev], SynchList_cor, Symmetry, sync_cache_cor[lev]);
// misc::tillherecheck(GH->Commlev[lev],GH->start_rank[lev],"after Corrector sync");
// Complete non-blocking error reduction and check
MPI_Wait(&err_req_cor, MPI_STATUS_IGNORE);
if (ERROR)
{
Parallel::Dump_Data(GH->PatL[lev], SynchList_pre, 0, PhysTime, dT_lev);
if (myrank == 0)
{
if (ErrorMonitor->outfile)
ErrorMonitor->outfile << "find NaN in RK4 substep#" << iter_count
<< " variables at t = " << PhysTime
<< ", lev = " << lev << endl;
MPI_Abort(MPI_COMM_WORLD, 1);
}
}
#if (MAPBH == 0)
// for black hole position
if (BH_num > 0 && lev == GH->levels - 1)
{
compute_Porg_rhs(Porg, Porg1, Sfx, Sfy, Sfz, lev);
for (int ithBH = 0; ithBH < BH_num; ithBH++)
{
f_rungekutta4_scalar(dT_lev, Porg0[ithBH][0], Porg1[ithBH][0], Porg_rhs[ithBH][0], iter_count);
f_rungekutta4_scalar(dT_lev, Porg0[ithBH][1], Porg1[ithBH][1], Porg_rhs[ithBH][1], iter_count);
f_rungekutta4_scalar(dT_lev, Porg0[ithBH][2], Porg1[ithBH][2], Porg_rhs[ithBH][2], iter_count);
if (Symmetry > 0)
Porg1[ithBH][2] = fabs(Porg1[ithBH][2]);
if (Symmetry == 2)
{
Porg1[ithBH][0] = fabs(Porg1[ithBH][0]);
Porg1[ithBH][1] = fabs(Porg1[ithBH][1]);
}
if (!finite(Porg1[ithBH][0]) || !finite(Porg1[ithBH][1]) || !finite(Porg1[ithBH][2]))
{
if (ErrorMonitor->outfile)
ErrorMonitor->outfile << iter_count << " corrector step finds NaN for BH's position from ("
<< Porg[ithBH][0] << "," << Porg[ithBH][1] << "," << Porg[ithBH][2]
<< ")" << endl;
MyList<var> *DG_List = new MyList<var>(Sfx0);
DG_List->insert(Sfx0);
DG_List->insert(Sfy0);
DG_List->insert(Sfz0);
Parallel::Dump_Data(GH->PatL[lev], DG_List, 0, PhysTime, dT_lev);
DG_List->clearList();
}
}
}
// misc::tillherecheck(GH->Commlev[lev],GH->start_rank[lev],"after Corrector of black hole position");
#endif
// swap time level
if (iter_count < 3)
{
Pp = GH->PatL[lev];
while (Pp)
{
MyList<Block> *BP = Pp->data->blb;
while (BP)
{
Block *cg = BP->data;
cg->swapList(SynchList_pre, SynchList_cor, myrank);
if (BP == Pp->data->ble)
break;
BP = BP->next;
}
Pp = Pp->next;
}
// misc::tillherecheck(GH->Commlev[lev],GH->start_rank[lev],"after pre cor swap");
#if (MAPBH == 0)
// for black hole position
if (BH_num > 0 && lev == GH->levels - 1)
{
for (int ithBH = 0; ithBH < BH_num; ithBH++)
{
Porg[ithBH][0] = Porg1[ithBH][0];
Porg[ithBH][1] = Porg1[ithBH][1];
Porg[ithBH][2] = Porg1[ithBH][2];
}
}
#endif
}
// misc::tillherecheck(GH->Commlev[lev],GH->start_rank[lev],"tail of corrector");
}
#if (RPS == 0)
// mesh refinement boundary part
// misc::tillherecheck(GH->Commlev[lev],GH->start_rank[lev],"before RestrictProlong");
RestrictProlong(lev, YN, BB);
#endif
// note the data structure before update
// SynchList_cor 1 -----------
//
// StateList 0 -----------
//
// OldStateList old -----------
// update
Pp = GH->PatL[lev];
while (Pp)
{
MyList<Block> *BP = Pp->data->blb;
while (BP)
{
Block *cg = BP->data;
cg->swapList(StateList, SynchList_cor, myrank);
cg->swapList(OldStateList, SynchList_cor, myrank);
if (BP == Pp->data->ble)
break;
BP = BP->next;
}
Pp = Pp->next;
}
// for black hole position
if (BH_num > 0 && lev == GH->levels - 1)
{
for (int ithBH = 0; ithBH < BH_num; ithBH++)
{
Porg0[ithBH][0] = Porg1[ithBH][0];
Porg0[ithBH][1] = Porg1[ithBH][1];
Porg0[ithBH][2] = Porg1[ithBH][2];
// if(myrank==GH->start_rank[lev])
// cout<<Porg0[ithBH][0]<<","<<Porg0[ithBH][1]<<","<<Porg0[ithBH][2]<<endl;
}
}
// if(myrank==GH->start_rank[lev]) cout<<GH->mylev<<endl;
// misc::tillherecheck(GH->Commlev[lev],GH->start_rank[lev],"complet GH Step");
}
//================================================================================================
//================================================================================================
// This member function configures a single time-step evolution for the spherical-shell grid portion.
//================================================================================================
#ifdef WithShell
void bssn_class::SHStep()
{
int lev = 0;
// #if (PSTR == 1 || PSTR == 2)
// misc::tillherecheck(GH->Commlev[lev],GH->start_rank[lev],"start Step");
// #endif
setpbh(BH_num, Porg0, Mass, BH_num_input);
double dT_lev = dT * pow(0.5, Mymax(lev, trfls));
// #if (PSTR == 1 || PSTR == 2)
// misc::tillherecheck(GH->Commlev[lev],GH->start_rank[lev],"before Predictor");
// #endif
#ifdef With_AHF
AH_Step_Find(lev, dT_lev);
#endif
bool BB = fgt(PhysTime, StartTime, dT_lev / 2);
double ndeps = numepss;
if (lev < GH->movls)
ndeps = numepsb;
double TRK4 = PhysTime;
int iter_count = 0; // count RK4 substeps
int pre = 0, cor = 1;
int ERROR = 0;
MyList<ss_patch> *sPp;
// Predictor
sPp = SH->PatL;
while (sPp)
{
MyList<Block> *BP = sPp->data->blb;
int fngfs = sPp->data->fngfs;
while (BP)
{
Block *cg = BP->data;
if (myrank == cg->rank)
{
#if (AGM == 0)
f_enforce_ga(cg->shape,
cg->fgfs[gxx0->sgfn], cg->fgfs[gxy0->sgfn], cg->fgfs[gxz0->sgfn],
cg->fgfs[gyy0->sgfn], cg->fgfs[gyz0->sgfn], cg->fgfs[gzz0->sgfn],
cg->fgfs[Axx0->sgfn], cg->fgfs[Axy0->sgfn], cg->fgfs[Axz0->sgfn],
cg->fgfs[Ayy0->sgfn], cg->fgfs[Ayz0->sgfn], cg->fgfs[Azz0->sgfn]);
#endif
if (f_compute_rhs_bssn_ss(cg->shape, TRK4, cg->X[0], cg->X[1], cg->X[2],
cg->fgfs[fngfs + ShellPatch::gx],
cg->fgfs[fngfs + ShellPatch::gy],
cg->fgfs[fngfs + ShellPatch::gz],
cg->fgfs[fngfs + ShellPatch::drhodx],
cg->fgfs[fngfs + ShellPatch::drhody],
cg->fgfs[fngfs + ShellPatch::drhodz],
cg->fgfs[fngfs + ShellPatch::dsigmadx],
cg->fgfs[fngfs + ShellPatch::dsigmady],
cg->fgfs[fngfs + ShellPatch::dsigmadz],
cg->fgfs[fngfs + ShellPatch::dRdx],
cg->fgfs[fngfs + ShellPatch::dRdy],
cg->fgfs[fngfs + ShellPatch::dRdz],
cg->fgfs[fngfs + ShellPatch::drhodxx],
cg->fgfs[fngfs + ShellPatch::drhodxy],
cg->fgfs[fngfs + ShellPatch::drhodxz],
cg->fgfs[fngfs + ShellPatch::drhodyy],
cg->fgfs[fngfs + ShellPatch::drhodyz],
cg->fgfs[fngfs + ShellPatch::drhodzz],
cg->fgfs[fngfs + ShellPatch::dsigmadxx],
cg->fgfs[fngfs + ShellPatch::dsigmadxy],
cg->fgfs[fngfs + ShellPatch::dsigmadxz],
cg->fgfs[fngfs + ShellPatch::dsigmadyy],
cg->fgfs[fngfs + ShellPatch::dsigmadyz],
cg->fgfs[fngfs + ShellPatch::dsigmadzz],
cg->fgfs[fngfs + ShellPatch::dRdxx],
cg->fgfs[fngfs + ShellPatch::dRdxy],
cg->fgfs[fngfs + ShellPatch::dRdxz],
cg->fgfs[fngfs + ShellPatch::dRdyy],
cg->fgfs[fngfs + ShellPatch::dRdyz],
cg->fgfs[fngfs + ShellPatch::dRdzz],
cg->fgfs[phi0->sgfn], cg->fgfs[trK0->sgfn],
cg->fgfs[gxx0->sgfn], cg->fgfs[gxy0->sgfn], cg->fgfs[gxz0->sgfn],
cg->fgfs[gyy0->sgfn], cg->fgfs[gyz0->sgfn], cg->fgfs[gzz0->sgfn],
cg->fgfs[Axx0->sgfn], cg->fgfs[Axy0->sgfn], cg->fgfs[Axz0->sgfn],
cg->fgfs[Ayy0->sgfn], cg->fgfs[Ayz0->sgfn], cg->fgfs[Azz0->sgfn],
cg->fgfs[Gmx0->sgfn], cg->fgfs[Gmy0->sgfn], cg->fgfs[Gmz0->sgfn],
cg->fgfs[Lap0->sgfn],
cg->fgfs[Sfx0->sgfn], cg->fgfs[Sfy0->sgfn], cg->fgfs[Sfz0->sgfn],
cg->fgfs[dtSfx0->sgfn], cg->fgfs[dtSfy0->sgfn], cg->fgfs[dtSfz0->sgfn],
cg->fgfs[phi_rhs->sgfn], cg->fgfs[trK_rhs->sgfn],
cg->fgfs[gxx_rhs->sgfn], cg->fgfs[gxy_rhs->sgfn], cg->fgfs[gxz_rhs->sgfn],
cg->fgfs[gyy_rhs->sgfn], cg->fgfs[gyz_rhs->sgfn], cg->fgfs[gzz_rhs->sgfn],
cg->fgfs[Axx_rhs->sgfn], cg->fgfs[Axy_rhs->sgfn], cg->fgfs[Axz_rhs->sgfn],
cg->fgfs[Ayy_rhs->sgfn], cg->fgfs[Ayz_rhs->sgfn], cg->fgfs[Azz_rhs->sgfn],
cg->fgfs[Gmx_rhs->sgfn], cg->fgfs[Gmy_rhs->sgfn], cg->fgfs[Gmz_rhs->sgfn],
cg->fgfs[Lap_rhs->sgfn],
cg->fgfs[Sfx_rhs->sgfn], cg->fgfs[Sfy_rhs->sgfn], cg->fgfs[Sfz_rhs->sgfn],
cg->fgfs[dtSfx_rhs->sgfn], cg->fgfs[dtSfy_rhs->sgfn], cg->fgfs[dtSfz_rhs->sgfn],
cg->fgfs[rho->sgfn], cg->fgfs[Sx->sgfn], cg->fgfs[Sy->sgfn], cg->fgfs[Sz->sgfn],
cg->fgfs[Sxx->sgfn], cg->fgfs[Sxy->sgfn], cg->fgfs[Sxz->sgfn],
cg->fgfs[Syy->sgfn], cg->fgfs[Syz->sgfn], cg->fgfs[Szz->sgfn],
cg->fgfs[Gamxxx->sgfn], cg->fgfs[Gamxxy->sgfn], cg->fgfs[Gamxxz->sgfn],
cg->fgfs[Gamxyy->sgfn], cg->fgfs[Gamxyz->sgfn], cg->fgfs[Gamxzz->sgfn],
cg->fgfs[Gamyxx->sgfn], cg->fgfs[Gamyxy->sgfn], cg->fgfs[Gamyxz->sgfn],
cg->fgfs[Gamyyy->sgfn], cg->fgfs[Gamyyz->sgfn], cg->fgfs[Gamyzz->sgfn],
cg->fgfs[Gamzxx->sgfn], cg->fgfs[Gamzxy->sgfn], cg->fgfs[Gamzxz->sgfn],
cg->fgfs[Gamzyy->sgfn], cg->fgfs[Gamzyz->sgfn], cg->fgfs[Gamzzz->sgfn],
cg->fgfs[Rxx->sgfn], cg->fgfs[Rxy->sgfn], cg->fgfs[Rxz->sgfn],
cg->fgfs[Ryy->sgfn], cg->fgfs[Ryz->sgfn], cg->fgfs[Rzz->sgfn],
cg->fgfs[Cons_Ham->sgfn],
cg->fgfs[Cons_Px->sgfn], cg->fgfs[Cons_Py->sgfn], cg->fgfs[Cons_Pz->sgfn],
cg->fgfs[Cons_Gx->sgfn], cg->fgfs[Cons_Gy->sgfn], cg->fgfs[Cons_Gz->sgfn],
Symmetry, lev, numepsh, sPp->data->sst, pre))
{
cout << "find NaN in Shell domain: sst = " << sPp->data->sst << ", ("
<< cg->bbox[0] << ":" << cg->bbox[3] << ","
<< cg->bbox[1] << ":" << cg->bbox[4] << ","
<< cg->bbox[2] << ":" << cg->bbox[5] << ")" << endl;
ERROR = 1;
}
// rk4 substep and boundary
{
MyList<var> *varl0 = StateList, *varl = SynchList_pre, *varlrhs = RHSList;
// we do not check the correspondence here
while (varl0)
{
// sommerfeld indeed for outter boudary while fix BD for inner boundary
f_sommerfeld_routbam_ss(cg->shape, cg->X[0], cg->X[1], cg->X[2],
sPp->data->bbox[0], sPp->data->bbox[1], sPp->data->bbox[2],
sPp->data->bbox[3], sPp->data->bbox[4], sPp->data->bbox[5],
cg->fgfs[varlrhs->data->sgfn],
cg->fgfs[varl0->data->sgfn],
varl0->data->propspeed, varl0->data->SoA,
Symmetry);
f_rungekutta4_rout(cg->shape, dT_lev,
cg->fgfs[varl0->data->sgfn],
cg->fgfs[varl->data->sgfn],
cg->fgfs[varlrhs->data->sgfn],
iter_count);
varl0 = varl0->next;
varl = varl->next;
varlrhs = varlrhs->next;
}
}
f_lowerboundset(cg->shape, cg->fgfs[phi->sgfn], chitiny);
}
if (BP == sPp->data->ble)
break;
BP = BP->next;
}
sPp = sPp->next;
}
#if (PSTR == 1 || PSTR == 2)
// misc::tillherecheck(GH->Commlev[lev],GH->start_rank[lev],"before Predictor's error check");
#endif
// Non-blocking error reduction overlapped with Synch to hide Allreduce latency
MPI_Request err_req;
{
int erh = ERROR;
MPI_Iallreduce(&erh, &ERROR, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD, &err_req);
}
{
clock_t prev_clock, curr_clock;
if (myrank == 0)
curr_clock = clock();
SH->Synch(SynchList_pre, Symmetry);
if (myrank == 0)
{
prev_clock = curr_clock;
curr_clock = clock();
cout << " Shell stuff synchronization used "
<< (double)(curr_clock - prev_clock) / ((double)CLOCKS_PER_SEC)
<< " seconds! " << endl;
}
}
// Complete non-blocking error reduction and check
MPI_Wait(&err_req, MPI_STATUS_IGNORE);
if (ERROR)
{
SH->Dump_Data(StateList, 0, PhysTime, dT_lev);
if (myrank == 0)
{
if (ErrorMonitor->outfile)
ErrorMonitor->outfile << "find NaN in state variables on Shell Patches at t = " << PhysTime << endl;
MPI_Abort(MPI_COMM_WORLD, 1);
}
}
// corrector
for (iter_count = 1; iter_count < 4; iter_count++)
{
// for RK4: t0, t0+dt/2, t0+dt/2, t0+dt;
if (iter_count == 1 || iter_count == 3)
TRK4 += dT_lev / 2;
{
sPp = SH->PatL;
while (sPp)
{
MyList<Block> *BP = sPp->data->blb;
int fngfs = sPp->data->fngfs;
while (BP)
{
Block *cg = BP->data;
if (myrank == cg->rank)
{
#if (AGM == 0)
f_enforce_ga(cg->shape,
cg->fgfs[gxx->sgfn], cg->fgfs[gxy->sgfn], cg->fgfs[gxz->sgfn],
cg->fgfs[gyy->sgfn], cg->fgfs[gyz->sgfn], cg->fgfs[gzz->sgfn],
cg->fgfs[Axx->sgfn], cg->fgfs[Axy->sgfn], cg->fgfs[Axz->sgfn],
cg->fgfs[Ayy->sgfn], cg->fgfs[Ayz->sgfn], cg->fgfs[Azz->sgfn]);
#elif (AGM == 1)
if (iter_count == 3)
f_enforce_ga(cg->shape,
cg->fgfs[gxx->sgfn], cg->fgfs[gxy->sgfn], cg->fgfs[gxz->sgfn],
cg->fgfs[gyy->sgfn], cg->fgfs[gyz->sgfn], cg->fgfs[gzz->sgfn],
cg->fgfs[Axx->sgfn], cg->fgfs[Axy->sgfn], cg->fgfs[Axz->sgfn],
cg->fgfs[Ayy->sgfn], cg->fgfs[Ayz->sgfn], cg->fgfs[Azz->sgfn]);
#endif
if (f_compute_rhs_bssn_ss(cg->shape, TRK4, cg->X[0], cg->X[1], cg->X[2],
cg->fgfs[fngfs + ShellPatch::gx],
cg->fgfs[fngfs + ShellPatch::gy],
cg->fgfs[fngfs + ShellPatch::gz],
cg->fgfs[fngfs + ShellPatch::drhodx],
cg->fgfs[fngfs + ShellPatch::drhody],
cg->fgfs[fngfs + ShellPatch::drhodz],
cg->fgfs[fngfs + ShellPatch::dsigmadx],
cg->fgfs[fngfs + ShellPatch::dsigmady],
cg->fgfs[fngfs + ShellPatch::dsigmadz],
cg->fgfs[fngfs + ShellPatch::dRdx],
cg->fgfs[fngfs + ShellPatch::dRdy],
cg->fgfs[fngfs + ShellPatch::dRdz],
cg->fgfs[fngfs + ShellPatch::drhodxx],
cg->fgfs[fngfs + ShellPatch::drhodxy],
cg->fgfs[fngfs + ShellPatch::drhodxz],
cg->fgfs[fngfs + ShellPatch::drhodyy],
cg->fgfs[fngfs + ShellPatch::drhodyz],
cg->fgfs[fngfs + ShellPatch::drhodzz],
cg->fgfs[fngfs + ShellPatch::dsigmadxx],
cg->fgfs[fngfs + ShellPatch::dsigmadxy],
cg->fgfs[fngfs + ShellPatch::dsigmadxz],
cg->fgfs[fngfs + ShellPatch::dsigmadyy],
cg->fgfs[fngfs + ShellPatch::dsigmadyz],
cg->fgfs[fngfs + ShellPatch::dsigmadzz],
cg->fgfs[fngfs + ShellPatch::dRdxx],
cg->fgfs[fngfs + ShellPatch::dRdxy],
cg->fgfs[fngfs + ShellPatch::dRdxz],
cg->fgfs[fngfs + ShellPatch::dRdyy],
cg->fgfs[fngfs + ShellPatch::dRdyz],
cg->fgfs[fngfs + ShellPatch::dRdzz],
cg->fgfs[phi->sgfn], cg->fgfs[trK->sgfn],
cg->fgfs[gxx->sgfn], cg->fgfs[gxy->sgfn], cg->fgfs[gxz->sgfn],
cg->fgfs[gyy->sgfn], cg->fgfs[gyz->sgfn], cg->fgfs[gzz->sgfn],
cg->fgfs[Axx->sgfn], cg->fgfs[Axy->sgfn], cg->fgfs[Axz->sgfn],
cg->fgfs[Ayy->sgfn], cg->fgfs[Ayz->sgfn], cg->fgfs[Azz->sgfn],
cg->fgfs[Gmx->sgfn], cg->fgfs[Gmy->sgfn], cg->fgfs[Gmz->sgfn],
cg->fgfs[Lap->sgfn],
cg->fgfs[Sfx->sgfn], cg->fgfs[Sfy->sgfn], cg->fgfs[Sfz->sgfn],
cg->fgfs[dtSfx->sgfn], cg->fgfs[dtSfy->sgfn], cg->fgfs[dtSfz->sgfn],
cg->fgfs[phi1->sgfn], cg->fgfs[trK1->sgfn],
cg->fgfs[gxx1->sgfn], cg->fgfs[gxy1->sgfn], cg->fgfs[gxz1->sgfn],
cg->fgfs[gyy1->sgfn], cg->fgfs[gyz1->sgfn], cg->fgfs[gzz1->sgfn],
cg->fgfs[Axx1->sgfn], cg->fgfs[Axy1->sgfn], cg->fgfs[Axz1->sgfn],
cg->fgfs[Ayy1->sgfn], cg->fgfs[Ayz1->sgfn], cg->fgfs[Azz1->sgfn],
cg->fgfs[Gmx1->sgfn], cg->fgfs[Gmy1->sgfn], cg->fgfs[Gmz1->sgfn],
cg->fgfs[Lap1->sgfn],
cg->fgfs[Sfx1->sgfn], cg->fgfs[Sfy1->sgfn], cg->fgfs[Sfz1->sgfn],
cg->fgfs[dtSfx1->sgfn], cg->fgfs[dtSfy1->sgfn], cg->fgfs[dtSfz1->sgfn],
cg->fgfs[rho->sgfn],
cg->fgfs[Sx->sgfn], cg->fgfs[Sy->sgfn], cg->fgfs[Sz->sgfn],
cg->fgfs[Sxx->sgfn], cg->fgfs[Sxy->sgfn], cg->fgfs[Sxz->sgfn],
cg->fgfs[Syy->sgfn], cg->fgfs[Syz->sgfn], cg->fgfs[Szz->sgfn],
cg->fgfs[Gamxxx->sgfn], cg->fgfs[Gamxxy->sgfn], cg->fgfs[Gamxxz->sgfn],
cg->fgfs[Gamxyy->sgfn], cg->fgfs[Gamxyz->sgfn], cg->fgfs[Gamxzz->sgfn],
cg->fgfs[Gamyxx->sgfn], cg->fgfs[Gamyxy->sgfn], cg->fgfs[Gamyxz->sgfn],
cg->fgfs[Gamyyy->sgfn], cg->fgfs[Gamyyz->sgfn], cg->fgfs[Gamyzz->sgfn],
cg->fgfs[Gamzxx->sgfn], cg->fgfs[Gamzxy->sgfn], cg->fgfs[Gamzxz->sgfn],
cg->fgfs[Gamzyy->sgfn], cg->fgfs[Gamzyz->sgfn], cg->fgfs[Gamzzz->sgfn],
cg->fgfs[Rxx->sgfn], cg->fgfs[Rxy->sgfn], cg->fgfs[Rxz->sgfn],
cg->fgfs[Ryy->sgfn], cg->fgfs[Ryz->sgfn], cg->fgfs[Rzz->sgfn],
cg->fgfs[Cons_Ham->sgfn],
cg->fgfs[Cons_Px->sgfn], cg->fgfs[Cons_Py->sgfn], cg->fgfs[Cons_Pz->sgfn],
cg->fgfs[Cons_Gx->sgfn], cg->fgfs[Cons_Gy->sgfn], cg->fgfs[Cons_Gz->sgfn],
Symmetry, lev, numepsh, sPp->data->sst, cor))
{
cout << "find NaN in Shell domain: sst = " << sPp->data->sst << ", ("
<< cg->bbox[0] << ":" << cg->bbox[3] << ","
<< cg->bbox[1] << ":" << cg->bbox[4] << ","
<< cg->bbox[2] << ":" << cg->bbox[5] << ")" << endl;
ERROR = 1;
}
// rk4 substep and boundary
{
MyList<var> *varl0 = StateList, *varl = SynchList_pre, *varl1 = SynchList_cor, *varlrhs = RHSList;
// we do not check the correspondence here
while (varl0)
{
// sommerfeld indeed for outter boudary while fix BD for inner boundary
f_sommerfeld_routbam_ss(cg->shape, cg->X[0], cg->X[1], cg->X[2],
sPp->data->bbox[0], sPp->data->bbox[1], sPp->data->bbox[2],
sPp->data->bbox[3], sPp->data->bbox[4], sPp->data->bbox[5],
cg->fgfs[varl1->data->sgfn],
cg->fgfs[varl->data->sgfn],
varl0->data->propspeed, varl0->data->SoA,
Symmetry);
f_rungekutta4_rout(cg->shape, dT_lev,
cg->fgfs[varl0->data->sgfn],
cg->fgfs[varl1->data->sgfn],
cg->fgfs[varlrhs->data->sgfn],
iter_count);
varl0 = varl0->next;
varl = varl->next;
varl1 = varl1->next;
varlrhs = varlrhs->next;
}
}
f_lowerboundset(cg->shape, cg->fgfs[phi1->sgfn], chitiny);
}
if (BP == sPp->data->ble)
break;
BP = BP->next;
}
sPp = sPp->next;
}
}
// Non-blocking error reduction overlapped with Synch to hide Allreduce latency
MPI_Request err_req_cor;
{
int erh = ERROR;
MPI_Iallreduce(&erh, &ERROR, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD, &err_req_cor);
}
{
clock_t prev_clock, curr_clock;
if (myrank == 0)
curr_clock = clock();
SH->Synch(SynchList_cor, Symmetry);
if (myrank == 0)
{
prev_clock = curr_clock;
curr_clock = clock();
cout << " Shell stuff synchronization used "
<< (double)(curr_clock - prev_clock) / ((double)CLOCKS_PER_SEC)
<< " seconds! " << endl;
}
}
// Complete non-blocking error reduction and check
MPI_Wait(&err_req_cor, MPI_STATUS_IGNORE);
if (ERROR)
{
SH->Dump_Data(SynchList_pre, 0, PhysTime, dT_lev);
if (myrank == 0)
{
if (ErrorMonitor->outfile)
ErrorMonitor->outfile << "find NaN on Shell Patches in RK4 substep#" << iter_count
<< " variables at t = " << PhysTime << endl;
MPI_Abort(MPI_COMM_WORLD, 1);
}
}
sPp = SH->PatL;
while (sPp)
{
MyList<Block> *BP = sPp->data->blb;
while (BP)
{
Block *cg = BP->data;
cg->swapList(SynchList_pre, SynchList_cor, myrank);
if (BP == sPp->data->ble)
break;
BP = BP->next;
}
sPp = sPp->next;
}
}
#if (RPS == 0)
{
clock_t prev_clock, curr_clock;
if (myrank == 0)
curr_clock = clock();
SH->CS_Inter(SynchList_cor, Symmetry);
if (myrank == 0)
{
prev_clock = curr_clock;
curr_clock = clock();
cout << " CS_Inter used " << (double)(curr_clock - prev_clock) / ((double)CLOCKS_PER_SEC)
<< " seconds! " << endl;
}
}
#endif
// note the data structure before update
// SynchList_cor 1 -----------
//
// StateList 0 -----------
//
// OldStateList old -----------
// update
sPp = SH->PatL;
while (sPp)
{
MyList<Block> *BP = sPp->data->blb;
while (BP)
{
Block *cg = BP->data;
cg->swapList(StateList, SynchList_cor, myrank);
cg->swapList(OldStateList, SynchList_cor, myrank);
if (BP == sPp->data->ble)
break;
BP = BP->next;
}
sPp = sPp->next;
}
}
#endif
#endif
//================================================================================================
//================================================================================================
// 0: do not use mixing two levels data for OutBD; 1: do use
#define MIXOUTB 0
void bssn_class::RestrictProlong(int lev, int YN, bool BB,
MyList<var> *SL, MyList<var> *OL, MyList<var> *corL)
// we assume
// StateList 1 -----------
//
// OldStateList 0 -----------
//
// SynchList_cor old -----------
{
#if (PSTR == 1 || PSTR == 2)
// stringstream a_stream;
// a_stream.setf(ios::left);
#endif
if (lev > 0)
{
MyList<Patch> *Pp, *Ppc;
if (lev > trfls && YN == 0) // time refinement levels and for intermediat time level
{
Pp = GH->PatL[lev - 1];
while (Pp)
{
if (BB)
Parallel::prepare_inter_time_level(Pp->data, SL, OL, corL,
SynchList_pre, 0); // use SynchList_pre as temporal storage space
else
Parallel::prepare_inter_time_level(Pp->data, SL, OL,
SynchList_pre, 0); // use SynchList_pre as temporal storage space
#if (PSTR == 1 || PSTR == 2)
// Pp->data->checkPatch(0,GH->start_rank[GH->mylev]);
#endif
Pp = Pp->next;
}
#if (PSTR == 1 || PSTR == 2)
// Pp=GH->PatL[lev];
// while(Pp)
// {
// Pp->data->checkPatch(0,GH->start_rank[GH->mylev]);
// Pp=Pp->next;
// }
// a_stream.clear();
// a_stream.str("");
// a_stream<<GH->mylev<<": 0 before Restrict";
// misc::tillherecheck(GH->Commlev[GH->mylev],GH->start_rank[GH->mylev],a_stream.str());
#endif
#if (RPB == 0)
Parallel::Restrict(GH->PatL[lev - 1], GH->PatL[lev], SL, SynchList_pre, Symmetry);
#elif (RPB == 1)
// Parallel::Restrict_bam(GH->PatL[lev-1],GH->PatL[lev],SL,SynchList_pre,Symmetry);
Parallel::Restrict_bam(GH->PatL[lev - 1], GH->PatL[lev], SL, SynchList_pre, GH->rsul[lev], Symmetry);
#endif
#if (PSTR == 1 || PSTR == 2)
// a_stream.clear();
// a_stream.str("");
// a_stream<<GH->mylev<<": 0 after Restrict";
// misc::tillherecheck(GH->Commlev[GH->mylev],GH->start_rank[GH->mylev],a_stream.str());
#endif
Parallel::Sync_cached(GH->PatL[lev - 1], SynchList_pre, Symmetry, sync_cache_rp_coarse[lev]);
#if (PSTR == 1 || PSTR == 2)
// a_stream.clear();
// a_stream.str("");
// a_stream<<GH->mylev<<": 0 after Sync";
// misc::tillherecheck(GH->Commlev[GH->mylev],GH->start_rank[GH->mylev],a_stream.str());
#endif
#if (RPB == 0)
Ppc = GH->PatL[lev - 1];
while (Ppc)
{
Pp = GH->PatL[lev];
while (Pp)
{
#if (MIXOUTB == 0)
Parallel::OutBdLow2Hi(Ppc->data, Pp->data, SynchList_pre, SL, Symmetry);
#elif (MIXOUTB == 1)
Parallel::OutBdLow2Himix(Ppc->data, Pp->data, SynchList_pre, SL, Symmetry);
#endif
Pp = Pp->next;
}
Ppc = Ppc->next;
}
#elif (RPB == 1)
// Parallel::OutBdLow2Hi_bam(GH->PatL[lev-1],GH->PatL[lev],SynchList_pre,SL,Symmetry);
Parallel::OutBdLow2Hi_bam(GH->PatL[lev - 1], GH->PatL[lev], SynchList_pre, SL, GH->bdsul[lev], Symmetry);
#endif
#if (PSTR == 1 || PSTR == 2)
// a_stream.clear();
// a_stream.str("");
// a_stream<<GH->mylev<<": 0 after OutBdLow2Hi";
// misc::tillherecheck(GH->Commlev[GH->mylev],GH->start_rank[GH->mylev],a_stream.str());
#endif
}
else // no time refinement levels and for all same time levels
{
#if (PSTR == 1 || PSTR == 2)
// a_stream.clear();
// a_stream.str("");
// a_stream<<GH->mylev<<": 1 before Restrict";
// misc::tillherecheck(GH->Commlev[GH->mylev],GH->start_rank[GH->mylev],a_stream.str());
#endif
#if (RPB == 0)
Parallel::Restrict(GH->PatL[lev - 1], GH->PatL[lev], SL, SL, Symmetry);
#elif (RPB == 1)
// Parallel::Restrict_bam(GH->PatL[lev-1],GH->PatL[lev],SL,SL,Symmetry);
Parallel::Restrict_bam(GH->PatL[lev - 1], GH->PatL[lev], SL, SL, GH->rsul[lev], Symmetry);
#endif
#if (PSTR == 1 || PSTR == 2)
// a_stream.clear();
// a_stream.str("");
// a_stream<<GH->mylev<<": 1 before Sync";
// misc::tillherecheck(GH->Commlev[GH->mylev],GH->start_rank[GH->mylev],a_stream.str());
#endif
Parallel::Sync_cached(GH->PatL[lev - 1], SL, Symmetry, sync_cache_rp_coarse[lev]);
#if (PSTR == 1 || PSTR == 2)
// a_stream.clear();
// a_stream.str("");
// a_stream<<GH->mylev<<": 1 after Sync";
// misc::tillherecheck(GH->Commlev[GH->mylev],GH->start_rank[GH->mylev],a_stream.str());
#endif
#if (RPB == 0)
Ppc = GH->PatL[lev - 1];
while (Ppc)
{
Pp = GH->PatL[lev];
while (Pp)
{
#if (MIXOUTB == 0)
Parallel::OutBdLow2Hi(Ppc->data, Pp->data, SL, SL, Symmetry);
#elif (MIXOUTB == 1)
Parallel::OutBdLow2Himix(Ppc->data, Pp->data, SL, SL, Symmetry);
#endif
Pp = Pp->next;
}
Ppc = Ppc->next;
}
#elif (RPB == 1)
// Parallel::OutBdLow2Hi_bam(GH->PatL[lev-1],GH->PatL[lev],SL,SL,Symmetry);
Parallel::OutBdLow2Hi_bam(GH->PatL[lev - 1], GH->PatL[lev], SL, SL, GH->bdsul[lev], Symmetry);
#endif
#if (PSTR == 1 || PSTR == 2)
// a_stream.clear();
// a_stream.str("");
// a_stream<<GH->mylev<<": 1 after OutBdLow2Hi";
// misc::tillherecheck(GH->Commlev[GH->mylev],GH->start_rank[GH->mylev],a_stream.str());
#endif
}
Parallel::Sync_cached(GH->PatL[lev], SL, Symmetry, sync_cache_rp_fine[lev]);
#if (PSTR == 1 || PSTR == 2)
// a_stream.clear();
// a_stream.str("");
// a_stream<<GH->mylev<<": after Sync";
// misc::tillherecheck(GH->Commlev[GH->mylev],GH->start_rank[GH->mylev],a_stream.str());
#endif
}
}
//================================================================================================
//================================================================================================
// auxiliary operation, input lev means original lev-1
void bssn_class::RestrictProlong_aux(int lev, int YN, bool BB,
MyList<var> *SL, MyList<var> *OL, MyList<var> *corL)
// we assume
// StateList 1 -----------
//
// OldStateList 0 -----------
//
// SynchList_cor old -----------
{
// misc::tillherecheck(GH->Commlev[lev],GH->start_rank[lev],"starting RestrictProlong_aux");
if (lev >= GH->levels - 1)
return;
lev = lev + 1;
if (lev > 0)
{
MyList<Patch> *Pp, *Ppc;
if (lev > trfls && YN == 0) // time refinement levels and for intermediat time level
{
Pp = GH->PatL[lev - 1];
while (Pp)
{
if (BB)
Parallel::prepare_inter_time_level(Pp->data, SL, OL, corL,
SynchList_pre, 0); // use SynchList_pre as temporal storage space
else
Parallel::prepare_inter_time_level(Pp->data, SL, OL,
SynchList_pre, 0); // use SynchList_pre as temporal storage space
Pp = Pp->next;
}
#if (RPB == 0)
Parallel::Restrict(GH->PatL[lev - 1], GH->PatL[lev], SL, SynchList_pre, Symmetry);
#elif (RPB == 1)
// Parallel::Restrict_bam(GH->PatL[lev-1],GH->PatL[lev],SL,SynchList_pre,Symmetry);
Parallel::Restrict_bam(GH->PatL[lev - 1], GH->PatL[lev], SL, SynchList_pre, GH->rsul[lev], Symmetry);
#endif
Parallel::Sync_cached(GH->PatL[lev - 1], SynchList_pre, Symmetry, sync_cache_rp_coarse[lev]);
#if (RPB == 0)
Ppc = GH->PatL[lev - 1];
while (Ppc)
{
Pp = GH->PatL[lev];
while (Pp)
{
#if (MIXOUTB == 0)
Parallel::OutBdLow2Hi(Ppc->data, Pp->data, SynchList_pre, SL, Symmetry);
#elif (MIXOUTB == 1)
Parallel::OutBdLow2Himix(Ppc->data, Pp->data, SynchList_pre, SL, Symmetry);
#endif
Pp = Pp->next;
}
Ppc = Ppc->next;
}
#elif (RPB == 1)
// Parallel::OutBdLow2Hi_bam(GH->PatL[lev-1],GH->PatL[lev],SynchList_pre,SL,Symmetry);
Parallel::OutBdLow2Hi_bam(GH->PatL[lev - 1], GH->PatL[lev], SynchList_pre, SL, GH->bdsul[lev], Symmetry);
#endif
}
else // no time refinement levels and for all same time levels
{
#if (RPB == 0)
Parallel::Restrict(GH->PatL[lev - 1], GH->PatL[lev], SL, SL, Symmetry);
#elif (RPB == 1)
// Parallel::Restrict_bam(GH->PatL[lev-1],GH->PatL[lev],SL,SL,Symmetry);
Parallel::Restrict_bam(GH->PatL[lev - 1], GH->PatL[lev], SL, SL, GH->rsul[lev], Symmetry);
#endif
Parallel::Sync_cached(GH->PatL[lev - 1], SL, Symmetry, sync_cache_rp_coarse[lev]);
#if (RPB == 0)
Ppc = GH->PatL[lev - 1];
while (Ppc)
{
Pp = GH->PatL[lev];
while (Pp)
{
#if (MIXOUTB == 0)
Parallel::OutBdLow2Hi(Ppc->data, Pp->data, SL, SL, Symmetry);
#elif (MIXOUTB == 1)
Parallel::OutBdLow2Himix(Ppc->data, Pp->data, SL, SL, Symmetry);
#endif
Pp = Pp->next;
}
Ppc = Ppc->next;
}
#elif (RPB == 1)
// Parallel::OutBdLow2Hi_bam(GH->PatL[lev-1],GH->PatL[lev],SL,SL,Symmetry);
Parallel::OutBdLow2Hi_bam(GH->PatL[lev - 1], GH->PatL[lev], SL, SL, GH->bdsul[lev], Symmetry);
#endif
}
Parallel::Sync_cached(GH->PatL[lev], SL, Symmetry, sync_cache_rp_fine[lev]);
}
}
//================================================================================================
//================================================================================================
void bssn_class::RestrictProlong(int lev, int YN, bool BB)
{
double dT_lev = dT * pow(0.5, Mymax(lev, trfls));
// we assume for fine
// SynchList_cor 1 -----------
//
// StateList 0 -----------
//
// OldStateList old -----------
// for coarse
// StateList 1 -----------
//
// OldStateList 0 -----------
//
// SynchList_cor old -----------
if (lev > 0)
{
MyList<Patch> *Pp, *Ppc;
if (lev > trfls && YN == 0) // time refinement levels and for intermediat time level
{
if (myrank == 0)
cout << "/=: " << GH->Lt[lev - 1] << "," << GH->Lt[lev] + dT_lev << endl;
Pp = GH->PatL[lev - 1];
while (Pp)
{
if (BB)
Parallel::prepare_inter_time_level(Pp->data, StateList, OldStateList, SynchList_cor,
SynchList_pre, 0); // use SynchList_pre as temporal storage space
else
Parallel::prepare_inter_time_level(Pp->data, StateList, OldStateList,
SynchList_pre, 0); // use SynchList_pre as temporal storage space
Pp = Pp->next;
}
#if (RPB == 0)
Parallel::Restrict(GH->PatL[lev - 1], GH->PatL[lev], SynchList_cor, SynchList_pre, Symmetry);
#elif (RPB == 1)
// Parallel::Restrict_bam(GH->PatL[lev-1],GH->PatL[lev],SynchList_cor,SynchList_pre,Symmetry);
Parallel::Restrict_bam(GH->PatL[lev - 1], GH->PatL[lev], SynchList_cor, SynchList_pre, GH->rsul[lev], Symmetry);
#endif
Parallel::Sync_cached(GH->PatL[lev - 1], SynchList_pre, Symmetry, sync_cache_rp_coarse[lev]);
#if (RPB == 0)
Ppc = GH->PatL[lev - 1];
while (Ppc)
{
Pp = GH->PatL[lev];
while (Pp)
{
#if (MIXOUTB == 0)
Parallel::OutBdLow2Hi(Ppc->data, Pp->data, SynchList_pre, SynchList_cor, Symmetry);
#elif (MIXOUTB == 1)
Parallel::OutBdLow2Himix(Ppc->data, Pp->data, SynchList_pre, SynchList_cor, Symmetry);
#endif
Pp = Pp->next;
}
Ppc = Ppc->next;
}
#elif (RPB == 1)
// Parallel::OutBdLow2Hi_bam(GH->PatL[lev-1],GH->PatL[lev],SynchList_pre,SynchList_cor,Symmetry);
Parallel::OutBdLow2Hi_bam(GH->PatL[lev - 1], GH->PatL[lev], SynchList_pre, SynchList_cor, GH->bdsul[lev], Symmetry);
#endif
}
else // no time refinement levels and for all same time levels
{
if (myrank == 0)
cout << "===: " << GH->Lt[lev - 1] << "," << GH->Lt[lev] + dT_lev << endl;
#if (RPB == 0)
Parallel::Restrict(GH->PatL[lev - 1], GH->PatL[lev], SynchList_cor, StateList, Symmetry);
#elif (RPB == 1)
// Parallel::Restrict_bam(GH->PatL[lev-1],GH->PatL[lev],SynchList_cor,StateList,Symmetry);
Parallel::Restrict_bam(GH->PatL[lev - 1], GH->PatL[lev], SynchList_cor, StateList, GH->rsul[lev], Symmetry);
#endif
Parallel::Sync_cached(GH->PatL[lev - 1], StateList, Symmetry, sync_cache_rp_coarse[lev]);
#if (RPB == 0)
Ppc = GH->PatL[lev - 1];
while (Ppc)
{
Pp = GH->PatL[lev];
while (Pp)
{
#if (MIXOUTB == 0)
Parallel::OutBdLow2Hi(Ppc->data, Pp->data, StateList, SynchList_cor, Symmetry);
#elif (MIXOUTB == 1)
Parallel::OutBdLow2Himix(Ppc->data, Pp->data, StateList, SynchList_cor, Symmetry);
#endif
Pp = Pp->next;
}
Ppc = Ppc->next;
}
#elif (RPB == 1)
// Parallel::OutBdLow2Hi_bam(GH->PatL[lev-1],GH->PatL[lev],StateList,SynchList_cor,Symmetry);
Parallel::OutBdLow2Hi_bam(GH->PatL[lev - 1], GH->PatL[lev], StateList, SynchList_cor, GH->bdsul[lev], Symmetry);
#endif
}
Parallel::Sync_cached(GH->PatL[lev], SynchList_cor, Symmetry, sync_cache_rp_fine[lev]);
}
}
//================================================================================================
//================================================================================================
void bssn_class::ProlongRestrict(int lev, int YN, bool BB)
{
if (lev > 0)
{
MyList<Patch> *Pp, *Ppc;
if (lev > trfls && YN == 0) // time refinement levels and for intermediat time level
{
Pp = GH->PatL[lev - 1];
while (Pp)
{
if (BB)
Parallel::prepare_inter_time_level(Pp->data, StateList, OldStateList, SynchList_cor,
SynchList_pre, 0); // use SynchList_pre as temporal storage space
else
Parallel::prepare_inter_time_level(Pp->data, StateList, OldStateList,
SynchList_pre, 0); // use SynchList_pre as temporal storage space
Pp = Pp->next;
}
#if (RPB == 0)
Ppc = GH->PatL[lev - 1];
while (Ppc)
{
Pp = GH->PatL[lev];
while (Pp)
{
#if (MIXOUTB == 0)
Parallel::OutBdLow2Hi(Ppc->data, Pp->data, SynchList_pre, SynchList_cor, Symmetry);
#elif (MIXOUTB == 1)
Parallel::OutBdLow2Himix(Ppc->data, Pp->data, SynchList_pre, SynchList_cor, Symmetry);
#endif
Pp = Pp->next;
}
Ppc = Ppc->next;
}
#elif (RPB == 1)
// Parallel::OutBdLow2Hi_bam(GH->PatL[lev-1],GH->PatL[lev],SynchList_pre,SynchList_cor,Symmetry);
Parallel::OutBdLow2Hi_bam(GH->PatL[lev - 1], GH->PatL[lev], SynchList_pre, SynchList_cor, GH->bdsul[lev], Symmetry);
#endif
}
else // no time refinement levels and for all same time levels
{
#if (RPB == 0)
Ppc = GH->PatL[lev - 1];
while (Ppc)
{
Pp = GH->PatL[lev];
while (Pp)
{
#if (MIXOUTB == 0)
Parallel::OutBdLow2Hi(Ppc->data, Pp->data, StateList, SynchList_cor, Symmetry);
#elif (MIXOUTB == 1)
Parallel::OutBdLow2Himix(Ppc->data, Pp->data, StateList, SynchList_cor, Symmetry);
#endif
Pp = Pp->next;
}
Ppc = Ppc->next;
}
#elif (RPB == 1)
// Parallel::OutBdLow2Hi_bam(GH->PatL[lev-1],GH->PatL[lev],StateList,SynchList_cor,Symmetry);
Parallel::OutBdLow2Hi_bam(GH->PatL[lev - 1], GH->PatL[lev], StateList, SynchList_cor, GH->bdsul[lev], Symmetry);
#endif
#if 0
#if (RPB == 0)
Parallel::Restrict(GH->PatL[lev-1],GH->PatL[lev],SynchList_cor,StateList,Symmetry);
#elif (RPB == 1)
// Parallel::Restrict_bam(GH->PatL[lev-1],GH->PatL[lev],SynchList_cor,StateList,Symmetry);
Parallel::Restrict_bam(GH->PatL[lev-1],GH->PatL[lev],SynchList_cor,StateList,GH->rsul[lev],Symmetry);
#endif
#else
Parallel::Restrict_after(GH->PatL[lev - 1], GH->PatL[lev], SynchList_cor, StateList, Symmetry);
#endif
Parallel::Sync_cached(GH->PatL[lev - 1], StateList, Symmetry, sync_cache_rp_coarse[lev]);
}
Parallel::Sync_cached(GH->PatL[lev], SynchList_cor, Symmetry, sync_cache_rp_fine[lev]);
}
}
#undef MIXOUTB
//================================================================================================
//================================================================================================
// This member function computes the gravitational-wave quantity Psi4
//================================================================================================
void bssn_class::Compute_Psi4(int lev)
{
MyList<var> *DG_List = new MyList<var>(Rpsi4);
DG_List->insert(Ipsi4);
#if 0 // test showes this operation does not help
for(int ilev = GH->levels-1;ilev>=lev;ilev--)
{
MyList<Patch> *Pp=GH->PatL[ilev];
#else
MyList<Patch> *Pp = GH->PatL[lev];
#endif
while (Pp)
{
MyList<Block> *BP = Pp->data->blb;
while (BP)
{
Block *cg = BP->data;
if (myrank == cg->rank)
{
#if (Psi4type == 0)
if (0) // if Gamma^i_jk and R_ij can be reused from the rhs calculation
f_ricci_gamma(cg->shape, cg->X[0], cg->X[1], cg->X[2],
cg->fgfs[phi0->sgfn],
cg->fgfs[gxx0->sgfn], cg->fgfs[gxy0->sgfn], cg->fgfs[gxz0->sgfn],
cg->fgfs[gyy0->sgfn], cg->fgfs[gyz0->sgfn], cg->fgfs[gzz0->sgfn],
cg->fgfs[Gmx0->sgfn], cg->fgfs[Gmy0->sgfn], cg->fgfs[Gmz0->sgfn],
cg->fgfs[Gamxxx->sgfn], cg->fgfs[Gamxxy->sgfn], cg->fgfs[Gamxxz->sgfn],
cg->fgfs[Gamxyy->sgfn], cg->fgfs[Gamxyz->sgfn], cg->fgfs[Gamxzz->sgfn],
cg->fgfs[Gamyxx->sgfn], cg->fgfs[Gamyxy->sgfn], cg->fgfs[Gamyxz->sgfn],
cg->fgfs[Gamyyy->sgfn], cg->fgfs[Gamyyz->sgfn], cg->fgfs[Gamyzz->sgfn],
cg->fgfs[Gamzxx->sgfn], cg->fgfs[Gamzxy->sgfn], cg->fgfs[Gamzxz->sgfn],
cg->fgfs[Gamzyy->sgfn], cg->fgfs[Gamzyz->sgfn], cg->fgfs[Gamzzz->sgfn],
cg->fgfs[Rxx->sgfn], cg->fgfs[Rxy->sgfn], cg->fgfs[Rxz->sgfn],
cg->fgfs[Ryy->sgfn], cg->fgfs[Ryz->sgfn], cg->fgfs[Rzz->sgfn],
Symmetry);
// the input arguments Gamma^i_jk and R_ij do not need synch, because we do not need to derivate them
f_getnp4(cg->shape, cg->X[0], cg->X[1], cg->X[2],
cg->fgfs[phi0->sgfn], cg->fgfs[trK0->sgfn],
cg->fgfs[gxx0->sgfn], cg->fgfs[gxy0->sgfn], cg->fgfs[gxz0->sgfn],
cg->fgfs[gyy0->sgfn], cg->fgfs[gyz0->sgfn], cg->fgfs[gzz0->sgfn],
cg->fgfs[Axx0->sgfn], cg->fgfs[Axy0->sgfn], cg->fgfs[Axz0->sgfn],
cg->fgfs[Ayy0->sgfn], cg->fgfs[Ayz0->sgfn], cg->fgfs[Azz0->sgfn],
cg->fgfs[Gamxxx->sgfn], cg->fgfs[Gamxxy->sgfn], cg->fgfs[Gamxxz->sgfn],
cg->fgfs[Gamxyy->sgfn], cg->fgfs[Gamxyz->sgfn], cg->fgfs[Gamxzz->sgfn],
cg->fgfs[Gamyxx->sgfn], cg->fgfs[Gamyxy->sgfn], cg->fgfs[Gamyxz->sgfn],
cg->fgfs[Gamyyy->sgfn], cg->fgfs[Gamyyz->sgfn], cg->fgfs[Gamyzz->sgfn],
cg->fgfs[Gamzxx->sgfn], cg->fgfs[Gamzxy->sgfn], cg->fgfs[Gamzxz->sgfn],
cg->fgfs[Gamzyy->sgfn], cg->fgfs[Gamzyz->sgfn], cg->fgfs[Gamzzz->sgfn],
cg->fgfs[Rxx->sgfn], cg->fgfs[Rxy->sgfn], cg->fgfs[Rxz->sgfn],
cg->fgfs[Ryy->sgfn], cg->fgfs[Ryz->sgfn], cg->fgfs[Rzz->sgfn],
cg->fgfs[Rpsi4->sgfn], cg->fgfs[Ipsi4->sgfn],
Symmetry);
#elif (Psi4type == 1)
f_getnp4old(cg->shape, cg->X[0], cg->X[1], cg->X[2],
cg->fgfs[phi0->sgfn], cg->fgfs[trK0->sgfn],
cg->fgfs[gxx0->sgfn], cg->fgfs[gxy0->sgfn], cg->fgfs[gxz0->sgfn],
cg->fgfs[gyy0->sgfn], cg->fgfs[gyz0->sgfn], cg->fgfs[gzz0->sgfn],
cg->fgfs[Axx0->sgfn], cg->fgfs[Axy0->sgfn], cg->fgfs[Axz0->sgfn],
cg->fgfs[Ayy0->sgfn], cg->fgfs[Ayz0->sgfn], cg->fgfs[Azz0->sgfn],
cg->fgfs[Gmx0->sgfn], cg->fgfs[Gmy0->sgfn], cg->fgfs[Gmz0->sgfn],
cg->fgfs[Lap0->sgfn],
cg->fgfs[Sfx0->sgfn], cg->fgfs[Sfy0->sgfn], cg->fgfs[Sfz0->sgfn],
cg->fgfs[Rpsi4->sgfn], cg->fgfs[Ipsi4->sgfn],
Symmetry);
#else
#error "not recognized Psi4type"
#endif
}
if (BP == Pp->data->ble)
break;
BP = BP->next;
}
Pp = Pp->next;
}
#if 0
Parallel::Sync(GH->PatL[ilev],DG_List,Symmetry);
}
// because of double level data change, you can not do this in above loop
// prolong restrict Psi4
for(int ilev=GH->levels-1;ilev>lev;ilev--)
RestrictProlong(ilev,1,false,DG_List,DG_List,DG_List);
#else
Parallel::Sync(GH->PatL[lev], DG_List, Symmetry);
#endif
#ifdef WithShell
// ShellPatch part
if (lev == 0)
{
MyList<ss_patch> *Pp = SH->PatL;
while (Pp)
{
MyList<Block> *BL = Pp->data->blb;
int fngfs = Pp->data->fngfs;
while (BL)
{
Block *cg = BL->data;
if (myrank == cg->rank)
{
#if (Psi4type == 0)
if (0) // if Gamma^i_jk and R_ij can be reused from the rhs calculation
f_ricci_gamma_ss(cg->shape, cg->X[0], cg->X[1], cg->X[2],
cg->fgfs[fngfs +
ShellPatch::gx], cg->fgfs[fngfs + ShellPatch::gy],
cg->fgfs[fngfs + ShellPatch::gz],
cg->fgfs[fngfs + ShellPatch::drhodx],
cg->fgfs[fngfs + ShellPatch::drhody],
cg->fgfs[fngfs + ShellPatch::drhodz],
cg->fgfs[fngfs + ShellPatch::dsigmadx],
cg->fgfs[fngfs + ShellPatch::dsigmady],
cg->fgfs[fngfs + ShellPatch::dsigmadz],
cg->fgfs[fngfs + ShellPatch::dRdx],
cg->fgfs[fngfs + ShellPatch::dRdy],
cg->fgfs[fngfs + ShellPatch::dRdz],
cg->fgfs[fngfs + ShellPatch::drhodxx],
cg->fgfs[fngfs + ShellPatch::drhodxy],
cg->fgfs[fngfs + ShellPatch::drhodxz],
cg->fgfs[fngfs + ShellPatch::drhodyy],
cg->fgfs[fngfs + ShellPatch::drhodyz],
cg->fgfs[fngfs + ShellPatch::drhodzz],
cg->fgfs[fngfs + ShellPatch::dsigmadxx],
cg->fgfs[fngfs + ShellPatch::dsigmadxy],
cg->fgfs[fngfs + ShellPatch::dsigmadxz],
cg->fgfs[fngfs + ShellPatch::dsigmadyy],
cg->fgfs[fngfs + ShellPatch::dsigmadyz],
cg->fgfs[fngfs + ShellPatch::dsigmadzz],
cg->fgfs[fngfs + ShellPatch::dRdxx],
cg->fgfs[fngfs + ShellPatch::dRdxy],
cg->fgfs[fngfs + ShellPatch::dRdxz],
cg->fgfs[fngfs + ShellPatch::dRdyy],
cg->fgfs[fngfs + ShellPatch::dRdyz],
cg->fgfs[fngfs + ShellPatch::dRdzz],
cg->fgfs[phi0->sgfn],
cg->fgfs[gxx0->sgfn], cg->fgfs[gxy0->sgfn], cg->fgfs[gxz0->sgfn],
cg->fgfs[gyy0->sgfn], cg->fgfs[gyz0->sgfn], cg->fgfs[gzz0->sgfn],
cg->fgfs[Gmx0->sgfn], cg->fgfs[Gmy0->sgfn], cg->fgfs[Gmz0->sgfn],
cg->fgfs[Gamxxx->sgfn], cg->fgfs[Gamxxy->sgfn], cg->fgfs[Gamxxz->sgfn],
cg->fgfs[Gamxyy->sgfn], cg->fgfs[Gamxyz->sgfn], cg->fgfs[Gamxzz->sgfn],
cg->fgfs[Gamyxx->sgfn], cg->fgfs[Gamyxy->sgfn], cg->fgfs[Gamyxz->sgfn],
cg->fgfs[Gamyyy->sgfn], cg->fgfs[Gamyyz->sgfn], cg->fgfs[Gamyzz->sgfn],
cg->fgfs[Gamzxx->sgfn], cg->fgfs[Gamzxy->sgfn], cg->fgfs[Gamzxz->sgfn],
cg->fgfs[Gamzyy->sgfn], cg->fgfs[Gamzyz->sgfn], cg->fgfs[Gamzzz->sgfn],
cg->fgfs[Rxx->sgfn], cg->fgfs[Rxy->sgfn], cg->fgfs[Rxz->sgfn],
cg->fgfs[Ryy->sgfn], cg->fgfs[Ryz->sgfn], cg->fgfs[Rzz->sgfn],
Symmetry, lev, Pp->data->sst);
f_getnp4_ss(cg->shape, cg->X[0], cg->X[1], cg->X[2],
cg->fgfs[fngfs + ShellPatch::gx],
cg->fgfs[fngfs + ShellPatch::gy],
cg->fgfs[fngfs + ShellPatch::gz],
cg->fgfs[fngfs + ShellPatch::drhodx],
cg->fgfs[fngfs + ShellPatch::drhody],
cg->fgfs[fngfs + ShellPatch::drhodz],
cg->fgfs[fngfs + ShellPatch::dsigmadx],
cg->fgfs[fngfs + ShellPatch::dsigmady],
cg->fgfs[fngfs + ShellPatch::dsigmadz],
cg->fgfs[fngfs + ShellPatch::dRdx],
cg->fgfs[fngfs + ShellPatch::dRdy],
cg->fgfs[fngfs + ShellPatch::dRdz],
cg->fgfs[fngfs + ShellPatch::drhodxx],
cg->fgfs[fngfs + ShellPatch::drhodxy],
cg->fgfs[fngfs + ShellPatch::drhodxz],
cg->fgfs[fngfs + ShellPatch::drhodyy],
cg->fgfs[fngfs + ShellPatch::drhodyz],
cg->fgfs[fngfs + ShellPatch::drhodzz],
cg->fgfs[fngfs + ShellPatch::dsigmadxx],
cg->fgfs[fngfs + ShellPatch::dsigmadxy],
cg->fgfs[fngfs + ShellPatch::dsigmadxz],
cg->fgfs[fngfs + ShellPatch::dsigmadyy],
cg->fgfs[fngfs + ShellPatch::dsigmadyz],
cg->fgfs[fngfs + ShellPatch::dsigmadzz],
cg->fgfs[fngfs + ShellPatch::dRdxx],
cg->fgfs[fngfs + ShellPatch::dRdxy],
cg->fgfs[fngfs + ShellPatch::dRdxz],
cg->fgfs[fngfs + ShellPatch::dRdyy],
cg->fgfs[fngfs + ShellPatch::dRdyz],
cg->fgfs[fngfs + ShellPatch::dRdzz],
cg->fgfs[phi0->sgfn], cg->fgfs[trK0->sgfn],
cg->fgfs[gxx0->sgfn], cg->fgfs[gxy0->sgfn], cg->fgfs[gxz0->sgfn],
cg->fgfs[gyy0->sgfn], cg->fgfs[gyz0->sgfn], cg->fgfs[gzz0->sgfn],
cg->fgfs[Axx0->sgfn], cg->fgfs[Axy0->sgfn], cg->fgfs[Axz0->sgfn],
cg->fgfs[Ayy0->sgfn], cg->fgfs[Ayz0->sgfn], cg->fgfs[Azz0->sgfn],
cg->fgfs[Gamxxx->sgfn], cg->fgfs[Gamxxy->sgfn], cg->fgfs[Gamxxz->sgfn],
cg->fgfs[Gamxyy->sgfn], cg->fgfs[Gamxyz->sgfn], cg->fgfs[Gamxzz->sgfn],
cg->fgfs[Gamyxx->sgfn], cg->fgfs[Gamyxy->sgfn], cg->fgfs[Gamyxz->sgfn],
cg->fgfs[Gamyyy->sgfn], cg->fgfs[Gamyyz->sgfn], cg->fgfs[Gamyzz->sgfn],
cg->fgfs[Gamzxx->sgfn], cg->fgfs[Gamzxy->sgfn], cg->fgfs[Gamzxz->sgfn],
cg->fgfs[Gamzyy->sgfn], cg->fgfs[Gamzyz->sgfn], cg->fgfs[Gamzzz->sgfn],
cg->fgfs[Rxx->sgfn], cg->fgfs[Rxy->sgfn], cg->fgfs[Rxz->sgfn],
cg->fgfs[Ryy->sgfn], cg->fgfs[Ryz->sgfn], cg->fgfs[Rzz->sgfn],
cg->fgfs[Rpsi4->sgfn], cg->fgfs[Ipsi4->sgfn],
Symmetry, Pp->data->sst);
#elif (Psi4type == 1)
f_getnp4old_ss(cg->shape, cg->X[0], cg->X[1], cg->X[2],
cg->fgfs[fngfs + ShellPatch::gx],
cg->fgfs[fngfs + ShellPatch::gy],
cg->fgfs[fngfs + ShellPatch::gz],
cg->fgfs[fngfs + ShellPatch::drhodx],
cg->fgfs[fngfs + ShellPatch::drhody],
cg->fgfs[fngfs + ShellPatch::drhodz],
cg->fgfs[fngfs + ShellPatch::dsigmadx],
cg->fgfs[fngfs + ShellPatch::dsigmady],
cg->fgfs[fngfs + ShellPatch::dsigmadz],
cg->fgfs[fngfs + ShellPatch::dRdx],
cg->fgfs[fngfs + ShellPatch::dRdy],
cg->fgfs[fngfs + ShellPatch::dRdz],
cg->fgfs[fngfs + ShellPatch::drhodxx],
cg->fgfs[fngfs + ShellPatch::drhodxy],
cg->fgfs[fngfs + ShellPatch::drhodxz],
cg->fgfs[fngfs + ShellPatch::drhodyy],
cg->fgfs[fngfs + ShellPatch::drhodyz],
cg->fgfs[fngfs + ShellPatch::drhodzz],
cg->fgfs[fngfs + ShellPatch::dsigmadxx],
cg->fgfs[fngfs + ShellPatch::dsigmadxy],
cg->fgfs[fngfs + ShellPatch::dsigmadxz],
cg->fgfs[fngfs + ShellPatch::dsigmadyy],
cg->fgfs[fngfs + ShellPatch::dsigmadyz],
cg->fgfs[fngfs + ShellPatch::dsigmadzz],
cg->fgfs[fngfs + ShellPatch::dRdxx],
cg->fgfs[fngfs + ShellPatch::dRdxy],
cg->fgfs[fngfs + ShellPatch::dRdxz],
cg->fgfs[fngfs + ShellPatch::dRdyy],
cg->fgfs[fngfs + ShellPatch::dRdyz],
cg->fgfs[fngfs + ShellPatch::dRdzz],
cg->fgfs[phi0->sgfn], cg->fgfs[trK0->sgfn],
cg->fgfs[gxx0->sgfn], cg->fgfs[gxy0->sgfn], cg->fgfs[gxz0->sgfn],
cg->fgfs[gyy0->sgfn], cg->fgfs[gyz0->sgfn], cg->fgfs[gzz0->sgfn],
cg->fgfs[Axx0->sgfn], cg->fgfs[Axy0->sgfn], cg->fgfs[Axz0->sgfn],
cg->fgfs[Ayy0->sgfn], cg->fgfs[Ayz0->sgfn], cg->fgfs[Azz0->sgfn],
cg->fgfs[Gmx0->sgfn], cg->fgfs[Gmy0->sgfn], cg->fgfs[Gmz0->sgfn],
cg->fgfs[Lap0->sgfn],
cg->fgfs[Sfx0->sgfn], cg->fgfs[Sfy0->sgfn], cg->fgfs[Sfz0->sgfn],
cg->fgfs[Rpsi4->sgfn], cg->fgfs[Ipsi4->sgfn],
Symmetry, Pp->data->sst);
#else
#error "not recognized Psi4type"
#endif
}
if (BL == Pp->data->ble)
break;
BL = BL->next;
}
Pp = Pp->next;
}
SH->Synch(DG_List, Symmetry);
#if 0
// interpolate Psi4
SH->CS_Inter(DG_List,Symmetry);
#endif
}
#endif
DG_List->clearList();
// misc::tillherecheck(GH->Commlev[lev],GH->start_rank[lev],"end of Compute_Psi4");
}
//================================================================================================
//================================================================================================
// This member function sets the black holes' initial puncture positions
//================================================================================================
void bssn_class::Setup_Black_Hole_position()
{
char filename[50];
{
map<string, string>::iterator iter = parameters::str_par.find("inputpar");
if (iter != parameters::str_par.end())
{
strcpy(filename, (iter->second).c_str());
}
else
{
cout << "Error inputpar" << endl;
exit(0);
}
}
// read parameter from file
{
const int LEN = 256;
char pline[LEN];
string str, sgrp, skey, sval;
int sind;
ifstream inf(filename, ifstream::in);
if (!inf.good() && myrank == 0)
{
if (ErrorMonitor->outfile)
ErrorMonitor->outfile << "Can not open parameter file " << filename
<< " for inputing information of black holes" << endl;
MPI_Abort(MPI_COMM_WORLD, 1);
}
for (int i = 1; inf.good(); i++)
{
inf.getline(pline, LEN);
str = pline;
int status = misc::parse_parts(str, sgrp, skey, sval, sind);
if (status == -1)
{
if (ErrorMonitor->outfile)
ErrorMonitor->outfile << "error reading parameter file " << filename << " in line " << i << endl;
MPI_Abort(MPI_COMM_WORLD, 1);
}
else if (status == 0)
continue;
if (sgrp == "BSSN" && skey == "BH_num")
{
BH_num_input = BH_num = atoi(sval.c_str());
break;
}
}
inf.close();
}
// set up the data for black holes
// these arrays will be deleted when bssn_class is deleted
Pmom = new double[3 * BH_num];
Spin = new double[3 * BH_num];
Mass = new double[BH_num];
Porg0 = new double *[BH_num];
Porgbr = new double *[BH_num];
Porg = new double *[BH_num];
Porg1 = new double *[BH_num];
Porg_rhs = new double *[BH_num];
for (int i = 0; i < BH_num; i++)
{
Porg0[i] = new double[3];
Porgbr[i] = new double[3];
Porg[i] = new double[3];
Porg1[i] = new double[3];
Porg_rhs[i] = new double[3];
}
// read parameter from file
{
const int LEN = 256;
char pline[LEN];
string str, sgrp, skey, sval;
int sind;
ifstream inf(filename, ifstream::in);
if (!inf.good() && myrank == 0)
{
if (ErrorMonitor->outfile)
ErrorMonitor->outfile << "Can not open parameter file " << filename
<< " for inputing information of black holes" << endl;
MPI_Abort(MPI_COMM_WORLD, 1);
}
for (int i = 1; inf.good(); i++)
{
inf.getline(pline, LEN);
str = pline;
int status = misc::parse_parts(str, sgrp, skey, sval, sind);
if (status == -1)
{
if (ErrorMonitor->outfile)
ErrorMonitor->outfile << "error reading parameter file " << filename << " in line " << i << endl;
MPI_Abort(MPI_COMM_WORLD, 1);
}
else if (status == 0)
continue;
if (sgrp == "BSSN" && sind < BH_num)
{
if (skey == "Mass")
Mass[sind] = atof(sval.c_str());
else if (skey == "Porgx")
Porg0[sind][0] = atof(sval.c_str());
else if (skey == "Porgy")
Porg0[sind][1] = atof(sval.c_str());
else if (skey == "Porgz")
Porg0[sind][2] = atof(sval.c_str());
else if (skey == "Spinx")
Spin[sind * 3] = atof(sval.c_str());
else if (skey == "Spiny")
Spin[sind * 3 + 1] = atof(sval.c_str());
else if (skey == "Spinz")
Spin[sind * 3 + 2] = atof(sval.c_str());
else if (skey == "Pmomx")
Pmom[sind * 3] = atof(sval.c_str());
else if (skey == "Pmomy")
Pmom[sind * 3 + 1] = atof(sval.c_str());
else if (skey == "Pmomz")
Pmom[sind * 3 + 2] = atof(sval.c_str());
}
}
inf.close();
}
// echo information of Black holes
if (myrank == 0)
{
cout << endl;
cout << " initial information of " << BH_num << " Black Hole(s) " << endl;
cout << setw(12) << "Mass"
<< setw(12) << "x"
<< setw(12) << "y"
<< setw(12) << "z"
<< setw(16) << "Px"
<< setw(16) << "Py"
<< setw(12) << "Pz"
<< setw(12) << "Sx"
<< setw(12) << "Sy"
<< setw(12) << "Sz" << endl;
for (int i = 0; i < BH_num; i++)
{
cout << setw(12) << Mass[i]
<< setw(12) << Porg0[i][0]
<< setw(12) << Porg0[i][1]
<< setw(12) << Porg0[i][2]
<< setw(16) << Pmom[i * 3]
<< setw(16) << Pmom[i * 3 + 1]
<< setw(12) << Pmom[i * 3 + 2]
<< setw(12) << Spin[i * 3]
<< setw(12) << Spin[i * 3 + 1]
<< setw(12) << Spin[i * 3 + 2] << endl;
}
}
int maxl = 1;
int levels;
int *grids;
double bbox[6];
// read parameter from file
{
const int LEN = 256;
char pline[LEN];
string str, sgrp, skey, sval;
int sind1, sind2, sind3;
ifstream inf(filename, ifstream::in);
if (!inf.good() && myrank == 0)
{
cout << "bssn_class::Setup_Black_Hole_position: Can not open parameter file " << filename
<< " for inputing information of black holes" << endl;
MPI_Abort(MPI_COMM_WORLD, 1);
}
for (int i = 1; inf.good(); i++)
{
inf.getline(pline, LEN);
str = pline;
int status = misc::parse_parts(str, sgrp, skey, sval, sind1);
if (status == -1)
{
cout << "error reading parameter file " << filename << " in line " << i << endl;
MPI_Abort(MPI_COMM_WORLD, 1);
}
else if (status == 0)
continue;
if (sgrp == "cgh" && skey == "levels")
{
levels = atoi(sval.c_str());
break;
}
}
inf.close();
}
grids = new int[levels];
// read parameter from file
{
const int LEN = 256;
char pline[LEN];
string str, sgrp, skey, sval;
int sind1, sind2, sind3;
ifstream inf(filename, ifstream::in);
if (!inf.good() && myrank == 0)
{
cout << "bssn_class::Setup_Black_Hole_position: Can not open parameter file " << filename
<< " for inputing information of black holes" << endl;
MPI_Abort(MPI_COMM_WORLD, 1);
}
for (int i = 1; inf.good(); i++)
{
inf.getline(pline, LEN);
str = pline;
int status = misc::parse_parts(str, sgrp, skey, sval, sind1, sind2, sind3);
if (status == -1)
{
cout << "error reading parameter file " << filename << " in line " << i << endl;
MPI_Abort(MPI_COMM_WORLD, 1);
}
else if (status == 0)
continue;
if (sgrp == "cgh" && skey == "grids" && sind1 < levels)
grids[sind1] = atoi(sval.c_str());
if (sgrp == "cgh" && skey == "bbox" && sind1 == 0 && sind2 == 0)
bbox[sind3] = atof(sval.c_str());
}
inf.close();
}
for (int i = 0; i < levels; i++)
if (maxl < grids[i])
maxl = grids[i];
delete[] grids;
if (BH_num > maxl)
{
int BH_numc = BH_num;
for (int i = 0; i < BH_num; i++)
if (Porg0[i][0] < bbox[0] || Porg0[i][0] > bbox[3] ||
Porg0[i][1] < bbox[1] || Porg0[i][1] > bbox[4] ||
Porg0[i][2] < bbox[2] || Porg0[i][2] > bbox[5])
{
delete[] Porg0[i];
Porg0[i] = 0;
BH_numc--;
}
if (BH_num > BH_numc)
{
maxl = BH_numc;
int bhi;
double *tmp;
tmp = Pmom;
Pmom = new double[3 * maxl];
bhi = 0;
for (int i = 0; i < BH_num; i++)
if (Porg0[i])
{
for (int j = 0; j < 3; j++)
Pmom[3 * bhi + j] = tmp[3 * i + j];
bhi++;
}
delete[] tmp;
tmp = Spin;
Spin = new double[3 * maxl];
bhi = 0;
for (int i = 0; i < BH_num; i++)
if (Porg0[i])
{
for (int j = 0; j < 3; j++)
Spin[3 * bhi + j] = tmp[3 * i + j];
bhi++;
}
delete[] tmp;
tmp = Mass;
Mass = new double[3 * maxl];
bhi = 0;
for (int i = 0; i < BH_num; i++)
if (Porg0[i])
{
Mass[bhi] = tmp[i];
bhi++;
}
delete[] tmp;
double **ttmp;
ttmp = Porg0;
Porg0 = new double *[maxl];
bhi = 0;
for (int i = 0; i < BH_num; i++)
if (ttmp[i])
{
Porg0[bhi] = ttmp[i];
bhi++;
}
delete[] ttmp;
for (int i = 0; i < BH_num; i++)
{
delete[] Porgbr[i];
delete[] Porg[i];
delete[] Porg1[i];
delete[] Porg_rhs[i];
}
delete[] Porgbr;
delete[] Porg;
delete[] Porg1;
delete[] Porg_rhs;
BH_num = maxl;
Porgbr = new double *[BH_num];
Porg = new double *[BH_num];
Porg1 = new double *[BH_num];
Porg_rhs = new double *[BH_num];
for (int i = 0; i < BH_num; i++)
{
Porgbr[i] = new double[3];
Porg[i] = new double[3];
Porg1[i] = new double[3];
Porg_rhs[i] = new double[3];
}
}
}
for (int i = 0; i < BH_num; i++)
{
for (int j = 0; j < dim; j++)
Porgbr[i][j] = Porg0[i][j];
}
setpbh(BH_num, Porg0, Mass, BH_num_input);
}
//================================================================================================
//================================================================================================
// This member function computes black hole positions
//================================================================================================
#if 0
// old code
void bssn_class::compute_Porg_rhs(double **BH_PS,double **BH_RHS,var *forx,var *fory,var *forz,int lev)
{
const int InList = 3;
MyList<var> * DG_List=new MyList<var>(forx);
DG_List->insert(fory); DG_List->insert(forz);
int n;
double *x1,*y1,*z1;
double *shellf;
shellf=new double[3*BH_num];
double *pox[3];
for(int i=0;i<3;i++) pox[i] = new double[BH_num];
for( n = 0; n < BH_num; n++)
{
pox[0][n] = BH_PS[n][0];
pox[1][n] = BH_PS[n][1];
pox[2][n] = BH_PS[n][2];
}
if(!Parallel::PatList_Interp_Points(GH->PatL[lev],DG_List,BH_num,pox,shellf,Symmetry))
{
ErrorMonitor->outfile<<"fail to find black holes at t = "<<PhysTime<<endl;
for( n = 0; n < BH_num; n++)
ErrorMonitor->outfile<<"(x,y,z) = ("<<pox[0][n]<<","<<pox[1][n]<<","<<pox[2][n]<<")"<<endl;
}
for( n = 0; n < BH_num; n++)
{
BH_RHS[n][0]=-shellf[3*n ];
BH_RHS[n][1]=-shellf[3*n+1];
BH_RHS[n][2]=-shellf[3*n+2];
}
DG_List->clearList();
delete[] shellf;
for(int i=0;i<3;i++) delete[] pox[i];
}
#else
// new code considering diferent levels for different black hole
void bssn_class::compute_Porg_rhs(double **BH_PS, double **BH_RHS, var *forx, var *fory, var *forz, int ilev)
{
const int InList = 3;
MyList<var> *DG_List = new MyList<var>(forx);
DG_List->insert(fory);
DG_List->insert(forz);
double *x1, *y1, *z1;
double *shellf;
shellf = new double[3];
double *pox[3];
for (int i = 0; i < 3; i++)
pox[i] = new double[1];
for (int n = 0; n < BH_num; n++)
{
pox[0][0] = BH_PS[n][0];
pox[1][0] = BH_PS[n][1];
pox[2][0] = BH_PS[n][2];
int lev = ilev;
#if (PSTR == 0)
while (!Parallel::PatList_Interp_Points(GH->PatL[lev], DG_List, 1, pox, shellf, Symmetry))
#elif (PSTR == 1 || PSTR == 2 || PSTR == 3)
while (!Parallel::PatList_Interp_Points(GH->PatL[lev], DG_List, 1, pox, shellf, Symmetry, GH->Commlev[lev]))
#endif
{
lev--;
if (lev < 0)
{
ErrorMonitor->outfile << "fail to find black holes at t = " << PhysTime << endl;
for (n = 0; n < BH_num; n++)
ErrorMonitor->outfile << "(x,y,z) = ("
<< pox[0][n] << "," << pox[1][n] << "," << pox[2][n]
<< ")" << endl;
break;
}
}
if (lev >= 0)
{
BH_RHS[n][0] = -shellf[0];
BH_RHS[n][1] = -shellf[1];
BH_RHS[n][2] = -shellf[2];
}
}
DG_List->clearList();
delete[] shellf;
for (int i = 0; i < 3; i++)
delete[] pox[i];
}
#endif
//================================================================================================
//================================================================================================
// This member function computes gravitational-wave related quantities and performs analysis
//================================================================================================
void bssn_class::AnalysisStuff(int lev, double dT_lev)
{
LastAnas += dT_lev;
if (LastAnas >= AnasTime)
{
#ifdef Point_Psi4
#error "not support parallel levels yet"
// Gam_ijk and R_ij have been calculated in Interp_Constraint()
double SYM = 1, ANT = -1;
for (int levh = lev; levh < GH->levels; levh++)
{
MyList<Patch> *Pp = GH->PatL[levh];
while (Pp)
{
MyList<Block> *BP = Pp->data->blb;
while (BP)
{
Block *cg = BP->data;
if (myrank == cg->rank)
{
f_fderivs(cg->shape, cg->fgfs[phi0->sgfn],
cg->fgfs[phix->sgfn], cg->fgfs[phiy->sgfn], cg->fgfs[phiz->sgfn],
cg->X[0], cg->X[1], cg->X[2],
SYM, SYM, SYM, Symmetry, levh);
f_fderivs(cg->shape, cg->fgfs[trK0->sgfn],
cg->fgfs[trKx->sgfn], cg->fgfs[trKy->sgfn], cg->fgfs[trKz->sgfn],
cg->X[0], cg->X[1], cg->X[2],
SYM, SYM, SYM, Symmetry, levh);
f_fderivs(cg->shape, cg->fgfs[Axx0->sgfn],
cg->fgfs[Axxx->sgfn], cg->fgfs[Axxy->sgfn], cg->fgfs[Axxz->sgfn],
cg->X[0], cg->X[1], cg->X[2],
SYM, SYM, SYM, Symmetry, levh);
f_fderivs(cg->shape, cg->fgfs[Axy0->sgfn],
cg->fgfs[Axyx->sgfn], cg->fgfs[Axyy->sgfn], cg->fgfs[Axyz->sgfn],
cg->X[0], cg->X[1], cg->X[2],
ANT, ANT, SYM, Symmetry, levh);
f_fderivs(cg->shape, cg->fgfs[Axz0->sgfn],
cg->fgfs[Axzx->sgfn], cg->fgfs[Axzy->sgfn], cg->fgfs[Axzz->sgfn],
cg->X[0], cg->X[1], cg->X[2],
ANT, SYM, ANT, Symmetry, levh);
f_fderivs(cg->shape, cg->fgfs[Ayy0->sgfn],
cg->fgfs[Ayyx->sgfn], cg->fgfs[Ayyy->sgfn], cg->fgfs[Ayyz->sgfn],
cg->X[0], cg->X[1], cg->X[2],
SYM, SYM, SYM, Symmetry, levh);
f_fderivs(cg->shape, cg->fgfs[Ayz0->sgfn],
cg->fgfs[Ayzx->sgfn], cg->fgfs[Ayzy->sgfn], cg->fgfs[Ayzz->sgfn],
cg->X[0], cg->X[1], cg->X[2],
SYM, ANT, ANT, Symmetry, levh);
f_fderivs(cg->shape, cg->fgfs[Azz0->sgfn],
cg->fgfs[Azzx->sgfn], cg->fgfs[Azzy->sgfn], cg->fgfs[Azzz->sgfn],
cg->X[0], cg->X[1], cg->X[2],
SYM, SYM, SYM, Symmetry, levh);
}
if (BP == Pp->data->ble)
break;
BP = BP->next;
}
Pp = Pp->next;
}
#ifdef WithShell
// ShellPatch part
if (lev == 0)
{
MyList<ss_patch> *Pp = SH->PatL;
while (Pp)
{
MyList<Block> *BL = Pp->data->blb;
int fngfs = Pp->data->fngfs;
while (BL)
{
Block *cg = BL->data;
if (myrank == cg->rank)
{
f_fderivs_shc(cg->shape, cg->fgfs[phi0->sgfn],
cg->fgfs[phix->sgfn], cg->fgfs[phiy->sgfn], cg->fgfs[phiz->sgfn],
cg->X[0], cg->X[1], cg->X[2],
phi0->SoA[0], phi0->SoA[1], phi0->SoA[2],
Symmetry, levh, Pp->data->sst,
cg->fgfs[fngfs + ShellPatch::drhodx],
cg->fgfs[fngfs + ShellPatch::drhody],
cg->fgfs[fngfs + ShellPatch::drhodz],
cg->fgfs[fngfs + ShellPatch::dsigmadx],
cg->fgfs[fngfs + ShellPatch::dsigmady],
cg->fgfs[fngfs + ShellPatch::dsigmadz],
cg->fgfs[fngfs + ShellPatch::dRdx],
cg->fgfs[fngfs + ShellPatch::dRdy],
cg->fgfs[fngfs + ShellPatch::dRdz]);
f_fderivs_shc(cg->shape, cg->fgfs[trK0->sgfn],
cg->fgfs[trKx->sgfn], cg->fgfs[trKy->sgfn], cg->fgfs[trKz->sgfn],
cg->X[0], cg->X[1], cg->X[2],
trK0->SoA[0], trK0->SoA[1], trK0->SoA[2],
Symmetry, levh, Pp->data->sst,
cg->fgfs[fngfs + ShellPatch::drhodx],
cg->fgfs[fngfs + ShellPatch::drhody],
cg->fgfs[fngfs + ShellPatch::drhodz],
cg->fgfs[fngfs + ShellPatch::dsigmadx],
cg->fgfs[fngfs + ShellPatch::dsigmady],
cg->fgfs[fngfs + ShellPatch::dsigmadz],
cg->fgfs[fngfs + ShellPatch::dRdx],
cg->fgfs[fngfs + ShellPatch::dRdy],
cg->fgfs[fngfs + ShellPatch::dRdz]);
f_fderivs_shc(cg->shape, cg->fgfs[Axx0->sgfn],
cg->fgfs[Axxx->sgfn], cg->fgfs[Axxy->sgfn], cg->fgfs[Axxz->sgfn],
cg->X[0], cg->X[1], cg->X[2],
Axx0->SoA[0], Axx0->SoA[1], Axx0->SoA[2],
Symmetry, levh, Pp->data->sst,
cg->fgfs[fngfs + ShellPatch::drhodx],
cg->fgfs[fngfs + ShellPatch::drhody],
cg->fgfs[fngfs + ShellPatch::drhodz],
cg->fgfs[fngfs + ShellPatch::dsigmadx],
cg->fgfs[fngfs + ShellPatch::dsigmady],
cg->fgfs[fngfs + ShellPatch::dsigmadz],
cg->fgfs[fngfs + ShellPatch::dRdx],
cg->fgfs[fngfs + ShellPatch::dRdy],
cg->fgfs[fngfs + ShellPatch::dRdz]);
f_fderivs_shc(cg->shape, cg->fgfs[Axy0->sgfn],
cg->fgfs[Axyx->sgfn], cg->fgfs[Axyy->sgfn], cg->fgfs[Axyz->sgfn],
cg->X[0], cg->X[1], cg->X[2],
Axy0->SoA[0], Axy0->SoA[1], Axy0->SoA[2],
Symmetry, levh, Pp->data->sst,
cg->fgfs[fngfs + ShellPatch::drhodx],
cg->fgfs[fngfs + ShellPatch::drhody],
cg->fgfs[fngfs + ShellPatch::drhodz],
cg->fgfs[fngfs + ShellPatch::dsigmadx],
cg->fgfs[fngfs + ShellPatch::dsigmady],
cg->fgfs[fngfs + ShellPatch::dsigmadz],
cg->fgfs[fngfs + ShellPatch::dRdx],
cg->fgfs[fngfs + ShellPatch::dRdy],
cg->fgfs[fngfs + ShellPatch::dRdz]);
f_fderivs_shc(cg->shape, cg->fgfs[Axz0->sgfn],
cg->fgfs[Axzx->sgfn], cg->fgfs[Axzy->sgfn], cg->fgfs[Axzz->sgfn],
cg->X[0], cg->X[1], cg->X[2],
Axz0->SoA[0], Axz0->SoA[1], Axz0->SoA[2],
Symmetry, levh, Pp->data->sst,
cg->fgfs[fngfs + ShellPatch::drhodx],
cg->fgfs[fngfs + ShellPatch::drhody],
cg->fgfs[fngfs + ShellPatch::drhodz],
cg->fgfs[fngfs + ShellPatch::dsigmadx],
cg->fgfs[fngfs + ShellPatch::dsigmady],
cg->fgfs[fngfs + ShellPatch::dsigmadz],
cg->fgfs[fngfs + ShellPatch::dRdx],
cg->fgfs[fngfs + ShellPatch::dRdy],
cg->fgfs[fngfs + ShellPatch::dRdz]);
f_fderivs_shc(cg->shape, cg->fgfs[Ayy0->sgfn],
cg->fgfs[Ayyx->sgfn], cg->fgfs[Ayyy->sgfn], cg->fgfs[Ayyz->sgfn],
cg->X[0], cg->X[1], cg->X[2],
Ayy0->SoA[0], Ayy0->SoA[1], Ayy0->SoA[2],
Symmetry, levh, Pp->data->sst,
cg->fgfs[fngfs + ShellPatch::drhodx],
cg->fgfs[fngfs + ShellPatch::drhody],
cg->fgfs[fngfs + ShellPatch::drhodz],
cg->fgfs[fngfs + ShellPatch::dsigmadx],
cg->fgfs[fngfs + ShellPatch::dsigmady],
cg->fgfs[fngfs + ShellPatch::dsigmadz],
cg->fgfs[fngfs + ShellPatch::dRdx],
cg->fgfs[fngfs + ShellPatch::dRdy],
cg->fgfs[fngfs + ShellPatch::dRdz]);
f_fderivs_shc(cg->shape, cg->fgfs[Ayz0->sgfn],
cg->fgfs[Ayzx->sgfn], cg->fgfs[Ayzy->sgfn], cg->fgfs[Ayzz->sgfn],
cg->X[0], cg->X[1], cg->X[2],
Ayz0->SoA[0], Ayz0->SoA[1], Ayz0->SoA[2],
Symmetry, levh, Pp->data->sst,
cg->fgfs[fngfs + ShellPatch::drhodx],
cg->fgfs[fngfs + ShellPatch::drhody],
cg->fgfs[fngfs + ShellPatch::drhodz],
cg->fgfs[fngfs + ShellPatch::dsigmadx],
cg->fgfs[fngfs + ShellPatch::dsigmady],
cg->fgfs[fngfs + ShellPatch::dsigmadz],
cg->fgfs[fngfs + ShellPatch::dRdx],
cg->fgfs[fngfs + ShellPatch::dRdy],
cg->fgfs[fngfs + ShellPatch::dRdz]);
f_fderivs_shc(cg->shape, cg->fgfs[Azz0->sgfn],
cg->fgfs[Azzx->sgfn], cg->fgfs[Azzy->sgfn], cg->fgfs[Azzz->sgfn],
cg->X[0], cg->X[1], cg->X[2],
Azz0->SoA[0], Azz0->SoA[1], Azz0->SoA[2],
Symmetry, levh, Pp->data->sst,
cg->fgfs[fngfs + ShellPatch::drhodx],
cg->fgfs[fngfs + ShellPatch::drhody],
cg->fgfs[fngfs + ShellPatch::drhodz],
cg->fgfs[fngfs + ShellPatch::dsigmadx],
cg->fgfs[fngfs + ShellPatch::dsigmady],
cg->fgfs[fngfs + ShellPatch::dsigmadz],
cg->fgfs[fngfs + ShellPatch::dRdx],
cg->fgfs[fngfs + ShellPatch::dRdy],
cg->fgfs[fngfs + ShellPatch::dRdz]);
}
if (BL == Pp->data->ble)
break;
BL = BL->next;
}
Pp = Pp->next;
}
}
#endif
}
#else
Compute_Psi4(lev);
#endif
double *RP, *IP, *RoutMAP;
int NN = 0;
for (int pl = 2; pl < maxl + 1; pl++)
for (int pm = -pl; pm < pl + 1; pm++)
NN++;
RP = new double[NN];
IP = new double[NN];
RoutMAP = new double[7];
double Rex = maxrex;
for (int i = 0; i < decn; i++)
{
#ifdef Point_Psi4
Waveshell->surf_Wave(Rex, GH, SH,
phi, trK,
gxx0, gxy0, gxz0, gyy0, gyz0, gzz0,
Axx0, Axy0, Axz0, Ayy0, Ayz0, Azz0,
phix, phiy, phiz,
trKx, trKy, trKz,
Axxx, Axxy, Axxz,
Axyx, Axyy, Axyz,
Axzx, Axzy, Axzz,
Ayyx, Ayyy, Ayyz,
Ayzx, Ayzy, Ayzz,
Azzx, Azzy, Azzz,
Gamxxx, Gamxxy, Gamxxz, Gamxyy, Gamxyz, Gamxzz,
Gamyxx, Gamyxy, Gamyxz, Gamyyy, Gamyyz, Gamyzz,
Gamzxx, Gamzxy, Gamzxz, Gamzyy, Gamzyz, Gamzzz,
Rxx, Rxy, Rxz, Ryy, Ryz, Rzz,
2, maxl, NN, RP, IP, ErrorMonitor);
#ifdef WithShell
if (lev > 0 || Rex < GH->bbox[0][0][3])
{
Waveshell->surf_MassPAng(Rex, lev, GH, phi0, trK0,
gxx0, gxy0, gxz0, gyy0, gyz0, gzz0,
Axx0, Axy0, Axz0, Ayy0, Ayz0, Azz0,
Gmx0, Gmy0, Gmz0, Sfx1, Sfy1, Sfz1, // here we can not touch rhs variables, but 1 variables
RoutMAP, ErrorMonitor);
}
else
{
Waveshell->surf_MassPAng(Rex, lev, SH, phi0, trK0,
gxx0, gxy0, gxz0, gyy0, gyz0, gzz0,
Axx0, Axy0, Axz0, Ayy0, Ayz0, Azz0,
Gmx0, Gmy0, Gmz0, Sfx1, Sfy1, Sfz1, // here we can not touch rhs variables, but 1 variables
RoutMAP, ErrorMonitor);
}
#else
Waveshell->surf_MassPAng(Rex, lev, GH, phi0, trK0,
gxx0, gxy0, gxz0, gyy0, gyz0, gzz0,
Axx0, Axy0, Axz0, Ayy0, Ayz0, Azz0,
Gmx0, Gmy0, Gmz0, Sfx1, Sfy1, Sfz1, // here we can not touch rhs variables, but 1 variables
RoutMAP, ErrorMonitor);
#endif
#else
// misc::tillherecheck(GH->Commlev[lev],GH->start_rank[lev],"before surface integral");
#ifdef WithShell
if (lev > 0 || Rex < GH->bbox[0][0][3])
{
Waveshell->surf_Wave(Rex, lev, GH, Rpsi4, Ipsi4, 2, maxl, NN, RP, IP, ErrorMonitor);
Waveshell->surf_MassPAng(Rex, lev, GH, phi0, trK0,
gxx0, gxy0, gxz0, gyy0, gyz0, gzz0,
Axx0, Axy0, Axz0, Ayy0, Ayz0, Azz0,
Gmx0, Gmy0, Gmz0, Sfx1, Sfy1, Sfz1, // here we can not touch rhs variables, but 1 variables
RoutMAP, ErrorMonitor);
}
else
{
Waveshell->surf_Wave(Rex, lev, SH, Rpsi4, Ipsi4, 2, maxl, NN, RP, IP, ErrorMonitor);
Waveshell->surf_MassPAng(Rex, lev, SH, phi0, trK0,
gxx0, gxy0, gxz0, gyy0, gyz0, gzz0,
Axx0, Axy0, Axz0, Ayy0, Ayz0, Azz0,
Gmx0, Gmy0, Gmz0, Sfx1, Sfy1, Sfz1, // here we can not touch rhs variables, but 1 variables
RoutMAP, ErrorMonitor);
}
#else
#if (PSTR == 0)
Waveshell->surf_Wave(Rex, lev, GH, Rpsi4, Ipsi4, 2, maxl, NN, RP, IP, ErrorMonitor);
Waveshell->surf_MassPAng(Rex, lev, GH, phi0, trK0,
gxx0, gxy0, gxz0, gyy0, gyz0, gzz0,
Axx0, Axy0, Axz0, Ayy0, Ayz0, Azz0,
Gmx0, Gmy0, Gmz0, Sfx1, Sfy1, Sfz1, // here we can not touch rhs variables, but 1 variables
RoutMAP, ErrorMonitor);
#elif (PSTR == 1 || PSTR == 2)
Waveshell->surf_Wave(Rex, lev, GH, Rpsi4, Ipsi4, 2, maxl, NN, RP, IP, ErrorMonitor, GH->Commlev[lev]);
// misc::tillherecheck(GH->Commlev[lev],GH->start_rank[lev],"after surf_Wave");
Waveshell->surf_MassPAng(Rex, lev, GH, phi0, trK0,
gxx0, gxy0, gxz0, gyy0, gyz0, gzz0,
Axx0, Axy0, Axz0, Ayy0, Ayz0, Azz0,
Gmx0, Gmy0, Gmz0, Sfx1, Sfy1, Sfz1, // here we can not touch rhs variables, but 1 variables
RoutMAP, ErrorMonitor, GH->Commlev[lev]);
#endif
#endif
// misc::tillherecheck(GH->Commlev[lev],GH->start_rank[lev],"end surface integral");
#endif
if (i == 0)
{
ADMMass = RoutMAP[0];
}
#if (PSTR == 1 || PSTR == 2)
if (GH->start_rank[a_lev] > 0)
{
MPI_Status status;
// receive
if (myrank == 0)
{
MPI_Recv(RP, NN, MPI_DOUBLE, GH->start_rank[a_lev], 1, MPI_COMM_WORLD, &status);
MPI_Recv(IP, NN, MPI_DOUBLE, GH->start_rank[a_lev], 2, MPI_COMM_WORLD, &status);
MPI_Recv(RoutMAP, 7, MPI_DOUBLE, GH->start_rank[a_lev], 3, MPI_COMM_WORLD, &status);
}
// send
if (myrank == GH->start_rank[a_lev])
{
MPI_Send(RP, NN, MPI_DOUBLE, 0, 1, MPI_COMM_WORLD);
MPI_Send(IP, NN, MPI_DOUBLE, 0, 2, MPI_COMM_WORLD);
MPI_Send(RoutMAP, 7, MPI_DOUBLE, 0, 3, MPI_COMM_WORLD);
}
}
#endif
Psi4Monitor->writefile(PhysTime, NN, RP, IP);
MAPMonitor->writefile(PhysTime, 7, RoutMAP);
Rex = Rex - drex;
}
delete[] RP;
delete[] IP;
delete[] RoutMAP;
// black hole's position
{
double *pox;
pox = new double[dim * BH_num];
for (int bhi = 0; bhi < BH_num; bhi++)
for (int i = 0; i < dim; i++)
pox[dim * bhi + i] = Porg0[bhi][i];
BHMonitor->writefile(PhysTime, dim * BH_num, pox);
delete[] pox;
}
LastAnas = 0;
}
}
//================================================================================================
//================================================================================================
// This member function computes and outputs constraint violations
//================================================================================================
void bssn_class::Constraint_Out()
{
LastConsOut += dT * pow(0.5, Mymax(0, trfls));
if (LastConsOut >= AnasTime)
// Constraint violation
{
// recompute least the constraint data lost for moved new grid
for (int lev = 0; lev < GH->levels; lev++)
{
// make sure the data consistent for higher levels
if (lev > 0) // if the constrait quantities can be reused from the step rhs calculation
{
double TRK4 = PhysTime;
double ndeps = numepsb;
int pre = 0;
MyList<Patch> *Pp = GH->PatL[lev];
while (Pp)
{
MyList<Block> *BP = Pp->data->blb;
while (BP)
{
Block *cg = BP->data;
if (myrank == cg->rank)
{
f_compute_rhs_bssn(cg->shape, TRK4, cg->X[0], cg->X[1], cg->X[2],
cg->fgfs[phi0->sgfn], cg->fgfs[trK0->sgfn],
cg->fgfs[gxx0->sgfn], cg->fgfs[gxy0->sgfn], cg->fgfs[gxz0->sgfn],
cg->fgfs[gyy0->sgfn], cg->fgfs[gyz0->sgfn], cg->fgfs[gzz0->sgfn],
cg->fgfs[Axx0->sgfn], cg->fgfs[Axy0->sgfn], cg->fgfs[Axz0->sgfn],
cg->fgfs[Ayy0->sgfn], cg->fgfs[Ayz0->sgfn], cg->fgfs[Azz0->sgfn],
cg->fgfs[Gmx0->sgfn], cg->fgfs[Gmy0->sgfn], cg->fgfs[Gmz0->sgfn],
cg->fgfs[Lap0->sgfn],
cg->fgfs[Sfx0->sgfn], cg->fgfs[Sfy0->sgfn], cg->fgfs[Sfz0->sgfn],
cg->fgfs[dtSfx0->sgfn], cg->fgfs[dtSfy0->sgfn], cg->fgfs[dtSfz0->sgfn],
cg->fgfs[phi_rhs->sgfn], cg->fgfs[trK_rhs->sgfn],
cg->fgfs[gxx_rhs->sgfn], cg->fgfs[gxy_rhs->sgfn], cg->fgfs[gxz_rhs->sgfn],
cg->fgfs[gyy_rhs->sgfn], cg->fgfs[gyz_rhs->sgfn], cg->fgfs[gzz_rhs->sgfn],
cg->fgfs[Axx_rhs->sgfn], cg->fgfs[Axy_rhs->sgfn], cg->fgfs[Axz_rhs->sgfn],
cg->fgfs[Ayy_rhs->sgfn], cg->fgfs[Ayz_rhs->sgfn], cg->fgfs[Azz_rhs->sgfn],
cg->fgfs[Gmx_rhs->sgfn], cg->fgfs[Gmy_rhs->sgfn], cg->fgfs[Gmz_rhs->sgfn],
cg->fgfs[Lap_rhs->sgfn],
cg->fgfs[Sfx_rhs->sgfn], cg->fgfs[Sfy_rhs->sgfn], cg->fgfs[Sfz_rhs->sgfn],
cg->fgfs[dtSfx_rhs->sgfn], cg->fgfs[dtSfy_rhs->sgfn], cg->fgfs[dtSfz_rhs->sgfn],
cg->fgfs[rho->sgfn], cg->fgfs[Sx->sgfn], cg->fgfs[Sy->sgfn], cg->fgfs[Sz->sgfn],
cg->fgfs[Sxx->sgfn], cg->fgfs[Sxy->sgfn], cg->fgfs[Sxz->sgfn],
cg->fgfs[Syy->sgfn], cg->fgfs[Syz->sgfn], cg->fgfs[Szz->sgfn],
cg->fgfs[Gamxxx->sgfn], cg->fgfs[Gamxxy->sgfn], cg->fgfs[Gamxxz->sgfn],
cg->fgfs[Gamxyy->sgfn], cg->fgfs[Gamxyz->sgfn], cg->fgfs[Gamxzz->sgfn],
cg->fgfs[Gamyxx->sgfn], cg->fgfs[Gamyxy->sgfn], cg->fgfs[Gamyxz->sgfn],
cg->fgfs[Gamyyy->sgfn], cg->fgfs[Gamyyz->sgfn], cg->fgfs[Gamyzz->sgfn],
cg->fgfs[Gamzxx->sgfn], cg->fgfs[Gamzxy->sgfn], cg->fgfs[Gamzxz->sgfn],
cg->fgfs[Gamzyy->sgfn], cg->fgfs[Gamzyz->sgfn], cg->fgfs[Gamzzz->sgfn],
cg->fgfs[Rxx->sgfn], cg->fgfs[Rxy->sgfn], cg->fgfs[Rxz->sgfn],
cg->fgfs[Ryy->sgfn], cg->fgfs[Ryz->sgfn], cg->fgfs[Rzz->sgfn],
cg->fgfs[Cons_Ham->sgfn],
cg->fgfs[Cons_Px->sgfn], cg->fgfs[Cons_Py->sgfn], cg->fgfs[Cons_Pz->sgfn],
cg->fgfs[Cons_Gx->sgfn], cg->fgfs[Cons_Gy->sgfn], cg->fgfs[Cons_Gz->sgfn],
Symmetry, lev, ndeps, pre);
}
if (BP == Pp->data->ble)
break;
BP = BP->next;
}
Pp = Pp->next;
}
}
Parallel::Sync(GH->PatL[lev], ConstraintList, Symmetry);
}
#ifdef WithShell
if (0) // if the constrait quantities can be reused from the step rhs calculation
{
MyList<ss_patch> *sPp;
sPp = SH->PatL;
while (sPp)
{
double TRK4 = PhysTime;
int pre = 0;
int lev = 0;
MyList<Block> *BP = sPp->data->blb;
int fngfs = sPp->data->fngfs;
while (BP)
{
Block *cg = BP->data;
if (myrank == cg->rank)
{
f_compute_rhs_bssn_ss(cg->shape, TRK4, cg->X[0], cg->X[1], cg->X[2],
cg->fgfs[fngfs + ShellPatch::gx],
cg->fgfs[fngfs + ShellPatch::gy],
cg->fgfs[fngfs + ShellPatch::gz],
cg->fgfs[fngfs + ShellPatch::drhodx],
cg->fgfs[fngfs + ShellPatch::drhody],
cg->fgfs[fngfs + ShellPatch::drhodz],
cg->fgfs[fngfs + ShellPatch::dsigmadx],
cg->fgfs[fngfs + ShellPatch::dsigmady],
cg->fgfs[fngfs + ShellPatch::dsigmadz],
cg->fgfs[fngfs + ShellPatch::dRdx],
cg->fgfs[fngfs + ShellPatch::dRdy],
cg->fgfs[fngfs + ShellPatch::dRdz],
cg->fgfs[fngfs + ShellPatch::drhodxx],
cg->fgfs[fngfs + ShellPatch::drhodxy],
cg->fgfs[fngfs + ShellPatch::drhodxz],
cg->fgfs[fngfs + ShellPatch::drhodyy],
cg->fgfs[fngfs + ShellPatch::drhodyz],
cg->fgfs[fngfs + ShellPatch::drhodzz],
cg->fgfs[fngfs + ShellPatch::dsigmadxx],
cg->fgfs[fngfs + ShellPatch::dsigmadxy],
cg->fgfs[fngfs + ShellPatch::dsigmadxz],
cg->fgfs[fngfs + ShellPatch::dsigmadyy],
cg->fgfs[fngfs + ShellPatch::dsigmadyz],
cg->fgfs[fngfs + ShellPatch::dsigmadzz],
cg->fgfs[fngfs + ShellPatch::dRdxx],
cg->fgfs[fngfs + ShellPatch::dRdxy],
cg->fgfs[fngfs + ShellPatch::dRdxz],
cg->fgfs[fngfs + ShellPatch::dRdyy],
cg->fgfs[fngfs + ShellPatch::dRdyz],
cg->fgfs[fngfs + ShellPatch::dRdzz],
cg->fgfs[phi0->sgfn], cg->fgfs[trK0->sgfn],
cg->fgfs[gxx0->sgfn], cg->fgfs[gxy0->sgfn], cg->fgfs[gxz0->sgfn],
cg->fgfs[gyy0->sgfn], cg->fgfs[gyz0->sgfn], cg->fgfs[gzz0->sgfn],
cg->fgfs[Axx0->sgfn], cg->fgfs[Axy0->sgfn], cg->fgfs[Axz0->sgfn],
cg->fgfs[Ayy0->sgfn], cg->fgfs[Ayz0->sgfn], cg->fgfs[Azz0->sgfn],
cg->fgfs[Gmx0->sgfn], cg->fgfs[Gmy0->sgfn], cg->fgfs[Gmz0->sgfn],
cg->fgfs[Lap0->sgfn],
cg->fgfs[Sfx0->sgfn], cg->fgfs[Sfy0->sgfn], cg->fgfs[Sfz0->sgfn],
cg->fgfs[dtSfx0->sgfn], cg->fgfs[dtSfy0->sgfn], cg->fgfs[dtSfz0->sgfn],
cg->fgfs[phi_rhs->sgfn], cg->fgfs[trK_rhs->sgfn],
cg->fgfs[gxx_rhs->sgfn], cg->fgfs[gxy_rhs->sgfn], cg->fgfs[gxz_rhs->sgfn],
cg->fgfs[gyy_rhs->sgfn], cg->fgfs[gyz_rhs->sgfn], cg->fgfs[gzz_rhs->sgfn],
cg->fgfs[Axx_rhs->sgfn], cg->fgfs[Axy_rhs->sgfn], cg->fgfs[Axz_rhs->sgfn],
cg->fgfs[Ayy_rhs->sgfn], cg->fgfs[Ayz_rhs->sgfn], cg->fgfs[Azz_rhs->sgfn],
cg->fgfs[Gmx_rhs->sgfn], cg->fgfs[Gmy_rhs->sgfn], cg->fgfs[Gmz_rhs->sgfn],
cg->fgfs[Lap_rhs->sgfn],
cg->fgfs[Sfx_rhs->sgfn], cg->fgfs[Sfy_rhs->sgfn], cg->fgfs[Sfz_rhs->sgfn],
cg->fgfs[dtSfx_rhs->sgfn], cg->fgfs[dtSfy_rhs->sgfn], cg->fgfs[dtSfz_rhs->sgfn],
cg->fgfs[rho->sgfn], cg->fgfs[Sx->sgfn], cg->fgfs[Sy->sgfn], cg->fgfs[Sz->sgfn],
cg->fgfs[Sxx->sgfn], cg->fgfs[Sxy->sgfn], cg->fgfs[Sxz->sgfn],
cg->fgfs[Syy->sgfn], cg->fgfs[Syz->sgfn], cg->fgfs[Szz->sgfn],
cg->fgfs[Gamxxx->sgfn], cg->fgfs[Gamxxy->sgfn], cg->fgfs[Gamxxz->sgfn],
cg->fgfs[Gamxyy->sgfn], cg->fgfs[Gamxyz->sgfn], cg->fgfs[Gamxzz->sgfn],
cg->fgfs[Gamyxx->sgfn], cg->fgfs[Gamyxy->sgfn], cg->fgfs[Gamyxz->sgfn],
cg->fgfs[Gamyyy->sgfn], cg->fgfs[Gamyyz->sgfn], cg->fgfs[Gamyzz->sgfn],
cg->fgfs[Gamzxx->sgfn], cg->fgfs[Gamzxy->sgfn], cg->fgfs[Gamzxz->sgfn],
cg->fgfs[Gamzyy->sgfn], cg->fgfs[Gamzyz->sgfn], cg->fgfs[Gamzzz->sgfn],
cg->fgfs[Rxx->sgfn], cg->fgfs[Rxy->sgfn], cg->fgfs[Rxz->sgfn],
cg->fgfs[Ryy->sgfn], cg->fgfs[Ryz->sgfn], cg->fgfs[Rzz->sgfn],
cg->fgfs[Cons_Ham->sgfn],
cg->fgfs[Cons_Px->sgfn], cg->fgfs[Cons_Py->sgfn], cg->fgfs[Cons_Pz->sgfn],
cg->fgfs[Cons_Gx->sgfn], cg->fgfs[Cons_Gy->sgfn], cg->fgfs[Cons_Gz->sgfn],
Symmetry, lev, numepsh, sPp->data->sst, pre);
}
if (BP == sPp->data->ble)
break;
BP = BP->next;
}
sPp = sPp->next;
}
}
SH->Synch(ConstraintList, Symmetry);
#endif
double ConV[7];
#if (PSTR == 1 || PSTR == 2)
double ConV_h[7];
#endif
#ifdef WithShell
ConV[0] = SH->L2Norm(Cons_Ham);
ConV[1] = SH->L2Norm(Cons_Px);
ConV[2] = SH->L2Norm(Cons_Py);
ConV[3] = SH->L2Norm(Cons_Pz);
ConV[4] = SH->L2Norm(Cons_Gx);
ConV[5] = SH->L2Norm(Cons_Gy);
ConV[6] = SH->L2Norm(Cons_Gz);
ConVMonitor->writefile(PhysTime, 7, ConV);
#endif
for (int levi = 0; levi < GH->levels; levi++)
{
#if (PSTR == 0)
ConV[0] = Parallel::L2Norm(GH->PatL[levi]->data, Cons_Ham);
ConV[1] = Parallel::L2Norm(GH->PatL[levi]->data, Cons_Px);
ConV[2] = Parallel::L2Norm(GH->PatL[levi]->data, Cons_Py);
ConV[3] = Parallel::L2Norm(GH->PatL[levi]->data, Cons_Pz);
ConV[4] = Parallel::L2Norm(GH->PatL[levi]->data, Cons_Gx);
ConV[5] = Parallel::L2Norm(GH->PatL[levi]->data, Cons_Gy);
ConV[6] = Parallel::L2Norm(GH->PatL[levi]->data, Cons_Gz);
#elif (PSTR == 1 || PSTR == 2)
ConV[0] = Parallel::L2Norm(GH->PatL[levi]->data, Cons_Ham, GH->Commlev[levi]);
ConV[1] = Parallel::L2Norm(GH->PatL[levi]->data, Cons_Px, GH->Commlev[levi]);
ConV[2] = Parallel::L2Norm(GH->PatL[levi]->data, Cons_Py, GH->Commlev[levi]);
ConV[3] = Parallel::L2Norm(GH->PatL[levi]->data, Cons_Pz, GH->Commlev[levi]);
ConV[4] = Parallel::L2Norm(GH->PatL[levi]->data, Cons_Gx, GH->Commlev[levi]);
ConV[5] = Parallel::L2Norm(GH->PatL[levi]->data, Cons_Gy, GH->Commlev[levi]);
ConV[6] = Parallel::L2Norm(GH->PatL[levi]->data, Cons_Gz, GH->Commlev[levi]);
// misc::tillherecheck("before collect data to cpu0");
// MPI_ALLREDUCE( sendbuf, recvbuf, count, datatype, op, comm), sendbu and recvbuf must be different
if (levi > 0)
{
if (GH->mylev == levi && myrank == GH->start_rank[levi])
for (int i = 0; i < 7; i++)
ConV_h[i] = ConV[i];
else
for (int i = 0; i < 7; i++)
ConV_h[i] = 0;
MPI_Allreduce(ConV_h, ConV, 7, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD);
}
#endif
ConVMonitor->writefile(PhysTime, 7, ConV);
/*
if(fabs(ConV[0])<0.00001)
{
MyList<var> * DG_List=new MyList<var>(Cons_Ham);
DG_List->insert(Cons_Px); DG_List->insert(Cons_Py); DG_List->insert(Cons_Px);
DG_List->insert(Cons_Gx); DG_List->insert(Cons_Gy); DG_List->insert(Cons_Gx);
Parallel::Dump_Data(GH->PatL[levi],DG_List,"jiu",0,1);
DG_List->clearList();
if(myrank==0) MPI_Abort(MPI_COMM_WORLD,1);
}
*/
}
Interp_Constraint(false);
LastConsOut = 0;
}
}
//================================================================================================
//================================================================================================
// This member function computes derivatives required for apparent-horizon calculations
//================================================================================================
#ifdef With_AHF
void bssn_class::AH_Prepare_derivatives()
{
double SYM = 1.0, ANT = -1.0;
int ZEO = 0;
for (int lev = 0; lev < GH->levels; lev++)
{
MyList<Patch> *Pp = GH->PatL[lev];
while (Pp)
{
MyList<Block> *BP = Pp->data->blb;
while (BP)
{
Block *cg = BP->data;
if (myrank == cg->rank)
{
f_fderivs(cg->shape, cg->fgfs[phi0->sgfn],
cg->fgfs[dtSfx_rhs->sgfn], cg->fgfs[dtSfy_rhs->sgfn], cg->fgfs[dtSfz_rhs->sgfn],
cg->X[0], cg->X[1], cg->X[2],
SYM, SYM, SYM, Symmetry, ZEO);
f_fderivs(cg->shape, cg->fgfs[gxx0->sgfn],
cg->fgfs[Gamxxx->sgfn], cg->fgfs[Gamyxx->sgfn], cg->fgfs[Gamzxx->sgfn],
cg->X[0], cg->X[1], cg->X[2],
SYM, SYM, SYM, Symmetry, ZEO);
f_fderivs(cg->shape, cg->fgfs[gxy0->sgfn],
cg->fgfs[Gamxxy->sgfn], cg->fgfs[Gamyxy->sgfn], cg->fgfs[Gamzxy->sgfn],
cg->X[0], cg->X[1], cg->X[2],
ANT, ANT, SYM, Symmetry, ZEO);
f_fderivs(cg->shape, cg->fgfs[gxz0->sgfn],
cg->fgfs[Gamxxz->sgfn], cg->fgfs[Gamyxz->sgfn], cg->fgfs[Gamzxz->sgfn],
cg->X[0], cg->X[1], cg->X[2],
ANT, SYM, ANT, Symmetry, ZEO);
f_fderivs(cg->shape, cg->fgfs[gyy0->sgfn],
cg->fgfs[Gamxyy->sgfn], cg->fgfs[Gamyyy->sgfn], cg->fgfs[Gamzyy->sgfn],
cg->X[0], cg->X[1], cg->X[2],
SYM, SYM, SYM, Symmetry, ZEO);
f_fderivs(cg->shape, cg->fgfs[gyz0->sgfn],
cg->fgfs[Gamxyz->sgfn], cg->fgfs[Gamyyz->sgfn], cg->fgfs[Gamzyz->sgfn],
cg->X[0], cg->X[1], cg->X[2],
SYM, ANT, ANT, Symmetry, ZEO);
f_fderivs(cg->shape, cg->fgfs[gzz0->sgfn],
cg->fgfs[Gamxzz->sgfn], cg->fgfs[Gamyzz->sgfn], cg->fgfs[Gamzzz->sgfn],
cg->X[0], cg->X[1], cg->X[2],
SYM, SYM, SYM, Symmetry, ZEO);
}
if (BP == Pp->data->ble)
break;
BP = BP->next;
}
Pp = Pp->next;
}
Parallel::Sync(GH->PatL[lev], AHDList, Symmetry);
}
}
//================================================================================================
//================================================================================================
// This member function interpolates apparent-horizon data
//================================================================================================
bool bssn_class::AH_Interp_Points(MyList<var> *VarList,
int NN, double **XX,
double *Shellf, int Symmetryi)
{
MyList<var> *varl;
int num_var = 0;
varl = VarList;
while (varl)
{
num_var++;
varl = varl->next;
}
double pox[3];
for (int i = 0; i < NN; i++)
{
for (int j = 0; j < 3; j++)
pox[j] = XX[j][i];
int lev = GH->levels - 1;
bool notfound = true;
while (notfound)
{
if (lev < 0)
{
#ifdef WithShell
if (SH->Interp_One_Point(VarList, pox, Shellf + i * num_var, Symmetryi))
{
return true;
}
if (myrank == 0)
{
cout << " bssn_class::AH_Interp_Points: point ("
<< pox[0] << "," << pox[1] << "," << pox[2]
<< ") is out of cgh and shell domain!" << endl;
if (ErrorMonitor->outfile)
ErrorMonitor->outfile << " bssn_class::AH_Interp_Points: point ("
<< pox[0] << "," << pox[1] << "," << pox[2]
<< ") is out of cgh and shell domain!" << endl;
}
MPI_Abort(MPI_COMM_WORLD, 1);
#else
if (myrank == 0)
{
cout << " bssn_class::AH_Interp_Points: point ("
<< pox[0] << "," << pox[1] << "," << pox[2]
<< ") is out of cgh domain!" << endl;
if (ErrorMonitor->outfile)
ErrorMonitor->outfile << " bssn_class::AH_Interp_Points: point ("
<< pox[0] << "," << pox[1] << "," << pox[2]
<< ") is out of cgh domain!" << endl;
}
MPI_Abort(MPI_COMM_WORLD, 1);
#endif
return false;
}
MyList<Patch> *Pp = GH->PatL[lev];
while (Pp)
{
if (Pp->data->Interp_ONE_Point(VarList, pox, Shellf + i * num_var, Symmetryi))
{
notfound = false;
break;
}
Pp = Pp->next;
}
lev--;
}
}
return true;
}
//================================================================================================
//================================================================================================
// This member function computes apparent horizons
//================================================================================================
void bssn_class::AH_Step_Find(int lev, double dT_lev)
{
if ((lev == GH->levels - 1))
{
int ncount = int(PhysTime / dT_lev);
bool tf = false;
for (int ihn = 0; ihn < HN_num; ihn++)
{
if (ncount % findeveryl[ihn] == 0)
{
tf = true;
break;
}
}
if (tf)
{
clock_t prev_clock, curr_clock;
if (myrank == 0)
prev_clock = clock();
const int cdumpid = int(PhysTime / AHdumptime) + 1;
for (int ihn = 0; ihn < HN_num; ihn++)
dumpid[ihn] = cdumpid;
double gam;
for (int ihn = 0; ihn < BH_num; ihn++)
{
xc[ihn] = Porg0[ihn][0];
yc[ihn] = Porg0[ihn][1];
zc[ihn] = Porg0[ihn][2];
gam = fabs(Pmom[ihn * 3]) / (Mass[ihn]);
gam = sqrt(1 - gam * gam);
xr[ihn] = Mass[ihn] * gam;
gam = fabs(Pmom[ihn * 3 + 1]) / (Mass[ihn]);
gam = sqrt(1 - gam * gam);
yr[ihn] = Mass[ihn] * gam;
gam = fabs(Pmom[ihn * 3 + 2]) / (Mass[ihn]);
gam = sqrt(1 - gam * gam);
zr[ihn] = Mass[ihn] * gam;
dTT[ihn] = -1;
if (ncount % findeveryl[ihn] == 0)
{
trigger[ihn] = true;
dTT[ihn] = findeveryl[ihn] * dT_lev;
}
else
trigger[ihn] = false;
if (trigger[ihn] && (dumpid[ihn] > lastahdumpid[ihn]))
lastahdumpid[ihn] = dumpid[ihn];
else
dumpid[ihn] = 0;
}
int ihn = BH_num;
for (int ia = 0; ia < BH_num; ia++)
for (int ib = ia + 1; ib < BH_num; ib++)
{
xc[ihn] = (Porg0[ia][0] + Porg0[ib][0]) / 2;
yc[ihn] = (Porg0[ia][1] + Porg0[ib][1]) / 2;
zc[ihn] = (Porg0[ia][2] + Porg0[ib][2]) / 2;
xr[ihn] = yr[ihn] = zr[ihn] = Mass[ia] + Mass[ib];
dTT[ihn] = -1;
if (fabs(Porg0[ia][0] - Porg0[ib][0]) < 2 * xr[ihn] &&
fabs(Porg0[ia][1] - Porg0[ib][1]) < 2 * xr[ihn] &&
fabs(Porg0[ia][2] - Porg0[ib][2]) < 2 * xr[ihn] &&
(ncount % findeveryl[ihn] == 0))
{
trigger[ihn] = true;
dTT[ihn] = findeveryl[ihn] * dT_lev;
}
else
trigger[ihn] = false;
if (trigger[ihn] && (dumpid[ihn] > lastahdumpid[ihn]))
lastahdumpid[ihn] = dumpid[ihn];
else
dumpid[ihn] = 0;
ihn++;
}
#if (ABEtype == 1)
if (PhysTime > 10)
{
ihn--;
trigger[ihn] = true;
xr[ihn] = yr[ihn] = zr[ihn] = 50;
// if(myrank==0) for(ihn=0;ihn<HN_num;ihn++) cout<<"trigger#"<<ihn<<": "<<trigger[ihn]<<endl;
}
#endif
AHFinderDirect::AHFinderDirect_find_horizons(HN_num, dumpid,
xc, yc, zc, xr, yr, zr, trigger, dTT);
// note rhs and Gamijk have been used as temp storage space
if (myrank == 0)
{
curr_clock = clock();
cout << " Finding horizon used "
<< (double)(curr_clock - prev_clock) / ((double)CLOCKS_PER_SEC)
<< " seconds! " << endl;
}
}
}
}
#endif
//================================================================================================
//================================================================================================
// This member function interpolates constraint data
//================================================================================================
void bssn_class::Interp_Constraint(bool infg)
{
if (infg)
{
// we do not support a_lev != 0 yet.
if (a_lev > 0)
return;
// recompute least the constraint data lost for moved new grid
for (int lev = 0; lev < GH->levels; lev++)
{
// make sure the data consistent for higher levels
if (lev > 0) // if the constrait quantities can be reused from the step rhs calculation
{
double TRK4 = PhysTime;
double ndeps = numepsb;
int pre = 0;
MyList<Patch> *Pp = GH->PatL[lev];
while (Pp)
{
MyList<Block> *BP = Pp->data->blb;
while (BP)
{
Block *cg = BP->data;
if (myrank == cg->rank)
{
f_compute_rhs_bssn(cg->shape, TRK4, cg->X[0], cg->X[1], cg->X[2],
cg->fgfs[phi0->sgfn], cg->fgfs[trK0->sgfn],
cg->fgfs[gxx0->sgfn], cg->fgfs[gxy0->sgfn], cg->fgfs[gxz0->sgfn],
cg->fgfs[gyy0->sgfn], cg->fgfs[gyz0->sgfn], cg->fgfs[gzz0->sgfn],
cg->fgfs[Axx0->sgfn], cg->fgfs[Axy0->sgfn], cg->fgfs[Axz0->sgfn],
cg->fgfs[Ayy0->sgfn], cg->fgfs[Ayz0->sgfn], cg->fgfs[Azz0->sgfn],
cg->fgfs[Gmx0->sgfn], cg->fgfs[Gmy0->sgfn], cg->fgfs[Gmz0->sgfn],
cg->fgfs[Lap0->sgfn],
cg->fgfs[Sfx0->sgfn], cg->fgfs[Sfy0->sgfn], cg->fgfs[Sfz0->sgfn],
cg->fgfs[dtSfx0->sgfn], cg->fgfs[dtSfy0->sgfn], cg->fgfs[dtSfz0->sgfn],
cg->fgfs[phi_rhs->sgfn], cg->fgfs[trK_rhs->sgfn],
cg->fgfs[gxx_rhs->sgfn], cg->fgfs[gxy_rhs->sgfn], cg->fgfs[gxz_rhs->sgfn],
cg->fgfs[gyy_rhs->sgfn], cg->fgfs[gyz_rhs->sgfn], cg->fgfs[gzz_rhs->sgfn],
cg->fgfs[Axx_rhs->sgfn], cg->fgfs[Axy_rhs->sgfn], cg->fgfs[Axz_rhs->sgfn],
cg->fgfs[Ayy_rhs->sgfn], cg->fgfs[Ayz_rhs->sgfn], cg->fgfs[Azz_rhs->sgfn],
cg->fgfs[Gmx_rhs->sgfn], cg->fgfs[Gmy_rhs->sgfn], cg->fgfs[Gmz_rhs->sgfn],
cg->fgfs[Lap_rhs->sgfn],
cg->fgfs[Sfx_rhs->sgfn], cg->fgfs[Sfy_rhs->sgfn], cg->fgfs[Sfz_rhs->sgfn],
cg->fgfs[dtSfx_rhs->sgfn], cg->fgfs[dtSfy_rhs->sgfn], cg->fgfs[dtSfz_rhs->sgfn],
cg->fgfs[rho->sgfn], cg->fgfs[Sx->sgfn], cg->fgfs[Sy->sgfn], cg->fgfs[Sz->sgfn],
cg->fgfs[Sxx->sgfn], cg->fgfs[Sxy->sgfn], cg->fgfs[Sxz->sgfn],
cg->fgfs[Syy->sgfn], cg->fgfs[Syz->sgfn], cg->fgfs[Szz->sgfn],
cg->fgfs[Gamxxx->sgfn], cg->fgfs[Gamxxy->sgfn], cg->fgfs[Gamxxz->sgfn],
cg->fgfs[Gamxyy->sgfn], cg->fgfs[Gamxyz->sgfn], cg->fgfs[Gamxzz->sgfn],
cg->fgfs[Gamyxx->sgfn], cg->fgfs[Gamyxy->sgfn], cg->fgfs[Gamyxz->sgfn],
cg->fgfs[Gamyyy->sgfn], cg->fgfs[Gamyyz->sgfn], cg->fgfs[Gamyzz->sgfn],
cg->fgfs[Gamzxx->sgfn], cg->fgfs[Gamzxy->sgfn], cg->fgfs[Gamzxz->sgfn],
cg->fgfs[Gamzyy->sgfn], cg->fgfs[Gamzyz->sgfn], cg->fgfs[Gamzzz->sgfn],
cg->fgfs[Rxx->sgfn], cg->fgfs[Rxy->sgfn], cg->fgfs[Rxz->sgfn],
cg->fgfs[Ryy->sgfn], cg->fgfs[Ryz->sgfn], cg->fgfs[Rzz->sgfn],
cg->fgfs[Cons_Ham->sgfn],
cg->fgfs[Cons_Px->sgfn], cg->fgfs[Cons_Py->sgfn], cg->fgfs[Cons_Pz->sgfn],
cg->fgfs[Cons_Gx->sgfn], cg->fgfs[Cons_Gy->sgfn], cg->fgfs[Cons_Gz->sgfn],
Symmetry, lev, ndeps, pre);
}
if (BP == Pp->data->ble)
break;
BP = BP->next;
}
Pp = Pp->next;
}
}
Parallel::Sync(GH->PatL[lev], ConstraintList, Symmetry);
}
#ifdef WithShell
if (0) // if the constrait quantities can be reused from the step rhs calculation
{
MyList<ss_patch> *sPp;
sPp = SH->PatL;
while (sPp)
{
double TRK4 = PhysTime;
int pre = 0;
int lev = 0;
MyList<Block> *BP = sPp->data->blb;
int fngfs = sPp->data->fngfs;
while (BP)
{
Block *cg = BP->data;
if (myrank == cg->rank)
{
f_compute_rhs_bssn_ss(cg->shape, TRK4, cg->X[0], cg->X[1], cg->X[2],
cg->fgfs[fngfs + ShellPatch::gx],
cg->fgfs[fngfs + ShellPatch::gy],
cg->fgfs[fngfs + ShellPatch::gz],
cg->fgfs[fngfs + ShellPatch::drhodx],
cg->fgfs[fngfs + ShellPatch::drhody],
cg->fgfs[fngfs + ShellPatch::drhodz],
cg->fgfs[fngfs + ShellPatch::dsigmadx],
cg->fgfs[fngfs + ShellPatch::dsigmady],
cg->fgfs[fngfs + ShellPatch::dsigmadz],
cg->fgfs[fngfs + ShellPatch::dRdx],
cg->fgfs[fngfs + ShellPatch::dRdy],
cg->fgfs[fngfs + ShellPatch::dRdz],
cg->fgfs[fngfs + ShellPatch::drhodxx],
cg->fgfs[fngfs + ShellPatch::drhodxy],
cg->fgfs[fngfs + ShellPatch::drhodxz],
cg->fgfs[fngfs + ShellPatch::drhodyy],
cg->fgfs[fngfs + ShellPatch::drhodyz],
cg->fgfs[fngfs + ShellPatch::drhodzz],
cg->fgfs[fngfs + ShellPatch::dsigmadxx],
cg->fgfs[fngfs + ShellPatch::dsigmadxy],
cg->fgfs[fngfs + ShellPatch::dsigmadxz],
cg->fgfs[fngfs + ShellPatch::dsigmadyy],
cg->fgfs[fngfs + ShellPatch::dsigmadyz],
cg->fgfs[fngfs + ShellPatch::dsigmadzz],
cg->fgfs[fngfs + ShellPatch::dRdxx],
cg->fgfs[fngfs + ShellPatch::dRdxy],
cg->fgfs[fngfs + ShellPatch::dRdxz],
cg->fgfs[fngfs + ShellPatch::dRdyy],
cg->fgfs[fngfs + ShellPatch::dRdyz],
cg->fgfs[fngfs + ShellPatch::dRdzz],
cg->fgfs[phi0->sgfn], cg->fgfs[trK0->sgfn],
cg->fgfs[gxx0->sgfn], cg->fgfs[gxy0->sgfn], cg->fgfs[gxz0->sgfn],
cg->fgfs[gyy0->sgfn], cg->fgfs[gyz0->sgfn], cg->fgfs[gzz0->sgfn],
cg->fgfs[Axx0->sgfn], cg->fgfs[Axy0->sgfn], cg->fgfs[Axz0->sgfn],
cg->fgfs[Ayy0->sgfn], cg->fgfs[Ayz0->sgfn], cg->fgfs[Azz0->sgfn],
cg->fgfs[Gmx0->sgfn], cg->fgfs[Gmy0->sgfn], cg->fgfs[Gmz0->sgfn],
cg->fgfs[Lap0->sgfn],
cg->fgfs[Sfx0->sgfn], cg->fgfs[Sfy0->sgfn], cg->fgfs[Sfz0->sgfn],
cg->fgfs[dtSfx0->sgfn], cg->fgfs[dtSfy0->sgfn], cg->fgfs[dtSfz0->sgfn],
cg->fgfs[phi_rhs->sgfn], cg->fgfs[trK_rhs->sgfn],
cg->fgfs[gxx_rhs->sgfn], cg->fgfs[gxy_rhs->sgfn], cg->fgfs[gxz_rhs->sgfn],
cg->fgfs[gyy_rhs->sgfn], cg->fgfs[gyz_rhs->sgfn], cg->fgfs[gzz_rhs->sgfn],
cg->fgfs[Axx_rhs->sgfn], cg->fgfs[Axy_rhs->sgfn], cg->fgfs[Axz_rhs->sgfn],
cg->fgfs[Ayy_rhs->sgfn], cg->fgfs[Ayz_rhs->sgfn], cg->fgfs[Azz_rhs->sgfn],
cg->fgfs[Gmx_rhs->sgfn], cg->fgfs[Gmy_rhs->sgfn], cg->fgfs[Gmz_rhs->sgfn],
cg->fgfs[Lap_rhs->sgfn],
cg->fgfs[Sfx_rhs->sgfn], cg->fgfs[Sfy_rhs->sgfn], cg->fgfs[Sfz_rhs->sgfn],
cg->fgfs[dtSfx_rhs->sgfn], cg->fgfs[dtSfy_rhs->sgfn], cg->fgfs[dtSfz_rhs->sgfn],
cg->fgfs[rho->sgfn], cg->fgfs[Sx->sgfn], cg->fgfs[Sy->sgfn], cg->fgfs[Sz->sgfn],
cg->fgfs[Sxx->sgfn], cg->fgfs[Sxy->sgfn], cg->fgfs[Sxz->sgfn],
cg->fgfs[Syy->sgfn], cg->fgfs[Syz->sgfn], cg->fgfs[Szz->sgfn],
cg->fgfs[Gamxxx->sgfn], cg->fgfs[Gamxxy->sgfn], cg->fgfs[Gamxxz->sgfn],
cg->fgfs[Gamxyy->sgfn], cg->fgfs[Gamxyz->sgfn], cg->fgfs[Gamxzz->sgfn],
cg->fgfs[Gamyxx->sgfn], cg->fgfs[Gamyxy->sgfn], cg->fgfs[Gamyxz->sgfn],
cg->fgfs[Gamyyy->sgfn], cg->fgfs[Gamyyz->sgfn], cg->fgfs[Gamyzz->sgfn],
cg->fgfs[Gamzxx->sgfn], cg->fgfs[Gamzxy->sgfn], cg->fgfs[Gamzxz->sgfn],
cg->fgfs[Gamzyy->sgfn], cg->fgfs[Gamzyz->sgfn], cg->fgfs[Gamzzz->sgfn],
cg->fgfs[Rxx->sgfn], cg->fgfs[Rxy->sgfn], cg->fgfs[Rxz->sgfn],
cg->fgfs[Ryy->sgfn], cg->fgfs[Ryz->sgfn], cg->fgfs[Rzz->sgfn],
cg->fgfs[Cons_Ham->sgfn],
cg->fgfs[Cons_Px->sgfn], cg->fgfs[Cons_Py->sgfn], cg->fgfs[Cons_Pz->sgfn],
cg->fgfs[Cons_Gx->sgfn], cg->fgfs[Cons_Gy->sgfn], cg->fgfs[Cons_Gz->sgfn],
Symmetry, lev, numepsh, sPp->data->sst, pre);
}
if (BP == sPp->data->ble)
break;
BP = BP->next;
}
sPp = sPp->next;
}
}
SH->Synch(ConstraintList, Symmetry);
#endif
}
// interpolate
double *x1, *y1, *z1;
const int n = 1000;
double lmax, lmin, dd;
lmin = 0;
#ifdef WithShell
lmax = SH->Rrange[1];
#else
lmax = GH->bbox[0][0][4];
#endif
#ifdef Vertex
#ifdef Cell
#error Both Cell and Vertex are defined
#endif
dd = (lmax - lmin) / (n - 1);
#else
#ifdef Cell
dd = (lmax - lmin) / n;
#else
#error Not define Vertex nor Cell
#endif
#endif
x1 = new double[n];
y1 = new double[n];
z1 = new double[n];
for (int i = 0; i < n; i++)
{
x1[i] = 0;
#ifdef Vertex
#ifdef Cell
#error Both Cell and Vertex are defined
#endif
y1[i] = lmin + i * dd;
#else
#ifdef Cell
y1[i] = lmin + (i + 0.5) * dd;
#else
#error Not define Vertex nor Cell
#endif
#endif
z1[i] = 0;
}
int InList = 0;
MyList<var> *varl = ConstraintList;
while (varl)
{
InList++;
varl = varl->next;
}
double *shellf;
shellf = new double[n * InList];
for (int i = 0; i < n; i++)
{
double XX[3];
XX[0] = x1[i];
XX[1] = y1[i];
XX[2] = z1[i];
bool fg = GH->Interp_One_Point(ConstraintList, XX, shellf + i * InList, Symmetry);
#ifdef WithShell
if (!fg)
fg = SH->Interp_One_Point(ConstraintList, XX, shellf + i * InList, Symmetry);
#endif
if (!fg && myrank == 0)
{
cout << "bssn_class::Interp_Constraint meets wrong" << endl;
MPI_Abort(MPI_COMM_WORLD, 1);
}
}
if (myrank == 0)
{
ofstream outfile;
char filename[50];
sprintf(filename, "%s/interp_constraint_%05d.dat", ErrorMonitor->out_dir.c_str(), int(PhysTime / dT + 0.5));
// 0.5 for round off
outfile.open(filename);
outfile << "# corrdinate, H_Res, Px_Res, Py_Res, Pz_Res, Gx_Res, Gy_Res, Gz_Res, ...." << endl;
for (int i = 0; i < n; i++)
{
outfile << setw(10) << setprecision(10) << y1[i];
for (int j = 0; j < InList; j++)
outfile << " " << setw(16) << setprecision(15) << shellf[InList * i + j];
outfile << endl;
}
outfile.close();
}
delete[] shellf;
}
//================================================================================================
//================================================================================================
// This member function computes constraint violations
//================================================================================================
void bssn_class::Compute_Constraint()
{
double TRK4 = PhysTime;
double ndeps = numepsb;
int pre = 0;
int lev;
for (lev = 0; lev < GH->levels; lev++)
{
{
MyList<Patch> *Pp = GH->PatL[lev];
while (Pp)
{
MyList<Block> *BP = Pp->data->blb;
while (BP)
{
Block *cg = BP->data;
if (myrank == cg->rank)
{
f_compute_rhs_bssn(cg->shape, TRK4, cg->X[0], cg->X[1], cg->X[2],
cg->fgfs[phi0->sgfn], cg->fgfs[trK0->sgfn],
cg->fgfs[gxx0->sgfn], cg->fgfs[gxy0->sgfn], cg->fgfs[gxz0->sgfn],
cg->fgfs[gyy0->sgfn], cg->fgfs[gyz0->sgfn], cg->fgfs[gzz0->sgfn],
cg->fgfs[Axx0->sgfn], cg->fgfs[Axy0->sgfn], cg->fgfs[Axz0->sgfn],
cg->fgfs[Ayy0->sgfn], cg->fgfs[Ayz0->sgfn], cg->fgfs[Azz0->sgfn],
cg->fgfs[Gmx0->sgfn], cg->fgfs[Gmy0->sgfn], cg->fgfs[Gmz0->sgfn],
cg->fgfs[Lap0->sgfn],
cg->fgfs[Sfx0->sgfn], cg->fgfs[Sfy0->sgfn], cg->fgfs[Sfz0->sgfn],
cg->fgfs[dtSfx0->sgfn], cg->fgfs[dtSfy0->sgfn], cg->fgfs[dtSfz0->sgfn],
cg->fgfs[phi_rhs->sgfn], cg->fgfs[trK_rhs->sgfn],
cg->fgfs[gxx_rhs->sgfn], cg->fgfs[gxy_rhs->sgfn], cg->fgfs[gxz_rhs->sgfn],
cg->fgfs[gyy_rhs->sgfn], cg->fgfs[gyz_rhs->sgfn], cg->fgfs[gzz_rhs->sgfn],
cg->fgfs[Axx_rhs->sgfn], cg->fgfs[Axy_rhs->sgfn], cg->fgfs[Axz_rhs->sgfn],
cg->fgfs[Ayy_rhs->sgfn], cg->fgfs[Ayz_rhs->sgfn], cg->fgfs[Azz_rhs->sgfn],
cg->fgfs[Gmx_rhs->sgfn], cg->fgfs[Gmy_rhs->sgfn], cg->fgfs[Gmz_rhs->sgfn],
cg->fgfs[Lap_rhs->sgfn],
cg->fgfs[Sfx_rhs->sgfn], cg->fgfs[Sfy_rhs->sgfn], cg->fgfs[Sfz_rhs->sgfn],
cg->fgfs[dtSfx_rhs->sgfn], cg->fgfs[dtSfy_rhs->sgfn], cg->fgfs[dtSfz_rhs->sgfn],
cg->fgfs[rho->sgfn], cg->fgfs[Sx->sgfn], cg->fgfs[Sy->sgfn], cg->fgfs[Sz->sgfn],
cg->fgfs[Sxx->sgfn], cg->fgfs[Sxy->sgfn], cg->fgfs[Sxz->sgfn],
cg->fgfs[Syy->sgfn], cg->fgfs[Syz->sgfn], cg->fgfs[Szz->sgfn],
cg->fgfs[Gamxxx->sgfn], cg->fgfs[Gamxxy->sgfn], cg->fgfs[Gamxxz->sgfn],
cg->fgfs[Gamxyy->sgfn], cg->fgfs[Gamxyz->sgfn], cg->fgfs[Gamxzz->sgfn],
cg->fgfs[Gamyxx->sgfn], cg->fgfs[Gamyxy->sgfn], cg->fgfs[Gamyxz->sgfn],
cg->fgfs[Gamyyy->sgfn], cg->fgfs[Gamyyz->sgfn], cg->fgfs[Gamyzz->sgfn],
cg->fgfs[Gamzxx->sgfn], cg->fgfs[Gamzxy->sgfn], cg->fgfs[Gamzxz->sgfn],
cg->fgfs[Gamzyy->sgfn], cg->fgfs[Gamzyz->sgfn], cg->fgfs[Gamzzz->sgfn],
cg->fgfs[Rxx->sgfn], cg->fgfs[Rxy->sgfn], cg->fgfs[Rxz->sgfn],
cg->fgfs[Ryy->sgfn], cg->fgfs[Ryz->sgfn], cg->fgfs[Rzz->sgfn],
cg->fgfs[Cons_Ham->sgfn],
cg->fgfs[Cons_Px->sgfn], cg->fgfs[Cons_Py->sgfn], cg->fgfs[Cons_Pz->sgfn],
cg->fgfs[Cons_Gx->sgfn], cg->fgfs[Cons_Gy->sgfn], cg->fgfs[Cons_Gz->sgfn],
Symmetry, lev, ndeps, pre);
}
if (BP == Pp->data->ble)
break;
BP = BP->next;
}
Pp = Pp->next;
}
}
Parallel::Sync(GH->PatL[lev], ConstraintList, Symmetry);
}
// prolong restrict constraint quantities
for (lev = GH->levels - 1; lev > 0; lev--)
RestrictProlong(lev, 1, false, ConstraintList, ConstraintList, ConstraintList);
#ifdef WithShell
lev = 0;
{
MyList<ss_patch> *sPp;
sPp = SH->PatL;
while (sPp)
{
MyList<Block> *BP = sPp->data->blb;
int fngfs = sPp->data->fngfs;
while (BP)
{
Block *cg = BP->data;
if (myrank == cg->rank)
{
f_compute_rhs_bssn_ss(cg->shape, TRK4, cg->X[0], cg->X[1], cg->X[2],
cg->fgfs[fngfs + ShellPatch::gx],
cg->fgfs[fngfs + ShellPatch::gy],
cg->fgfs[fngfs + ShellPatch::gz],
cg->fgfs[fngfs + ShellPatch::drhodx],
cg->fgfs[fngfs + ShellPatch::drhody],
cg->fgfs[fngfs + ShellPatch::drhodz],
cg->fgfs[fngfs + ShellPatch::dsigmadx],
cg->fgfs[fngfs + ShellPatch::dsigmady],
cg->fgfs[fngfs + ShellPatch::dsigmadz],
cg->fgfs[fngfs + ShellPatch::dRdx],
cg->fgfs[fngfs + ShellPatch::dRdy],
cg->fgfs[fngfs + ShellPatch::dRdz],
cg->fgfs[fngfs + ShellPatch::drhodxx],
cg->fgfs[fngfs + ShellPatch::drhodxy],
cg->fgfs[fngfs + ShellPatch::drhodxz],
cg->fgfs[fngfs + ShellPatch::drhodyy],
cg->fgfs[fngfs + ShellPatch::drhodyz],
cg->fgfs[fngfs + ShellPatch::drhodzz],
cg->fgfs[fngfs + ShellPatch::dsigmadxx],
cg->fgfs[fngfs + ShellPatch::dsigmadxy],
cg->fgfs[fngfs + ShellPatch::dsigmadxz],
cg->fgfs[fngfs + ShellPatch::dsigmadyy],
cg->fgfs[fngfs + ShellPatch::dsigmadyz],
cg->fgfs[fngfs + ShellPatch::dsigmadzz],
cg->fgfs[fngfs + ShellPatch::dRdxx],
cg->fgfs[fngfs + ShellPatch::dRdxy],
cg->fgfs[fngfs + ShellPatch::dRdxz],
cg->fgfs[fngfs + ShellPatch::dRdyy],
cg->fgfs[fngfs + ShellPatch::dRdyz],
cg->fgfs[fngfs + ShellPatch::dRdzz],
cg->fgfs[phi0->sgfn], cg->fgfs[trK0->sgfn],
cg->fgfs[gxx0->sgfn], cg->fgfs[gxy0->sgfn], cg->fgfs[gxz0->sgfn],
cg->fgfs[gyy0->sgfn], cg->fgfs[gyz0->sgfn], cg->fgfs[gzz0->sgfn],
cg->fgfs[Axx0->sgfn], cg->fgfs[Axy0->sgfn], cg->fgfs[Axz0->sgfn],
cg->fgfs[Ayy0->sgfn], cg->fgfs[Ayz0->sgfn], cg->fgfs[Azz0->sgfn],
cg->fgfs[Gmx0->sgfn], cg->fgfs[Gmy0->sgfn], cg->fgfs[Gmz0->sgfn],
cg->fgfs[Lap0->sgfn],
cg->fgfs[Sfx0->sgfn], cg->fgfs[Sfy0->sgfn], cg->fgfs[Sfz0->sgfn],
cg->fgfs[dtSfx0->sgfn], cg->fgfs[dtSfy0->sgfn], cg->fgfs[dtSfz0->sgfn],
cg->fgfs[phi_rhs->sgfn], cg->fgfs[trK_rhs->sgfn],
cg->fgfs[gxx_rhs->sgfn], cg->fgfs[gxy_rhs->sgfn], cg->fgfs[gxz_rhs->sgfn],
cg->fgfs[gyy_rhs->sgfn], cg->fgfs[gyz_rhs->sgfn], cg->fgfs[gzz_rhs->sgfn],
cg->fgfs[Axx_rhs->sgfn], cg->fgfs[Axy_rhs->sgfn], cg->fgfs[Axz_rhs->sgfn],
cg->fgfs[Ayy_rhs->sgfn], cg->fgfs[Ayz_rhs->sgfn], cg->fgfs[Azz_rhs->sgfn],
cg->fgfs[Gmx_rhs->sgfn], cg->fgfs[Gmy_rhs->sgfn], cg->fgfs[Gmz_rhs->sgfn],
cg->fgfs[Lap_rhs->sgfn],
cg->fgfs[Sfx_rhs->sgfn], cg->fgfs[Sfy_rhs->sgfn], cg->fgfs[Sfz_rhs->sgfn],
cg->fgfs[dtSfx_rhs->sgfn], cg->fgfs[dtSfy_rhs->sgfn], cg->fgfs[dtSfz_rhs->sgfn],
cg->fgfs[rho->sgfn], cg->fgfs[Sx->sgfn], cg->fgfs[Sy->sgfn], cg->fgfs[Sz->sgfn],
cg->fgfs[Sxx->sgfn], cg->fgfs[Sxy->sgfn], cg->fgfs[Sxz->sgfn],
cg->fgfs[Syy->sgfn], cg->fgfs[Syz->sgfn], cg->fgfs[Szz->sgfn],
cg->fgfs[Gamxxx->sgfn], cg->fgfs[Gamxxy->sgfn], cg->fgfs[Gamxxz->sgfn],
cg->fgfs[Gamxyy->sgfn], cg->fgfs[Gamxyz->sgfn], cg->fgfs[Gamxzz->sgfn],
cg->fgfs[Gamyxx->sgfn], cg->fgfs[Gamyxy->sgfn], cg->fgfs[Gamyxz->sgfn],
cg->fgfs[Gamyyy->sgfn], cg->fgfs[Gamyyz->sgfn], cg->fgfs[Gamyzz->sgfn],
cg->fgfs[Gamzxx->sgfn], cg->fgfs[Gamzxy->sgfn], cg->fgfs[Gamzxz->sgfn],
cg->fgfs[Gamzyy->sgfn], cg->fgfs[Gamzyz->sgfn], cg->fgfs[Gamzzz->sgfn],
cg->fgfs[Rxx->sgfn], cg->fgfs[Rxy->sgfn], cg->fgfs[Rxz->sgfn],
cg->fgfs[Ryy->sgfn], cg->fgfs[Ryz->sgfn], cg->fgfs[Rzz->sgfn],
cg->fgfs[Cons_Ham->sgfn],
cg->fgfs[Cons_Px->sgfn], cg->fgfs[Cons_Py->sgfn], cg->fgfs[Cons_Pz->sgfn],
cg->fgfs[Cons_Gx->sgfn], cg->fgfs[Cons_Gy->sgfn], cg->fgfs[Cons_Gz->sgfn],
Symmetry, lev, numepsh, sPp->data->sst, pre);
}
if (BP == sPp->data->ble)
break;
BP = BP->next;
}
sPp = sPp->next;
}
}
SH->Synch(ConstraintList, Symmetry);
// interpolate constraint quantities
SH->CS_Inter(ConstraintList, Symmetry);
#endif
}
//================================================================================================
//================================================================================================
void bssn_class::testRestrict()
{
MyList<var> *DG_List = new MyList<var>(phi0);
int lev = 0;
double ZEO = 0, ONE = 1;
MyList<Patch> *Pp = GH->PatL[lev];
while (Pp)
{
MyList<Block> *BP = Pp->data->blb;
while (BP)
{
Block *cg = BP->data;
if (myrank == cg->rank)
{
f_set_value(cg->shape, cg->fgfs[phi0->sgfn], ZEO);
}
if (BP == Pp->data->ble)
break;
BP = BP->next;
}
Pp = Pp->next;
}
lev = 1;
Pp = GH->PatL[lev];
while (Pp)
{
MyList<Block> *BP = Pp->data->blb;
while (BP)
{
Block *cg = BP->data;
if (myrank == cg->rank)
{
f_set_value(cg->shape, cg->fgfs[phi0->sgfn], ONE);
}
if (BP == Pp->data->ble)
break;
BP = BP->next;
}
Pp = Pp->next;
}
Parallel::Restrict(GH->PatL[lev - 1], GH->PatL[lev], DG_List, DG_List, Symmetry);
Parallel::Sync(GH->PatL[lev - 1], DG_List, Symmetry);
Parallel::Dump_Data(GH->PatL[lev - 1], DG_List, 0, PhysTime, dT);
Parallel::Dump_Data(GH->PatL[lev], DG_List, 0, PhysTime, dT);
DG_List->clearList();
exit(0);
}
//================================================================================================
//================================================================================================
void bssn_class::testOutBd()
{
MyList<var> *DG_List = new MyList<var>(phi0);
int lev = 1;
double ZEO = 0, ONE = 1;
MyList<Patch> *Pp = GH->PatL[lev];
while (Pp)
{
MyList<Block> *BP = Pp->data->blb;
while (BP)
{
Block *cg = BP->data;
if (myrank == cg->rank)
{
f_set_value(cg->shape, cg->fgfs[phi0->sgfn], ZEO);
}
if (BP == Pp->data->ble)
break;
BP = BP->next;
}
Pp = Pp->next;
}
lev = 0;
Pp = GH->PatL[lev];
while (Pp)
{
MyList<Block> *BP = Pp->data->blb;
while (BP)
{
Block *cg = BP->data;
if (myrank == cg->rank)
{
f_set_value(cg->shape, cg->fgfs[phi0->sgfn], ONE);
}
if (BP == Pp->data->ble)
break;
BP = BP->next;
}
Pp = Pp->next;
}
lev = 1;
MyList<Patch> *Ppc = GH->PatL[lev - 1];
while (Ppc)
{
Pp = GH->PatL[lev];
while (Pp)
{
Parallel::OutBdLow2Hi(Ppc->data, Pp->data, DG_List, DG_List, Symmetry);
Pp = Pp->next;
}
Ppc = Ppc->next;
}
Parallel::Sync(GH->PatL[lev], DG_List, Symmetry);
Parallel::Dump_Data(GH->PatL[lev], DG_List, 0, PhysTime, dT);
Parallel::Dump_Data(GH->PatL[lev - 1], DG_List, 0, PhysTime, dT);
DG_List->clearList();
exit(0);
}
//================================================================================================
//================================================================================================
// This member function enforces/checks the traceless condition
//================================================================================================
void bssn_class::Enforce_algcon(int lev, int fg)
{
MyList<Patch> *Pp = GH->PatL[lev];
while (Pp)
{
MyList<Block> *BP = Pp->data->blb;
while (BP)
{
Block *cg = BP->data;
if (myrank == cg->rank)
{
if (fg == 0)
f_enforce_ga(cg->shape,
cg->fgfs[gxx0->sgfn], cg->fgfs[gxy0->sgfn], cg->fgfs[gxz0->sgfn],
cg->fgfs[gyy0->sgfn], cg->fgfs[gyz0->sgfn], cg->fgfs[gzz0->sgfn],
cg->fgfs[Axx0->sgfn], cg->fgfs[Axy0->sgfn], cg->fgfs[Axz0->sgfn],
cg->fgfs[Ayy0->sgfn], cg->fgfs[Ayz0->sgfn], cg->fgfs[Azz0->sgfn]);
else
f_enforce_ga(cg->shape,
cg->fgfs[gxx->sgfn], cg->fgfs[gxy->sgfn], cg->fgfs[gxz->sgfn],
cg->fgfs[gyy->sgfn], cg->fgfs[gyz->sgfn], cg->fgfs[gzz->sgfn],
cg->fgfs[Axx->sgfn], cg->fgfs[Axy->sgfn], cg->fgfs[Axz->sgfn],
cg->fgfs[Ayy->sgfn], cg->fgfs[Ayz->sgfn], cg->fgfs[Azz->sgfn]);
}
if (BP == Pp->data->ble)
break;
BP = BP->next;
}
Pp = Pp->next;
}
#ifdef WithShell
if (lev == 0)
{
MyList<ss_patch> *sPp = SH->PatL;
while (sPp)
{
MyList<Block> *BP = sPp->data->blb;
int fngfs = sPp->data->fngfs;
while (BP)
{
Block *cg = BP->data;
if (myrank == cg->rank)
{
if (fg == 0)
f_enforce_ga(cg->shape,
cg->fgfs[gxx0->sgfn], cg->fgfs[gxy0->sgfn], cg->fgfs[gxz0->sgfn],
cg->fgfs[gyy0->sgfn], cg->fgfs[gyz0->sgfn], cg->fgfs[gzz0->sgfn],
cg->fgfs[Axx0->sgfn], cg->fgfs[Axy0->sgfn], cg->fgfs[Axz0->sgfn],
cg->fgfs[Ayy0->sgfn], cg->fgfs[Ayz0->sgfn], cg->fgfs[Azz0->sgfn]);
else
f_enforce_ga(cg->shape,
cg->fgfs[gxx->sgfn], cg->fgfs[gxy->sgfn], cg->fgfs[gxz->sgfn],
cg->fgfs[gyy->sgfn], cg->fgfs[gyz->sgfn], cg->fgfs[gzz->sgfn],
cg->fgfs[Axx->sgfn], cg->fgfs[Axy->sgfn], cg->fgfs[Axz->sgfn],
cg->fgfs[Ayy->sgfn], cg->fgfs[Ayz->sgfn], cg->fgfs[Azz->sgfn]);
}
if (BP == sPp->data->ble)
break;
BP = BP->next;
}
sPp = sPp->next;
}
}
#endif
}
//================================================================================================
//================================================================================================
// This member function monitors stdin for an 'abort' input
//================================================================================================
bool bssn_class::check_Stdin_Abort()
{
fd_set readfds;
struct timeval timeout;
FD_ZERO(&readfds);
FD_SET(STDIN_FILENO, &readfds);
// Set timeout to 0 — perform a non-blocking check
timeout.tv_sec = 0;
timeout.tv_usec = 0;
int activity = select(STDIN_FILENO + 1, &readfds, nullptr, nullptr, &timeout);
if (activity > 0 && FD_ISSET(STDIN_FILENO, &readfds)) {
string input_abort;
if (cin >> input_abort) {
if (input_abort == "stop") {
return true;
}
}
}
return false;
}
//================================================================================================
Reference in New Issue View Git Blame Copy Permalink