64-BitBrainstorm_2026/AMSS-NCKU
8
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Optimize BSSN EScalar GPU path baseline

Browse Source
This commit is contained in:
CGH0S7
2026-05-02 18:19:15 +08:00
parent 52beb4d153
commit 59a216ad93
13 changed files with 1366 additions and 177 deletions
Download Patch File Download Diff File Expand all files Collapse all files

516
AMSS_NCKU_source/bssnEScalar_class.C
View File

@@ -15,10 +15,13 @@ using namespace std;
#include "misc.h"
#include "Ansorg.h"
#include "fmisc.h"
#include "Parallel.h"
#include "bssnEScalar_class.h"
#include "bssn_rhs.h"
#include "initial_puncture.h"
#include "Parallel.h"
#include "bssnEScalar_class.h"
#include "bssn_rhs.h"
#if USE_CUDA_BSSN
#include "bssn_rhs_cuda.h"
#endif
#include "initial_puncture.h"
#include "enforce_algebra.h"
#include "rungekutta4_rout.h"
#include "sommerfeld_rout.h"
@@ -26,14 +29,216 @@ using namespace std;
#include "shellfunctions.h"
#include "parameters.h"
#ifdef With_AHF
#include "derivatives.h"
#include "myglobal.h"
#endif
//================================================================================================
// Define bssnEScalar_class
#ifdef With_AHF
#include "derivatives.h"
#include "myglobal.h"
#endif
//================================================================================================
namespace {
int amss_escalar_analysis_map_every()
{
static int every = -1;
if (every < 0)
{
const char *env = getenv("AMSS_ANALYSIS_MAP_EVERY");
every = (env && atoi(env) > 0) ? atoi(env) : 1;
}
return every;
}
}
//================================================================================================
#if USE_CUDA_BSSN
extern "C" {
#ifdef fortran1
void set_escalar_parameter(double &, double &, double &, double &, double &);
#endif
#ifdef fortran2
void SET_ESCALAR_PARAMETER(double &, double &, double &, double &, double &);
#endif
#ifdef fortran3
void set_escalar_parameter_(double &, double &, double &, double &, double &);
#endif
}
namespace {
bool fill_bssn_cuda_views_prefix(Block *cg, MyList<var> *vars,
double **host_views,
double *propspeeds = nullptr,
double *soa_flat = nullptr)
{
int idx = 0;
while (vars && idx < BSSN_CUDA_STATE_COUNT)
{
host_views[idx] = cg->fgfs[vars->data->sgfn];
if (propspeeds)
propspeeds[idx] = vars->data->propspeed;
if (soa_flat)
{
soa_flat[3 * idx + 0] = vars->data->SoA[0];
soa_flat[3 * idx + 1] = vars->data->SoA[1];
soa_flat[3 * idx + 2] = vars->data->SoA[2];
}
vars = vars->next;
++idx;
}
return idx == BSSN_CUDA_STATE_COUNT;
}
void skip_bssn_cuda_prefix(MyList<var> *&a, MyList<var> *&b, MyList<var> *&c)
{
for (int i = 0; i < BSSN_CUDA_STATE_COUNT && a && b && c; ++i)
{
a = a->next;
b = b->next;
c = c->next;
}
}
void skip_bssn_cuda_prefix(MyList<var> *&a, MyList<var> *&b,
MyList<var> *&c, MyList<var> *&d)
{
for (int i = 0; i < BSSN_CUDA_STATE_COUNT && a && b && c && d; ++i)
{
a = a->next;
b = b->next;
c = c->next;
d = d->next;
}
}
MyList<var> *clone_var_list_prefix(MyList<var> *src, int count)
{
MyList<var> *dst = nullptr;
MyList<var> *tail = nullptr;
for (int i = 0; i < count && src; ++i, src = src->next)
{
MyList<var> *node = new MyList<var>(src->data);
if (!dst)
dst = node;
else
tail->next = node;
tail = node;
}
return dst;
}
void clear_var_list(MyList<var> *&list)
{
if (list)
{
list->clearList();
list = nullptr;
}
}
void download_bssn_cuda_prefix_if_present(MyList<Patch> *PatL,
MyList<var> *vars,
int myrank)
{
while (PatL)
{
MyList<Block> *BP = PatL->data->blb;
while (BP)
{
Block *cg = BP->data;
if (myrank == cg->rank)
{
double *views[BSSN_CUDA_STATE_COUNT];
if (fill_bssn_cuda_views_prefix(cg, vars, views))
bssn_cuda_download_resident_state_if_present(cg, cg->shape, views);
}
if (BP == PatL->data->ble)
break;
BP = BP->next;
}
PatL = PatL->next;
}
}
int run_bssn_escalar_cuda_substep(Block *cg,
MyList<var> *state_in_list,
MyList<var> *state_out_list,
Patch *patch,
double &dT_lev,
double &TRK4,
int &iter_count,
int &Symmetry,
int lev,
double &ndeps,
int &co,
double &chitiny,
var *Sphi_in, var *Spi_in,
var *Sphi_rhs, var *Spi_rhs,
var *rho, var *Sx, var *Sy, var *Sz,
var *Sxx, var *Sxy, var *Sxz,
var *Syy, var *Syz, var *Szz)
{
double *state_in[BSSN_CUDA_STATE_COUNT];
double *state_out[BSSN_CUDA_STATE_COUNT];
double propspeed[BSSN_CUDA_STATE_COUNT];
double soa_flat[3 * BSSN_CUDA_STATE_COUNT];
if (!fill_bssn_cuda_views_prefix(cg, state_in_list, state_in, propspeed, soa_flat) ||
!fill_bssn_cuda_views_prefix(cg, state_out_list, state_out))
return 1;
double a2 = 0.0, phi0 = 0.0, r0 = 0.0, sigma0 = 0.0, l2 = 0.0;
#ifdef fortran1
set_escalar_parameter(a2, phi0, r0, sigma0, l2);
#endif
#ifdef fortran2
SET_ESCALAR_PARAMETER(a2, phi0, r0, sigma0, l2);
#endif
#ifdef fortran3
set_escalar_parameter_(a2, phi0, r0, sigma0, l2);
#endif
int apply_enforce_ga = 0;
#if (AGM == 0)
apply_enforce_ga = 1;
#elif (AGM == 1)
apply_enforce_ga = (iter_count == 3) ? 1 : 0;
#endif
if (bssn_cuda_compute_escalar_matter(cg,
cg->shape, cg->X[0], cg->X[1], cg->X[2],
state_in,
cg->fgfs[Sphi_in->sgfn],
cg->fgfs[Spi_in->sgfn],
cg->fgfs[Sphi_rhs->sgfn],
cg->fgfs[Spi_rhs->sgfn],
a2, Symmetry, lev, ndeps, co, apply_enforce_ga))
return 1;
int apply_bam_bc = 0;
#if (SommerType == 0)
#ifndef WithShell
apply_bam_bc = (lev == 0) ? 1 : 0;
#endif
#endif
int use_zero_matter = 0;
int keep_resident_state = 1;
double **matter_precomputed = nullptr;
return bssn_cuda_rk4_substep(cg,
cg->shape, cg->X[0], cg->X[1], cg->X[2],
state_in, state_out, matter_precomputed,
propspeed, soa_flat, patch->bbox,
dT_lev, TRK4, iter_count, apply_bam_bc,
Symmetry, lev, ndeps, co,
use_zero_matter,
keep_resident_state, apply_enforce_ga, chitiny);
}
}
#endif
//================================================================================================
// Define bssnEScalar_class
// It inherits some members and methods from the parent class bssn_class and modifies others.
// The modified members and methods are defined below (and in the header bssnEScalar_class.h).
@@ -41,19 +246,27 @@ using namespace std;
//================================================================================================
bssnEScalar_class::bssnEScalar_class(double Couranti, double StartTimei, double TotalTimei,
bssnEScalar_class::bssnEScalar_class(double Couranti, double StartTimei, double TotalTimei,
double DumpTimei, double d2DumpTimei,
double CheckTimei, double AnasTimei,
int Symmetryi, int checkruni, char *checkfilenamei,
int Symmetryi, int checkruni, char *checkfilenamei,
double numepssi, double numepsbi, double numepshi,
int a_levi, int maxli, int decni, double maxrexi, double drexi)
: bssn_class(Couranti, StartTimei, TotalTimei,
DumpTimei, d2DumpTimei, CheckTimei, AnasTimei,
Symmetryi, checkruni, checkfilenamei, numepssi, numepsbi, numepshi,
a_levi, maxli, decni, maxrexi, drexi)
{
// setup Monitors
{
int a_levi, int maxli, int decni, double maxrexi, double drexi)
: bssn_class(Couranti, StartTimei, TotalTimei,
DumpTimei, d2DumpTimei, CheckTimei, AnasTimei,
Symmetryi, checkruni, checkfilenamei, numepssi, numepsbi, numepshi,
a_levi, maxli, decni, maxrexi, drexi)
{
BSSNStateList = nullptr;
BSSNSynchList_pre = nullptr;
BSSNSynchList_cor = nullptr;
ScalarSynchList_pre = nullptr;
ScalarSynchList_cor = nullptr;
sync_cache_scalar_pre = nullptr;
sync_cache_scalar_cor = nullptr;
// setup Monitors
{
char str[50];
stringstream a_stream;
a_stream.setf(ios::left);
@@ -106,12 +319,22 @@ void bssnEScalar_class::Initialize()
ConstraintList->insert(Cons_Gz);
DumpList->insert(Sphi0);
DumpList->insert(Spi0);
DumpList->insert(Cons_fR);
CheckPoint->addvariablelist(StateList);
CheckPoint->addvariablelist(OldStateList);
DumpList->insert(Sphi0);
DumpList->insert(Spi0);
DumpList->insert(Cons_fR);
#if USE_CUDA_BSSN
BSSNStateList = clone_var_list_prefix(StateList, BSSN_CUDA_STATE_COUNT);
BSSNSynchList_pre = clone_var_list_prefix(SynchList_pre, BSSN_CUDA_STATE_COUNT);
BSSNSynchList_cor = clone_var_list_prefix(SynchList_cor, BSSN_CUDA_STATE_COUNT);
ScalarSynchList_pre = new MyList<var>(Sphi);
ScalarSynchList_pre->insert(Spi);
ScalarSynchList_cor = new MyList<var>(Sphi1);
ScalarSynchList_cor->insert(Spi1);
#endif
CheckPoint->addvariablelist(StateList);
CheckPoint->addvariablelist(OldStateList);
int myrank = 0;
@@ -152,6 +375,12 @@ void bssnEScalar_class::Initialize()
#endif
Initialize_Level_Runtime();
#if USE_CUDA_BSSN
if (!sync_cache_scalar_pre)
sync_cache_scalar_pre = new Parallel::SyncCache[GH->levels];
if (!sync_cache_scalar_cor)
sync_cache_scalar_cor = new Parallel::SyncCache[GH->levels];
#endif
double h = GH->PatL[0]->data->blb->data->getdX(0);
for (int i = 1; i < dim; i++)
@@ -179,10 +408,34 @@ void bssnEScalar_class::Initialize()
//================================================================================================
bssnEScalar_class::~bssnEScalar_class()
{
delete Sphio;
delete Spio;
bssnEScalar_class::~bssnEScalar_class()
{
#if USE_CUDA_BSSN
clear_var_list(BSSNStateList);
clear_var_list(BSSNSynchList_pre);
clear_var_list(BSSNSynchList_cor);
clear_var_list(ScalarSynchList_pre);
clear_var_list(ScalarSynchList_cor);
if (sync_cache_scalar_pre)
{
const int levels = GH ? GH->levels : 0;
for (int i = 0; i < levels; ++i)
sync_cache_scalar_pre[i].destroy();
delete[] sync_cache_scalar_pre;
sync_cache_scalar_pre = nullptr;
}
if (sync_cache_scalar_cor)
{
const int levels = GH ? GH->levels : 0;
for (int i = 0; i < levels; ++i)
sync_cache_scalar_cor[i].destroy();
delete[] sync_cache_scalar_cor;
sync_cache_scalar_cor = nullptr;
}
#endif
delete Sphio;
delete Spio;
delete Sphi0;
delete Spi0;
delete Sphi;
@@ -729,20 +982,34 @@ void bssnEScalar_class::Step(int lev, int YN)
{
MyList<Block> *BP = Pp->data->blb;
while (BP)
{
Block *cg = BP->data;
if (myrank == cg->rank)
{
#if (AGM == 0)
f_enforce_ga(cg->shape,
cg->fgfs[gxx0->sgfn], cg->fgfs[gxy0->sgfn], cg->fgfs[gxz0->sgfn],
{
Block *cg = BP->data;
if (myrank == cg->rank)
{
#if !USE_CUDA_BSSN
#if (AGM == 0)
f_enforce_ga(cg->shape,
cg->fgfs[gxx0->sgfn], cg->fgfs[gxy0->sgfn], cg->fgfs[gxz0->sgfn],
cg->fgfs[gyy0->sgfn], cg->fgfs[gyz0->sgfn], cg->fgfs[gzz0->sgfn],
cg->fgfs[Axx0->sgfn], cg->fgfs[Axy0->sgfn], cg->fgfs[Axz0->sgfn],
cg->fgfs[Ayy0->sgfn], cg->fgfs[Ayz0->sgfn], cg->fgfs[Azz0->sgfn]);
#endif
if (f_compute_rhs_bssn_escalar(cg->shape, TRK4, cg->X[0], cg->X[1], cg->X[2],
cg->fgfs[phi0->sgfn], cg->fgfs[trK0->sgfn],
cg->fgfs[Axx0->sgfn], cg->fgfs[Axy0->sgfn], cg->fgfs[Axz0->sgfn],
cg->fgfs[Ayy0->sgfn], cg->fgfs[Ayz0->sgfn], cg->fgfs[Azz0->sgfn]);
#endif
#endif
bool used_gpu_substep = false;
if (
#if USE_CUDA_BSSN
((used_gpu_substep =
(run_bssn_escalar_cuda_substep(cg, StateList, SynchList_pre, Pp->data,
dT_lev, TRK4, iter_count, Symmetry, lev,
ndeps, pre, chitiny,
Sphi0, Spi0, Sphi_rhs, Spi_rhs,
rho, Sx, Sy, Sz, Sxx, Sxy, Sxz, Syy, Syz, Szz) == 0))
? 0
: 1) ||
#endif
(!used_gpu_substep && f_compute_rhs_bssn_escalar(cg->shape, TRK4, cg->X[0], cg->X[1], cg->X[2],
cg->fgfs[phi0->sgfn], cg->fgfs[trK0->sgfn],
cg->fgfs[gxx0->sgfn], cg->fgfs[gxy0->sgfn], cg->fgfs[gxz0->sgfn],
cg->fgfs[gyy0->sgfn], cg->fgfs[gyz0->sgfn], cg->fgfs[gzz0->sgfn],
cg->fgfs[Axx0->sgfn], cg->fgfs[Axy0->sgfn], cg->fgfs[Axz0->sgfn],
@@ -773,10 +1040,10 @@ void bssnEScalar_class::Step(int lev, int YN)
cg->fgfs[Gamzyy->sgfn], cg->fgfs[Gamzyz->sgfn], cg->fgfs[Gamzzz->sgfn],
cg->fgfs[Rxx->sgfn], cg->fgfs[Rxy->sgfn], cg->fgfs[Rxz->sgfn],
cg->fgfs[Ryy->sgfn], cg->fgfs[Ryz->sgfn], cg->fgfs[Rzz->sgfn],
cg->fgfs[Cons_Ham->sgfn],
cg->fgfs[Cons_Px->sgfn], cg->fgfs[Cons_Py->sgfn], cg->fgfs[Cons_Pz->sgfn],
cg->fgfs[Cons_Gx->sgfn], cg->fgfs[Cons_Gy->sgfn], cg->fgfs[Cons_Gz->sgfn],
Symmetry, lev, ndeps, pre))
cg->fgfs[Cons_Ham->sgfn],
cg->fgfs[Cons_Px->sgfn], cg->fgfs[Cons_Py->sgfn], cg->fgfs[Cons_Pz->sgfn],
cg->fgfs[Cons_Gx->sgfn], cg->fgfs[Cons_Gy->sgfn], cg->fgfs[Cons_Gz->sgfn],
Symmetry, lev, ndeps, pre)))
{
cout << "find NaN in domain: ("
<< cg->bbox[0] << ":" << cg->bbox[3] << ","
@@ -786,9 +1053,13 @@ void bssnEScalar_class::Step(int lev, int YN)
}
// rk4 substep and boundary
{
MyList<var> *varl0 = StateList, *varl = SynchList_pre, *varlrhs = RHSList; // we do not check the correspondence here
while (varl0)
{
MyList<var> *varl0 = StateList, *varl = SynchList_pre, *varlrhs = RHSList; // we do not check the correspondence here
#if USE_CUDA_BSSN
if (used_gpu_substep)
skip_bssn_cuda_prefix(varl0, varl, varlrhs);
#endif
while (varl0)
{
#ifndef WithShell
if (lev == 0) // sommerfeld indeed
@@ -823,8 +1094,9 @@ void bssnEScalar_class::Step(int lev, int YN)
varlrhs = varlrhs->next;
}
}
f_lowerboundset(cg->shape, cg->fgfs[phi->sgfn], chitiny);
}
if (!used_gpu_substep)
f_lowerboundset(cg->shape, cg->fgfs[phi->sgfn], chitiny);
}
if (BP == Pp->data->ble)
break;
BP = BP->next;
@@ -995,7 +1267,12 @@ void bssnEScalar_class::Step(int lev, int YN)
}
#endif
#if USE_CUDA_BSSN
Parallel::Sync_cached(GH->PatL[lev], BSSNSynchList_pre, Symmetry, sync_cache_pre[lev]);
Parallel::Sync_cached(GH->PatL[lev], ScalarSynchList_pre, Symmetry, sync_cache_scalar_pre[lev]);
#else
Parallel::Sync_cached(GH->PatL[lev], SynchList_pre, Symmetry, sync_cache_pre[lev]);
#endif
#ifdef WithShell
if (lev == 0)
@@ -1065,26 +1342,40 @@ void bssnEScalar_class::Step(int lev, int YN)
MyList<Block> *BP = Pp->data->blb;
while (BP)
{
Block *cg = BP->data;
if (myrank == cg->rank)
{
#if (AGM == 0)
f_enforce_ga(cg->shape,
cg->fgfs[gxx->sgfn], cg->fgfs[gxy->sgfn], cg->fgfs[gxz->sgfn],
Block *cg = BP->data;
if (myrank == cg->rank)
{
#if !USE_CUDA_BSSN
#if (AGM == 0)
f_enforce_ga(cg->shape,
cg->fgfs[gxx->sgfn], cg->fgfs[gxy->sgfn], cg->fgfs[gxz->sgfn],
cg->fgfs[gyy->sgfn], cg->fgfs[gyz->sgfn], cg->fgfs[gzz->sgfn],
cg->fgfs[Axx->sgfn], cg->fgfs[Axy->sgfn], cg->fgfs[Axz->sgfn],
cg->fgfs[Axx->sgfn], cg->fgfs[Axy->sgfn], cg->fgfs[Axz->sgfn],
cg->fgfs[Ayy->sgfn], cg->fgfs[Ayz->sgfn], cg->fgfs[Azz->sgfn]);
#elif (AGM == 1)
if (iter_count == 3)
f_enforce_ga(cg->shape,
cg->fgfs[gxx->sgfn], cg->fgfs[gxy->sgfn], cg->fgfs[gxz->sgfn],
cg->fgfs[gxx->sgfn], cg->fgfs[gxy->sgfn], cg->fgfs[gxz->sgfn],
cg->fgfs[gyy->sgfn], cg->fgfs[gyz->sgfn], cg->fgfs[gzz->sgfn],
cg->fgfs[Axx->sgfn], cg->fgfs[Axy->sgfn], cg->fgfs[Axz->sgfn],
cg->fgfs[Ayy->sgfn], cg->fgfs[Ayz->sgfn], cg->fgfs[Azz->sgfn]);
#endif
if (f_compute_rhs_bssn_escalar(cg->shape, TRK4, cg->X[0], cg->X[1], cg->X[2],
cg->fgfs[phi->sgfn], cg->fgfs[trK->sgfn],
cg->fgfs[Axx->sgfn], cg->fgfs[Axy->sgfn], cg->fgfs[Axz->sgfn],
cg->fgfs[Ayy->sgfn], cg->fgfs[Ayz->sgfn], cg->fgfs[Azz->sgfn]);
#endif
#endif
bool used_gpu_substep = false;
if (
#if USE_CUDA_BSSN
((used_gpu_substep =
(run_bssn_escalar_cuda_substep(cg, SynchList_pre, SynchList_cor, Pp->data,
dT_lev, TRK4, iter_count, Symmetry, lev,
ndeps, cor, chitiny,
Sphi, Spi, Sphi_rhs, Spi_rhs,
rho, Sx, Sy, Sz, Sxx, Sxy, Sxz, Syy, Syz, Szz) == 0))
? 0
: 1) ||
#endif
(!used_gpu_substep && f_compute_rhs_bssn_escalar(cg->shape, TRK4, cg->X[0], cg->X[1], cg->X[2],
cg->fgfs[phi->sgfn], cg->fgfs[trK->sgfn],
cg->fgfs[gxx->sgfn], cg->fgfs[gxy->sgfn], cg->fgfs[gxz->sgfn],
cg->fgfs[gyy->sgfn], cg->fgfs[gyz->sgfn], cg->fgfs[gzz->sgfn],
cg->fgfs[Axx->sgfn], cg->fgfs[Axy->sgfn], cg->fgfs[Axz->sgfn],
@@ -1116,10 +1407,10 @@ void bssnEScalar_class::Step(int lev, int YN)
cg->fgfs[Gamzyy->sgfn], cg->fgfs[Gamzyz->sgfn], cg->fgfs[Gamzzz->sgfn],
cg->fgfs[Rxx->sgfn], cg->fgfs[Rxy->sgfn], cg->fgfs[Rxz->sgfn],
cg->fgfs[Ryy->sgfn], cg->fgfs[Ryz->sgfn], cg->fgfs[Rzz->sgfn],
cg->fgfs[Cons_Ham->sgfn],
cg->fgfs[Cons_Px->sgfn], cg->fgfs[Cons_Py->sgfn], cg->fgfs[Cons_Pz->sgfn],
cg->fgfs[Cons_Gx->sgfn], cg->fgfs[Cons_Gy->sgfn], cg->fgfs[Cons_Gz->sgfn],
Symmetry, lev, ndeps, cor))
cg->fgfs[Cons_Ham->sgfn],
cg->fgfs[Cons_Px->sgfn], cg->fgfs[Cons_Py->sgfn], cg->fgfs[Cons_Pz->sgfn],
cg->fgfs[Cons_Gx->sgfn], cg->fgfs[Cons_Gy->sgfn], cg->fgfs[Cons_Gz->sgfn],
Symmetry, lev, ndeps, cor)))
{
cout << "find NaN in domain: ("
<< cg->bbox[0] << ":" << cg->bbox[3] << ","
@@ -1128,11 +1419,15 @@ void bssnEScalar_class::Step(int lev, int YN)
ERROR = 1;
}
// rk4 substep and boundary
{
MyList<var> *varl0 = StateList, *varl = SynchList_pre, *varl1 = SynchList_cor, *varlrhs = RHSList;
// we do not check the correspondence here
while (varl0)
{
MyList<var> *varl0 = StateList, *varl = SynchList_pre, *varl1 = SynchList_cor, *varlrhs = RHSList;
// we do not check the correspondence here
#if USE_CUDA_BSSN
if (used_gpu_substep)
skip_bssn_cuda_prefix(varl0, varl, varl1, varlrhs);
#endif
while (varl0)
{
#ifndef WithShell
if (lev == 0) // sommerfeld indeed
@@ -1168,8 +1463,9 @@ void bssnEScalar_class::Step(int lev, int YN)
varlrhs = varlrhs->next;
}
}
f_lowerboundset(cg->shape, cg->fgfs[phi1->sgfn], chitiny);
}
if (!used_gpu_substep)
f_lowerboundset(cg->shape, cg->fgfs[phi1->sgfn], chitiny);
}
if (BP == Pp->data->ble)
break;
BP = BP->next;
@@ -1351,7 +1647,12 @@ void bssnEScalar_class::Step(int lev, int YN)
}
#endif
#if USE_CUDA_BSSN
Parallel::Sync_cached(GH->PatL[lev], BSSNSynchList_cor, Symmetry, sync_cache_cor[lev]);
Parallel::Sync_cached(GH->PatL[lev], ScalarSynchList_cor, Symmetry, sync_cache_scalar_cor[lev]);
#else
Parallel::Sync_cached(GH->PatL[lev], SynchList_cor, Symmetry, sync_cache_cor[lev]);
#endif
#ifdef WithShell
if (lev == 0)
@@ -1451,9 +1752,13 @@ void bssnEScalar_class::Step(int lev, int YN)
}
}
#if (RPS == 0)
// mesh refinement boundary part
RestrictProlong(lev, YN, BB);
#if (RPS == 0)
// mesh refinement boundary part
#if USE_CUDA_BSSN
if (!getenv("AMSS_ESCALAR_SPLIT_RP") || atoi(getenv("AMSS_ESCALAR_SPLIT_RP")) == 0)
download_bssn_cuda_prefix_if_present(GH->PatL[lev], SynchList_cor, myrank);
#endif
RestrictProlong(lev, YN, BB);
#ifdef WithShell
if (lev == 0)
@@ -1472,15 +1777,15 @@ void bssnEScalar_class::Step(int lev, int YN)
}
#endif
#endif
// note the data structure before update
#endif
// note the data structure before update
// SynchList_cor 1 -----------
//
// StateList 0 -----------
//
// OldStateList old -----------
// update
Pp = GH->PatL[lev];
// update
Pp = GH->PatL[lev];
while (Pp)
{
MyList<Block> *BP = Pp->data->blb;
@@ -2053,17 +2358,34 @@ void bssnEScalar_class::Interp_Constraint()
//================================================================================================
void bssnEScalar_class::Constraint_Out()
{
// Use the same variables as in the parent class here
// Otherwise the correct time will not be passed
LastConsOut += dT * pow(0.5, Mymax(0, trfls));
if (LastConsOut >= AnasTime)
// Constraint violation
{
// recompute least the constraint data lost for moved new grid
for (int lev = 0; lev < GH->levels; lev++)
void bssnEScalar_class::Constraint_Out()
{
// Use the same variables as in the parent class here
// Otherwise the correct time will not be passed
LastConsOut += dT * pow(0.5, Mymax(0, trfls));
if (LastConsOut >= AnasTime)
// Constraint violation
{
const int constraint_map_every = amss_escalar_analysis_map_every();
static long long constraint_map_counter = 0;
const bool refresh_constraints =
constraint_map_every <= 1 ||
(constraint_map_counter % constraint_map_every) == 0;
constraint_map_counter++;
if (!refresh_constraints)
{
LastConsOut = 0;
return;
}
#if USE_CUDA_BSSN
for (int lev = 0; lev < GH->levels; lev++)
download_bssn_cuda_prefix_if_present(GH->PatL[lev], StateList, myrank);
#endif
// recompute least the constraint data lost for moved new grid
for (int lev = 0; lev < GH->levels; lev++)
{
// make sure the data consistent for higher levels
{