64-BitBrainstorm_2026/AMSS-NCKU
8
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Optimize BSSN-EScalar CUDA path

Browse Source
This commit is contained in:
CGH0S7
2026-05-05 10:47:46 +08:00
parent 06f62dee36
commit 85fe29cc2e
9 changed files with 1821 additions and 276 deletions
Download Patch File Download Diff File Expand all files Collapse all files

612
AMSS_NCKU_source/bssnEScalar_class.C
View File

@@ -24,16 +24,165 @@ using namespace std;
#include "sommerfeld_rout.h"
#include "getnp4.h"
#include "shellfunctions.h"
#include "parameters.h"
#include "parameters.h"
#if USE_CUDA_BSSN
#include "bssn_rhs_cuda.h"
#endif
#ifdef With_AHF
#include "derivatives.h"
#include "myglobal.h"
#endif
//================================================================================================
// Define bssnEScalar_class
//================================================================================================
namespace
{
#if USE_CUDA_BSSN
bool fill_bssn_escalar_cuda_views(Block *cg, MyList<var> *vars,
double **host_views,
double *propspeeds = 0,
double *soa_flat = 0)
{
int idx = 0;
while (vars && idx < BSSN_ESCALAR_CUDA_STATE_COUNT)
{
host_views[idx] = cg->fgfs[vars->data->sgfn];
if (propspeeds)
propspeeds[idx] = vars->data->propspeed;
if (soa_flat)
{
soa_flat[3 * idx + 0] = vars->data->SoA[0];
soa_flat[3 * idx + 1] = vars->data->SoA[1];
soa_flat[3 * idx + 2] = vars->data->SoA[2];
}
vars = vars->next;
++idx;
}
return idx == BSSN_ESCALAR_CUDA_STATE_COUNT && vars == 0;
}
bool bssn_escalar_cuda_use_resident_sync(int lev)
{
#ifdef WithShell
(void)lev;
return false;
#else
return true;
#endif
}
bool bssn_escalar_cuda_keep_resident_after_step(int lev, int trfls_in, int analysis_lev)
{
static int keep_all_levels = -1;
if (keep_all_levels < 0)
{
const char *env = getenv("AMSS_CUDA_ESCALAR_KEEP_ALL_LEVELS");
keep_all_levels = (env && atoi(env) != 0) ? 1 : 0;
}
static int enabled = -1;
if (enabled < 0)
{
const char *env = getenv("AMSS_CUDA_ESCALAR_KEEP_RESIDENT_AFTER_STEP");
enabled = (env && atoi(env) != 0) ? 1 : 0;
}
if (!enabled)
return false;
if (lev == analysis_lev)
return false;
if (keep_all_levels)
return true;
return lev < trfls_in;
}
bool bssn_escalar_sync_merged_enabled()
{
static int enabled = -1;
if (enabled < 0)
{
const char *env = getenv("AMSS_ESCALAR_SYNC_MERGED");
enabled = (env && atoi(env) != 0) ? 1 : 0;
}
return enabled != 0;
}
void bssn_escalar_sync_level(MyList<Patch> *PatL, MyList<var> *VarList, int Symmetry)
{
if (bssn_escalar_sync_merged_enabled())
Parallel::Sync_merged(PatL, VarList, Symmetry);
else
Parallel::Sync(PatL, VarList, Symmetry);
}
bool bssn_escalar_timing_enabled()
{
static int enabled = -1;
if (enabled < 0)
{
const char *env = getenv("AMSS_ESCALAR_STEP_TIMING");
enabled = (env && atoi(env) != 0) ? 1 : 0;
}
return enabled != 0;
}
void bssn_escalar_timing_report(int myrank, int lev, int YN, double total, double rhs,
double sync, double bh, double analysis, double swap,
double resident, double rp)
{
if (!bssn_escalar_timing_enabled())
return;
double local[8] = {total, rhs, sync, bh, analysis, swap, resident, rp};
double maxv[8] = {};
MPI_Reduce(local, maxv, 8, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD);
if (myrank == 0)
fprintf(stderr,
"[AMSS-ESCALAR-STEP] lev=%d YN=%d total=%.6f rhs=%.6f sync=%.6f "
"bh=%.6f analysis=%.6f swap=%.6f resident=%.6f rp=%.6f other=%.6f\n",
lev, YN, maxv[0], maxv[1], maxv[2], maxv[3], maxv[4], maxv[5],
maxv[6], maxv[7],
maxv[0] - maxv[1] - maxv[2] - maxv[3] - maxv[4] - maxv[5] - maxv[6] - maxv[7]);
}
void bssn_escalar_cuda_download_level_state(MyList<Patch> *PatL, MyList<var> *vars,
int myrank, bool release_ctx)
{
MyList<Patch> *Pp = PatL;
while (Pp)
{
MyList<Block> *BP = Pp->data->blb;
while (BP)
{
Block *cg = BP->data;
if (myrank == cg->rank && bssn_cuda_has_resident_state(cg))
{
double *state_out[BSSN_ESCALAR_CUDA_STATE_COUNT];
if (!fill_bssn_escalar_cuda_views(cg, vars, state_out))
{
cout << "CUDA BSSN-EScalar resident state list mismatch during download" << endl;
MPI_Abort(MPI_COMM_WORLD, 1);
}
if (bssn_escalar_cuda_download_resident_state(cg, cg->shape, state_out))
{
cout << "CUDA BSSN-EScalar resident state download failed" << endl;
MPI_Abort(MPI_COMM_WORLD, 1);
}
if (release_ctx)
bssn_cuda_release_step_ctx(cg);
}
if (BP == Pp->data->ble)
break;
BP = BP->next;
}
Pp = Pp->next;
}
}
#endif
}
//================================================================================================
// Define bssnEScalar_class
// It inherits some members and methods from the parent class bssn_class and modifies others.
// The modified members and methods are defined below (and in the header bssnEScalar_class.h).
@@ -177,11 +326,16 @@ void bssnEScalar_class::Initialize()
//================================================================================================
bssnEScalar_class::~bssnEScalar_class()
{
delete Sphio;
delete Spio;
delete Sphi0;
bssnEScalar_class::~bssnEScalar_class()
{
#if USE_CUDA_BSSN
for (int lev = 0; GH && lev < GH->levels; ++lev)
bssn_escalar_cuda_download_level_state(GH->PatL[lev], StateList, myrank, true);
#endif
delete Sphio;
delete Spio;
delete Sphi0;
delete Spi0;
delete Sphi;
delete Spi;
@@ -707,7 +861,12 @@ void bssnEScalar_class::Read_Pablo()
void bssnEScalar_class::Step(int lev, int YN)
{
double dT_lev = dT * pow(0.5, Mymax(lev, trfls));
double dT_lev = dT * pow(0.5, Mymax(lev, trfls));
#if USE_CUDA_BSSN
const bool use_cuda_resident_sync = bssn_escalar_cuda_use_resident_sync(lev);
#else
const bool use_cuda_resident_sync = false;
#endif
#ifdef With_AHF
AH_Step_Find(lev, dT_lev);
#endif
@@ -716,13 +875,23 @@ void bssnEScalar_class::Step(int lev, int YN)
if (lev < GH->movls)
ndeps = numepsb;
double TRK4 = PhysTime;
int iter_count = 0; // count RK4 substeps
int pre = 0, cor = 1;
int ERROR = 0;
MyList<ss_patch> *sPp;
// Predictor
MyList<Patch> *Pp = GH->PatL[lev];
int iter_count = 0; // count RK4 substeps
int pre = 0, cor = 1;
int ERROR = 0;
const bool escalar_step_timing = bssn_escalar_timing_enabled();
const double escalar_step_t0 = escalar_step_timing ? MPI_Wtime() : 0.0;
double escalar_t_rhs = 0.0;
double escalar_t_sync = 0.0;
double escalar_t_bh = 0.0;
double escalar_t_analysis = 0.0;
double escalar_t_swap = 0.0;
double escalar_t_resident = 0.0;
double escalar_t_rp = 0.0;
MyList<ss_patch> *sPp;
// Predictor
double escalar_t0 = escalar_step_timing ? MPI_Wtime() : 0.0;
MyList<Patch> *Pp = GH->PatL[lev];
while (Pp)
{
MyList<Block> *BP = Pp->data->blb;
@@ -731,15 +900,60 @@ void bssnEScalar_class::Step(int lev, int YN)
Block *cg = BP->data;
if (myrank == cg->rank)
{
#if (AGM == 0)
f_enforce_ga(cg->shape,
cg->fgfs[gxx0->sgfn], cg->fgfs[gxy0->sgfn], cg->fgfs[gxz0->sgfn],
cg->fgfs[gyy0->sgfn], cg->fgfs[gyz0->sgfn], cg->fgfs[gzz0->sgfn],
cg->fgfs[Axx0->sgfn], cg->fgfs[Axy0->sgfn], cg->fgfs[Axz0->sgfn],
cg->fgfs[Ayy0->sgfn], cg->fgfs[Ayz0->sgfn], cg->fgfs[Azz0->sgfn]);
#endif
if (f_compute_rhs_bssn_escalar(cg->shape, TRK4, cg->X[0], cg->X[1], cg->X[2],
#if (AGM == 0)
#if !USE_CUDA_BSSN
f_enforce_ga(cg->shape,
cg->fgfs[gxx0->sgfn], cg->fgfs[gxy0->sgfn], cg->fgfs[gxz0->sgfn],
cg->fgfs[gyy0->sgfn], cg->fgfs[gyz0->sgfn], cg->fgfs[gzz0->sgfn],
cg->fgfs[Axx0->sgfn], cg->fgfs[Axy0->sgfn], cg->fgfs[Axz0->sgfn],
cg->fgfs[Ayy0->sgfn], cg->fgfs[Ayz0->sgfn], cg->fgfs[Azz0->sgfn]);
#endif
#endif
bool used_gpu_substep = false;
#if USE_CUDA_BSSN
{
double *state_in[BSSN_ESCALAR_CUDA_STATE_COUNT];
double *state_out[BSSN_ESCALAR_CUDA_STATE_COUNT];
double propspeed[BSSN_ESCALAR_CUDA_STATE_COUNT];
double soa_flat[3 * BSSN_ESCALAR_CUDA_STATE_COUNT];
if (!fill_bssn_escalar_cuda_views(cg, StateList, state_in, propspeed, soa_flat) ||
!fill_bssn_escalar_cuda_views(cg, SynchList_pre, state_out))
{
cout << "CUDA BSSN-EScalar state list mismatch on predictor step" << endl;
MPI_Abort(MPI_COMM_WORLD, 1);
}
int apply_bam_bc = 0;
int apply_enforce_ga = 0;
#if (AGM == 0)
apply_enforce_ga = 1;
#endif
#if (SommerType == 0)
#ifndef WithShell
apply_bam_bc = (lev == 0) ? 1 : 0;
#endif
#endif
int keep_resident_state = use_cuda_resident_sync ? 1 : 0;
if (bssn_escalar_cuda_rk4_substep(cg,
cg->shape, cg->X[0], cg->X[1], cg->X[2],
state_in, state_out,
propspeed, soa_flat, Pp->data->bbox,
dT_lev, TRK4, iter_count, apply_bam_bc,
Symmetry, lev, ndeps, pre,
keep_resident_state, apply_enforce_ga, chitiny))
{
cout << "CUDA BSSN-EScalar predictor substep failed in domain: ("
<< cg->bbox[0] << ":" << cg->bbox[3] << ","
<< cg->bbox[1] << ":" << cg->bbox[4] << ","
<< cg->bbox[2] << ":" << cg->bbox[5] << ")" << endl;
ERROR = 1;
}
used_gpu_substep = true;
}
#endif
if (!used_gpu_substep &&
f_compute_rhs_bssn_escalar(cg->shape, TRK4, cg->X[0], cg->X[1], cg->X[2],
cg->fgfs[phi0->sgfn], cg->fgfs[trK0->sgfn],
cg->fgfs[gxx0->sgfn], cg->fgfs[gxy0->sgfn], cg->fgfs[gxz0->sgfn],
cg->fgfs[gyy0->sgfn], cg->fgfs[gyz0->sgfn], cg->fgfs[gzz0->sgfn],
@@ -783,9 +997,11 @@ void bssnEScalar_class::Step(int lev, int YN)
ERROR = 1;
}
// rk4 substep and boundary
{
MyList<var> *varl0 = StateList, *varl = SynchList_pre, *varlrhs = RHSList; // we do not check the correspondence here
if (!used_gpu_substep)
{
// rk4 substep and boundary
{
MyList<var> *varl0 = StateList, *varl = SynchList_pre, *varlrhs = RHSList; // we do not check the correspondence here
while (varl0)
{
#ifndef WithShell
@@ -820,9 +1036,10 @@ void bssnEScalar_class::Step(int lev, int YN)
varl = varl->next;
varlrhs = varlrhs->next;
}
}
f_lowerboundset(cg->shape, cg->fgfs[phi->sgfn], chitiny);
}
}
f_lowerboundset(cg->shape, cg->fgfs[phi->sgfn], chitiny);
}
}
if (BP == Pp->data->ble)
break;
BP = BP->next;
@@ -834,19 +1051,21 @@ void bssnEScalar_class::Step(int lev, int YN)
int erh = ERROR;
MPI_Allreduce(&erh, &ERROR, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD);
}
if (ERROR)
{
if (ERROR)
{
Parallel::Dump_Data(GH->PatL[lev], StateList, 0, PhysTime, dT_lev);
if (myrank == 0)
{
if (ErrorMonitor->outfile)
ErrorMonitor->outfile << "find NaN in state variables at t = " << PhysTime
<< ", lev = " << lev << endl;
MPI_Abort(MPI_COMM_WORLD, 1);
}
}
#ifdef WithShell
MPI_Abort(MPI_COMM_WORLD, 1);
}
}
if (escalar_step_timing)
escalar_t_rhs += MPI_Wtime() - escalar_t0;
#ifdef WithShell
// evolve Shell Patches
if (lev == 0)
{
@@ -993,7 +1212,14 @@ void bssnEScalar_class::Step(int lev, int YN)
}
#endif
Parallel::Sync_cached(GH->PatL[lev], SynchList_pre, Symmetry, sync_cache_pre[lev]);
escalar_t0 = escalar_step_timing ? MPI_Wtime() : 0.0;
#if USE_CUDA_BSSN
bssn_escalar_sync_level(GH->PatL[lev], SynchList_pre, Symmetry);
#else
Parallel::Sync(GH->PatL[lev], SynchList_pre, Symmetry);
#endif
if (escalar_step_timing)
escalar_t_sync += MPI_Wtime() - escalar_t0;
#ifdef WithShell
if (lev == 0)
@@ -1013,10 +1239,14 @@ void bssnEScalar_class::Step(int lev, int YN)
}
#endif
// for black hole position
if (BH_num > 0 && lev == GH->levels - 1)
{
compute_Porg_rhs(Porg0, Porg_rhs, Sfx0, Sfy0, Sfz0, lev);
// for black hole position
if (BH_num > 0 && lev == GH->levels - 1)
{
escalar_t0 = escalar_step_timing ? MPI_Wtime() : 0.0;
#if USE_CUDA_BSSN
(void)use_cuda_resident_sync;
#endif
compute_Porg_rhs(Porg0, Porg_rhs, Sfx0, Sfy0, Sfz0, lev);
for (int ithBH = 0; ithBH < BH_num; ithBH++)
{
f_rungekutta4_scalar(dT_lev, Porg0[ithBH][0], Porg[ithBH][0], Porg_rhs[ithBH][0], iter_count);
@@ -1041,19 +1271,29 @@ void bssnEScalar_class::Step(int lev, int YN)
DG_List->insert(Sfy0);
DG_List->insert(Sfz0);
Parallel::Dump_Data(GH->PatL[lev], DG_List, 0, PhysTime, dT_lev);
DG_List->clearList();
}
}
}
DG_List->clearList();
}
}
if (escalar_step_timing)
escalar_t_bh += MPI_Wtime() - escalar_t0;
}
// data analysis part
// Warning NOTE: the variables1 are used as temp storege room
if (lev == a_lev)
{
AnalysisStuff_EScalar(lev, dT_lev);
}
// corrector
for (iter_count = 1; iter_count < 4; iter_count++)
{
if (lev == a_lev)
{
escalar_t0 = escalar_step_timing ? MPI_Wtime() : 0.0;
#if USE_CUDA_BSSN
if (use_cuda_resident_sync)
bssn_escalar_cuda_download_level_state(GH->PatL[lev], SynchList_pre, myrank, false);
#endif
AnalysisStuff_EScalar(lev, dT_lev);
if (escalar_step_timing)
escalar_t_analysis += MPI_Wtime() - escalar_t0;
}
// corrector
for (iter_count = 1; iter_count < 4; iter_count++)
{
escalar_t0 = escalar_step_timing ? MPI_Wtime() : 0.0;
// for RK4: t0, t0+dt/2, t0+dt/2, t0+dt;
if (iter_count == 1 || iter_count == 3)
TRK4 += dT_lev / 2;
@@ -1066,22 +1306,67 @@ void bssnEScalar_class::Step(int lev, int YN)
Block *cg = BP->data;
if (myrank == cg->rank)
{
#if (AGM == 0)
f_enforce_ga(cg->shape,
cg->fgfs[gxx->sgfn], cg->fgfs[gxy->sgfn], cg->fgfs[gxz->sgfn],
cg->fgfs[gyy->sgfn], cg->fgfs[gyz->sgfn], cg->fgfs[gzz->sgfn],
cg->fgfs[Axx->sgfn], cg->fgfs[Axy->sgfn], cg->fgfs[Axz->sgfn],
cg->fgfs[Ayy->sgfn], cg->fgfs[Ayz->sgfn], cg->fgfs[Azz->sgfn]);
#elif (AGM == 1)
if (iter_count == 3)
f_enforce_ga(cg->shape,
#if (AGM == 0)
#if !USE_CUDA_BSSN
f_enforce_ga(cg->shape,
cg->fgfs[gxx->sgfn], cg->fgfs[gxy->sgfn], cg->fgfs[gxz->sgfn],
cg->fgfs[gyy->sgfn], cg->fgfs[gyz->sgfn], cg->fgfs[gzz->sgfn],
cg->fgfs[Axx->sgfn], cg->fgfs[Axy->sgfn], cg->fgfs[Axz->sgfn],
cg->fgfs[Ayy->sgfn], cg->fgfs[Ayz->sgfn], cg->fgfs[Azz->sgfn]);
#endif
#elif (AGM == 1)
if (iter_count == 3)
f_enforce_ga(cg->shape,
cg->fgfs[gxx->sgfn], cg->fgfs[gxy->sgfn], cg->fgfs[gxz->sgfn],
cg->fgfs[gyy->sgfn], cg->fgfs[gyz->sgfn], cg->fgfs[gzz->sgfn],
cg->fgfs[Axx->sgfn], cg->fgfs[Axy->sgfn], cg->fgfs[Axz->sgfn],
cg->fgfs[Ayy->sgfn], cg->fgfs[Ayz->sgfn], cg->fgfs[Azz->sgfn]);
#endif
if (f_compute_rhs_bssn_escalar(cg->shape, TRK4, cg->X[0], cg->X[1], cg->X[2],
#endif
bool used_gpu_substep = false;
#if USE_CUDA_BSSN
{
double *state_in[BSSN_ESCALAR_CUDA_STATE_COUNT];
double *state_out[BSSN_ESCALAR_CUDA_STATE_COUNT];
double propspeed[BSSN_ESCALAR_CUDA_STATE_COUNT];
double soa_flat[3 * BSSN_ESCALAR_CUDA_STATE_COUNT];
if (!fill_bssn_escalar_cuda_views(cg, SynchList_pre, state_in, propspeed, soa_flat) ||
!fill_bssn_escalar_cuda_views(cg, SynchList_cor, state_out))
{
cout << "CUDA BSSN-EScalar state list mismatch on corrector step" << endl;
MPI_Abort(MPI_COMM_WORLD, 1);
}
int apply_bam_bc = 0;
int apply_enforce_ga = 0;
#if (AGM == 0)
apply_enforce_ga = 1;
#endif
#if (SommerType == 0)
#ifndef WithShell
apply_bam_bc = (lev == 0) ? 1 : 0;
#endif
#endif
int keep_resident_state = use_cuda_resident_sync ? 1 : 0;
if (bssn_escalar_cuda_rk4_substep(cg,
cg->shape, cg->X[0], cg->X[1], cg->X[2],
state_in, state_out,
propspeed, soa_flat, Pp->data->bbox,
dT_lev, TRK4, iter_count, apply_bam_bc,
Symmetry, lev, ndeps, cor,
keep_resident_state, apply_enforce_ga, chitiny))
{
cout << "CUDA BSSN-EScalar corrector substep failed in domain: ("
<< cg->bbox[0] << ":" << cg->bbox[3] << ","
<< cg->bbox[1] << ":" << cg->bbox[4] << ","
<< cg->bbox[2] << ":" << cg->bbox[5] << ")" << endl;
ERROR = 1;
}
used_gpu_substep = true;
}
#endif
if (!used_gpu_substep &&
f_compute_rhs_bssn_escalar(cg->shape, TRK4, cg->X[0], cg->X[1], cg->X[2],
cg->fgfs[phi->sgfn], cg->fgfs[trK->sgfn],
cg->fgfs[gxx->sgfn], cg->fgfs[gxy->sgfn], cg->fgfs[gxz->sgfn],
cg->fgfs[gyy->sgfn], cg->fgfs[gyz->sgfn], cg->fgfs[gzz->sgfn],
@@ -1125,9 +1410,11 @@ void bssnEScalar_class::Step(int lev, int YN)
<< cg->bbox[2] << ":" << cg->bbox[5] << ")" << endl;
ERROR = 1;
}
// rk4 substep and boundary
{
MyList<var> *varl0 = StateList, *varl = SynchList_pre, *varl1 = SynchList_cor, *varlrhs = RHSList;
if (!used_gpu_substep)
{
// rk4 substep and boundary
{
MyList<var> *varl0 = StateList, *varl = SynchList_pre, *varl1 = SynchList_cor, *varlrhs = RHSList;
// we do not check the correspondence here
while (varl0)
@@ -1165,9 +1452,10 @@ void bssnEScalar_class::Step(int lev, int YN)
varl1 = varl1->next;
varlrhs = varlrhs->next;
}
}
f_lowerboundset(cg->shape, cg->fgfs[phi1->sgfn], chitiny);
}
}
f_lowerboundset(cg->shape, cg->fgfs[phi1->sgfn], chitiny);
}
}
if (BP == Pp->data->ble)
break;
BP = BP->next;
@@ -1180,8 +1468,8 @@ void bssnEScalar_class::Step(int lev, int YN)
int erh = ERROR;
MPI_Allreduce(&erh, &ERROR, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD);
}
if (ERROR)
{
if (ERROR)
{
Parallel::Dump_Data(GH->PatL[lev], SynchList_pre, 0, PhysTime, dT_lev);
if (myrank == 0)
{
@@ -1189,11 +1477,13 @@ void bssnEScalar_class::Step(int lev, int YN)
ErrorMonitor->outfile << "find NaN in RK4 substep#" << iter_count
<< " variables at t = " << PhysTime
<< ", lev = " << lev << endl;
MPI_Abort(MPI_COMM_WORLD, 1);
}
}
#ifdef WithShell
MPI_Abort(MPI_COMM_WORLD, 1);
}
}
if (escalar_step_timing)
escalar_t_rhs += MPI_Wtime() - escalar_t0;
#ifdef WithShell
// evolve Shell Patches
if (lev == 0)
{
@@ -1349,7 +1639,14 @@ void bssnEScalar_class::Step(int lev, int YN)
}
#endif
Parallel::Sync_cached(GH->PatL[lev], SynchList_cor, Symmetry, sync_cache_cor[lev]);
escalar_t0 = escalar_step_timing ? MPI_Wtime() : 0.0;
#if USE_CUDA_BSSN
bssn_escalar_sync_level(GH->PatL[lev], SynchList_cor, Symmetry);
#else
Parallel::Sync(GH->PatL[lev], SynchList_cor, Symmetry);
#endif
if (escalar_step_timing)
escalar_t_sync += MPI_Wtime() - escalar_t0;
#ifdef WithShell
if (lev == 0)
@@ -1368,10 +1665,14 @@ void bssnEScalar_class::Step(int lev, int YN)
}
}
#endif
// for black hole position
if (BH_num > 0 && lev == GH->levels - 1)
{
compute_Porg_rhs(Porg, Porg1, Sfx, Sfy, Sfz, lev);
// for black hole position
if (BH_num > 0 && lev == GH->levels - 1)
{
escalar_t0 = escalar_step_timing ? MPI_Wtime() : 0.0;
#if USE_CUDA_BSSN
(void)use_cuda_resident_sync;
#endif
compute_Porg_rhs(Porg, Porg1, Sfx, Sfy, Sfz, lev);
for (int ithBH = 0; ithBH < BH_num; ithBH++)
{
f_rungekutta4_scalar(dT_lev, Porg0[ithBH][0], Porg1[ithBH][0], Porg_rhs[ithBH][0], iter_count);
@@ -1396,14 +1697,17 @@ void bssnEScalar_class::Step(int lev, int YN)
DG_List->insert(Sfy0);
DG_List->insert(Sfz0);
Parallel::Dump_Data(GH->PatL[lev], DG_List, 0, PhysTime, dT_lev);
DG_List->clearList();
}
}
}
// swap time level
if (iter_count < 3)
{
Pp = GH->PatL[lev];
DG_List->clearList();
}
}
if (escalar_step_timing)
escalar_t_bh += MPI_Wtime() - escalar_t0;
}
// swap time level
if (iter_count < 3)
{
escalar_t0 = escalar_step_timing ? MPI_Wtime() : 0.0;
Pp = GH->PatL[lev];
while (Pp)
{
MyList<Block> *BP = Pp->data->blb;
@@ -1444,16 +1748,32 @@ void bssnEScalar_class::Step(int lev, int YN)
Porg[ithBH][0] = Porg1[ithBH][0];
Porg[ithBH][1] = Porg1[ithBH][1];
Porg[ithBH][2] = Porg1[ithBH][2];
}
}
}
}
#if (RPS == 0)
// mesh refinement boundary part
RestrictProlong(lev, YN, BB);
#ifdef WithShell
}
}
if (escalar_step_timing)
escalar_t_swap += MPI_Wtime() - escalar_t0;
}
}
#if USE_CUDA_BSSN
if (use_cuda_resident_sync)
{
escalar_t0 = escalar_step_timing ? MPI_Wtime() : 0.0;
if (!bssn_escalar_cuda_keep_resident_after_step(lev, trfls, a_lev))
bssn_escalar_cuda_download_level_state(GH->PatL[lev], SynchList_cor, myrank, true);
if (escalar_step_timing)
escalar_t_resident += MPI_Wtime() - escalar_t0;
}
#endif
#if (RPS == 0)
// mesh refinement boundary part
escalar_t0 = escalar_step_timing ? MPI_Wtime() : 0.0;
RestrictProlong(lev, YN, BB);
if (escalar_step_timing)
escalar_t_rp += MPI_Wtime() - escalar_t0;
#ifdef WithShell
if (lev == 0)
{
clock_t prev_clock, curr_clock;
@@ -1477,8 +1797,9 @@ void bssnEScalar_class::Step(int lev, int YN)
// StateList 0 -----------
//
// OldStateList old -----------
// update
Pp = GH->PatL[lev];
// update
escalar_t0 = escalar_step_timing ? MPI_Wtime() : 0.0;
Pp = GH->PatL[lev];
while (Pp)
{
MyList<Block> *BP = Pp->data->blb;
@@ -1520,10 +1841,18 @@ void bssnEScalar_class::Step(int lev, int YN)
{
Porg0[ithBH][0] = Porg1[ithBH][0];
Porg0[ithBH][1] = Porg1[ithBH][1];
Porg0[ithBH][2] = Porg1[ithBH][2];
}
}
}
Porg0[ithBH][2] = Porg1[ithBH][2];
}
}
if (escalar_step_timing)
{
escalar_t_swap += MPI_Wtime() - escalar_t0;
bssn_escalar_timing_report(myrank, lev, YN, MPI_Wtime() - escalar_step_t0,
escalar_t_rhs, escalar_t_sync, escalar_t_bh,
escalar_t_analysis, escalar_t_swap,
escalar_t_resident, escalar_t_rp);
}
}
//================================================================================================
@@ -2074,14 +2403,44 @@ void bssnEScalar_class::Constraint_Out()
MyList<Block> *BP = Pp->data->blb;
while (BP)
{
Block *cg = BP->data;
if (myrank == cg->rank)
{
if (lev > 0)
f_compute_rhs_bssn_escalar(cg->shape, TRK4, cg->X[0], cg->X[1], cg->X[2],
cg->fgfs[phi0->sgfn], cg->fgfs[trK0->sgfn],
cg->fgfs[gxx0->sgfn], cg->fgfs[gxy0->sgfn], cg->fgfs[gxz0->sgfn],
cg->fgfs[gyy0->sgfn], cg->fgfs[gyz0->sgfn], cg->fgfs[gzz0->sgfn],
Block *cg = BP->data;
if (myrank == cg->rank)
{
bool used_cuda_constraints = false;
#if USE_CUDA_BSSN
{
double *state_in[BSSN_ESCALAR_CUDA_STATE_COUNT];
if (!fill_bssn_escalar_cuda_views(cg, StateList, state_in))
{
cout << "CUDA BSSN-EScalar constraint state list mismatch" << endl;
MPI_Abort(MPI_COMM_WORLD, 1);
}
double *constraint_out[8] = {
cg->fgfs[Cons_Ham->sgfn], cg->fgfs[Cons_Px->sgfn],
cg->fgfs[Cons_Py->sgfn], cg->fgfs[Cons_Pz->sgfn],
cg->fgfs[Cons_Gx->sgfn], cg->fgfs[Cons_Gy->sgfn],
cg->fgfs[Cons_Gz->sgfn], cg->fgfs[Cons_fR->sgfn]};
int lev_arg = lev;
int sym_arg = Symmetry;
double eps_arg = ndeps;
if (bssn_escalar_cuda_compute_constraints(cg->shape, cg->X[0], cg->X[1], cg->X[2],
state_in, constraint_out,
sym_arg, lev_arg, eps_arg))
{
cout << "CUDA BSSN-EScalar constraint compute failed in domain: ("
<< cg->bbox[0] << ":" << cg->bbox[3] << ","
<< cg->bbox[1] << ":" << cg->bbox[4] << ","
<< cg->bbox[2] << ":" << cg->bbox[5] << ")" << endl;
MPI_Abort(MPI_COMM_WORLD, 1);
}
used_cuda_constraints = true;
}
#endif
if (!used_cuda_constraints && lev > 0)
f_compute_rhs_bssn_escalar(cg->shape, TRK4, cg->X[0], cg->X[1], cg->X[2],
cg->fgfs[phi0->sgfn], cg->fgfs[trK0->sgfn],
cg->fgfs[gxx0->sgfn], cg->fgfs[gxy0->sgfn], cg->fgfs[gxz0->sgfn],
cg->fgfs[gyy0->sgfn], cg->fgfs[gyz0->sgfn], cg->fgfs[gzz0->sgfn],
cg->fgfs[Axx0->sgfn], cg->fgfs[Axy0->sgfn], cg->fgfs[Axz0->sgfn],
cg->fgfs[Ayy0->sgfn], cg->fgfs[Ayz0->sgfn], cg->fgfs[Azz0->sgfn],
cg->fgfs[Gmx0->sgfn], cg->fgfs[Gmy0->sgfn], cg->fgfs[Gmz0->sgfn],
@@ -2110,15 +2469,16 @@ void bssnEScalar_class::Constraint_Out()
cg->fgfs[Gamzyy->sgfn], cg->fgfs[Gamzyz->sgfn], cg->fgfs[Gamzzz->sgfn],
cg->fgfs[Rxx->sgfn], cg->fgfs[Rxy->sgfn], cg->fgfs[Rxz->sgfn],
cg->fgfs[Ryy->sgfn], cg->fgfs[Ryz->sgfn], cg->fgfs[Rzz->sgfn],
cg->fgfs[Cons_Ham->sgfn],
cg->fgfs[Cons_Px->sgfn], cg->fgfs[Cons_Py->sgfn], cg->fgfs[Cons_Pz->sgfn],
cg->fgfs[Cons_Gx->sgfn], cg->fgfs[Cons_Gy->sgfn], cg->fgfs[Cons_Gz->sgfn],
Symmetry, lev, ndeps, pre);
f_compute_constraint_fr(cg->shape, cg->X[0], cg->X[1], cg->X[2],
cg->fgfs[phi0->sgfn], cg->fgfs[trK0->sgfn],
cg->fgfs[rho->sgfn], cg->fgfs[Sphi0->sgfn],
cg->fgfs[gxx0->sgfn], cg->fgfs[gxy0->sgfn], cg->fgfs[gxz0->sgfn],
cg->fgfs[gyy0->sgfn], cg->fgfs[gyz0->sgfn], cg->fgfs[gzz0->sgfn],
cg->fgfs[Cons_Ham->sgfn],
cg->fgfs[Cons_Px->sgfn], cg->fgfs[Cons_Py->sgfn], cg->fgfs[Cons_Pz->sgfn],
cg->fgfs[Cons_Gx->sgfn], cg->fgfs[Cons_Gy->sgfn], cg->fgfs[Cons_Gz->sgfn],
Symmetry, lev, ndeps, pre);
if (!used_cuda_constraints)
f_compute_constraint_fr(cg->shape, cg->X[0], cg->X[1], cg->X[2],
cg->fgfs[phi0->sgfn], cg->fgfs[trK0->sgfn],
cg->fgfs[rho->sgfn], cg->fgfs[Sphi0->sgfn],
cg->fgfs[gxx0->sgfn], cg->fgfs[gxy0->sgfn], cg->fgfs[gxz0->sgfn],
cg->fgfs[gyy0->sgfn], cg->fgfs[gyz0->sgfn], cg->fgfs[gzz0->sgfn],
cg->fgfs[Axx0->sgfn], cg->fgfs[Axy0->sgfn], cg->fgfs[Axz0->sgfn],
cg->fgfs[Ayy0->sgfn], cg->fgfs[Ayz0->sgfn], cg->fgfs[Azz0->sgfn],
cg->fgfs[Rxx->sgfn], cg->fgfs[Rxy->sgfn], cg->fgfs[Rxz->sgfn],