64-BitBrainstorm_2026/AMSS-NCKU
8
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Stabilize EScalar CUDA fallback path

Browse Source
This commit is contained in:
CGH0S7
2026-05-03 16:05:47 +08:00
parent 4430d04ee7
commit e4c10eca0f
5 changed files with 1542 additions and 127 deletions
Download Patch File Download Diff File Expand all files Collapse all files

489
AMSS_NCKU_source/Parallel.C
View File

@@ -173,13 +173,24 @@ int cuda_state_var_count(MyList<var> *src_vars, MyList<var> *dst_vars)
return (src_vars || dst_vars) ? -1 : count;
}
int cuda_var_list_count(MyList<var> *vars)
{
int count = 0;
while (vars)
{
++count;
vars = vars->next;
}
return count;
}
#if USE_CUDA_BSSN
bool cuda_build_bssn_host_views(Block *block,
MyList<var> *vars,
int state_count,
double **views)
{
if (!block || !vars || !views || state_count != BSSN_CUDA_STATE_COUNT)
if (!block || !block->fgfs || !vars || !views || state_count != BSSN_CUDA_STATE_COUNT)
return false;
MyList<var> *v = vars;
for (int i = 0; i < BSSN_CUDA_STATE_COUNT; ++i)
@@ -191,6 +202,37 @@ bool cuda_build_bssn_host_views(Block *block,
}
return v == 0;
}
bool cuda_build_escalar_host_views(Block *block,
MyList<var> *vars,
double **views)
{
if (!block || !block->fgfs || !vars || !views)
return false;
MyList<var> *v = vars;
for (int i = 0; i < 2; ++i)
{
if (!v)
return false;
views[i] = block->fgfs[v->data->sgfn];
if (!views[i])
return false;
v = v->next;
}
return v == 0;
}
bool cuda_escalar_list_looks_valid(MyList<var> *vars)
{
MyList<var> *v = vars;
for (int i = 0; i < 2; ++i)
{
if (!v || !v->data)
return false;
v = v->next;
}
return v == 0;
}
#endif
#if USE_CUDA_Z4C && (ABEtype == 2)
@@ -341,7 +383,7 @@ bool cuda_state_count_direct_supported(int state_count)
#if USE_CUDA_Z4C && (ABEtype == 2)
return state_count == Z4C_CUDA_STATE_COUNT;
#elif USE_CUDA_BSSN
return state_count == BSSN_CUDA_STATE_COUNT;
return state_count == BSSN_CUDA_STATE_COUNT || state_count == 2;
#else
(void)state_count;
return false;
@@ -391,9 +433,42 @@ bool cuda_can_direct_pack(const Parallel::gridseg *src, const Parallel::gridseg
}
return true;
#elif USE_CUDA_BSSN
if (VarLists)
{
int count = 0;
for (MyList<var> *v = VarLists; v; v = v->next) ++count;
if (count == 2)
{
if (type != 1)
{
int a[3], b[3];
if (type == 2)
{
if (!cuda_amr_restrict_device_enabled())
return false;
if (!cuda_cell_gw3_restrict_params(src, dst, a))
return false;
}
else if (type == 3)
{
if (!cuda_amr_prolong_device_enabled())
return false;
if (!cuda_cell_gw3_prolong_params(src, dst, a, b))
return false;
}
else
return false;
}
double *views[2];
if (cuda_build_escalar_host_views(src->Bg, VarLists, views))
return bssn_cuda_escalar_has_resident_fields(src->Bg, views[0], views[1]) != 0;
return cuda_escalar_list_looks_valid(VarLists) &&
bssn_cuda_escalar_has_any_resident_fields(src->Bg) != 0;
}
}
if (bssn_cuda_has_resident_state(src->Bg) == 0)
return false;
if (VarLists)
if (VarLists && src->Bg->fgfs)
{
double *view_ptrs[BSSN_CUDA_STATE_COUNT];
if (!cuda_build_bssn_host_views(src->Bg, VarLists, BSSN_CUDA_STATE_COUNT, view_ptrs))
@@ -435,6 +510,21 @@ bool cuda_can_direct_unpack(const Parallel::gridseg *dst, int type, MyList<var>
(void)VarListd;
return true;
#elif USE_CUDA_BSSN
if (VarListd)
{
int count = 0;
for (MyList<var> *v = VarListd; v; v = v->next) ++count;
if (count == 2)
{
double *views[2];
if (!cuda_build_escalar_host_views(dst->Bg, VarListd, views))
return cuda_escalar_list_looks_valid(VarListd) &&
(type == 1 || type == 2 || type == 3);
if (bssn_cuda_escalar_has_resident_fields(dst->Bg, views[0], views[1]) != 0)
return true;
return type == 1 || type == 2 || type == 3;
}
}
if (bssn_cuda_has_resident_state(dst->Bg) == 0)
return false;
if (VarListd)
@@ -461,6 +551,26 @@ bool cuda_direct_pack_segment(double *buffer,
if (state_count != Z4C_CUDA_STATE_COUNT)
return false;
#elif USE_CUDA_BSSN
if (state_count == 2)
{
const double t0 = sync_profile_enabled() ? MPI_Wtime() : 0.0;
double *views[2];
double **key = 0;
if (cuda_build_escalar_host_views(src->Bg, VarLists, views))
key = views;
else if (!cuda_escalar_list_looks_valid(VarLists))
return false;
const int i0 = cuda_seg_begin(dst, src->Bg, 0);
const int j0 = cuda_seg_begin(dst, src->Bg, 1);
const int k0 = cuda_seg_begin(dst, src->Bg, 2);
const bool ok = bssn_cuda_pack_escalar_batch_to_host_buffer(
src->Bg, key, buffer, src->Bg->shape,
i0, j0, k0,
dst->shape[0], dst->shape[1], dst->shape[2]) == 0;
if (sync_profile_enabled())
sync_profile_stats().direct_pack_sec += MPI_Wtime() - t0;
return ok;
}
if (state_count != BSSN_CUDA_STATE_COUNT)
return false;
#else
@@ -508,6 +618,26 @@ bool cuda_direct_unpack_segment(double *buffer,
if (state_count != Z4C_CUDA_STATE_COUNT)
return false;
#elif USE_CUDA_BSSN
if (state_count == 2)
{
const double t0 = sync_profile_enabled() ? MPI_Wtime() : 0.0;
double *views[2];
double **key = 0;
if (cuda_build_escalar_host_views(dst->Bg, VarListd, views))
key = views;
else if (!cuda_escalar_list_looks_valid(VarListd))
return false;
const int i0 = cuda_seg_begin(dst, dst->Bg, 0);
const int j0 = cuda_seg_begin(dst, dst->Bg, 1);
const int k0 = cuda_seg_begin(dst, dst->Bg, 2);
const bool ok = bssn_cuda_unpack_escalar_batch_from_host_buffer(
dst->Bg, key, buffer, dst->Bg->shape,
i0, j0, k0,
dst->shape[0], dst->shape[1], dst->shape[2]) == 0;
if (sync_profile_enabled())
sync_profile_stats().direct_unpack_sec += MPI_Wtime() - t0;
return ok;
}
if (state_count != BSSN_CUDA_STATE_COUNT)
return false;
#else
@@ -560,8 +690,10 @@ bool cuda_direct_pack_bssn_prefix_to_host(double *buffer,
return false;
double *views[BSSN_CUDA_STATE_COUNT];
double soa_flat[3 * BSSN_CUDA_STATE_COUNT];
if (!cuda_build_bssn_host_views(src->Bg, VarLists, BSSN_CUDA_STATE_COUNT, views) ||
!cuda_build_state_soa(VarLists, BSSN_CUDA_STATE_COUNT, soa_flat))
const bool have_views =
src->Bg->fgfs &&
cuda_build_bssn_host_views(src->Bg, VarLists, BSSN_CUDA_STATE_COUNT, views);
if (!cuda_build_state_soa(VarLists, BSSN_CUDA_STATE_COUNT, soa_flat))
return false;
const double t0 = sync_profile_enabled() ? MPI_Wtime() : 0.0;
bool ok = false;
@@ -570,33 +702,51 @@ bool cuda_direct_pack_bssn_prefix_to_host(double *buffer,
const int i0 = cuda_seg_begin(dst, src->Bg, 0);
const int j0 = cuda_seg_begin(dst, src->Bg, 1);
const int k0 = cuda_seg_begin(dst, src->Bg, 2);
ok = bssn_cuda_pack_state_batch_to_host_buffer_for_host_views(
src->Bg, views, BSSN_CUDA_STATE_COUNT, buffer, src->Bg->shape,
i0, j0, k0,
dst->shape[0], dst->shape[1], dst->shape[2]) == 0;
ok = have_views
? bssn_cuda_pack_state_batch_to_host_buffer_for_host_views(
src->Bg, views, BSSN_CUDA_STATE_COUNT, buffer, src->Bg->shape,
i0, j0, k0,
dst->shape[0], dst->shape[1], dst->shape[2]) == 0
: bssn_cuda_pack_state_batch_to_host_buffer(
src->Bg, BSSN_CUDA_STATE_COUNT, buffer, src->Bg->shape,
i0, j0, k0,
dst->shape[0], dst->shape[1], dst->shape[2]) == 0;
}
else if (type == 2)
{
int first_fine[3];
if (!cuda_cell_gw3_restrict_params(src, dst, first_fine))
return false;
ok = bssn_cuda_restrict_state_batch_to_host_buffer_for_host_views(
src->Bg, views, BSSN_CUDA_STATE_COUNT, buffer, src->Bg->shape,
dst->shape[0], dst->shape[1], dst->shape[2],
first_fine[0], first_fine[1], first_fine[2],
soa_flat) == 0;
ok = have_views
? bssn_cuda_restrict_state_batch_to_host_buffer_for_host_views(
src->Bg, views, BSSN_CUDA_STATE_COUNT, buffer, src->Bg->shape,
dst->shape[0], dst->shape[1], dst->shape[2],
first_fine[0], first_fine[1], first_fine[2],
soa_flat) == 0
: bssn_cuda_restrict_state_batch_to_host_buffer(
src->Bg, BSSN_CUDA_STATE_COUNT, buffer, src->Bg->shape,
dst->shape[0], dst->shape[1], dst->shape[2],
first_fine[0], first_fine[1], first_fine[2],
soa_flat) == 0;
}
else if (type == 3)
{
int first_fine_ii[3], coarse_lb[3];
if (!cuda_cell_gw3_prolong_params(src, dst, first_fine_ii, coarse_lb))
return false;
ok = bssn_cuda_prolong_state_batch_to_host_buffer_for_host_views(
src->Bg, views, BSSN_CUDA_STATE_COUNT, buffer, src->Bg->shape,
dst->shape[0], dst->shape[1], dst->shape[2],
first_fine_ii[0], first_fine_ii[1], first_fine_ii[2],
coarse_lb[0], coarse_lb[1], coarse_lb[2],
soa_flat) == 0;
ok = have_views
? bssn_cuda_prolong_state_batch_to_host_buffer_for_host_views(
src->Bg, views, BSSN_CUDA_STATE_COUNT, buffer, src->Bg->shape,
dst->shape[0], dst->shape[1], dst->shape[2],
first_fine_ii[0], first_fine_ii[1], first_fine_ii[2],
coarse_lb[0], coarse_lb[1], coarse_lb[2],
soa_flat) == 0
: bssn_cuda_prolong_state_batch_to_host_buffer(
src->Bg, BSSN_CUDA_STATE_COUNT, buffer, src->Bg->shape,
dst->shape[0], dst->shape[1], dst->shape[2],
first_fine_ii[0], first_fine_ii[1], first_fine_ii[2],
coarse_lb[0], coarse_lb[1], coarse_lb[2],
soa_flat) == 0;
}
if (sync_profile_enabled())
sync_profile_stats().direct_pack_sec += MPI_Wtime() - t0;
@@ -881,6 +1031,53 @@ bool cuda_direct_pack_segment_to_device(double *buffer,
}
#endif
#if USE_CUDA_BSSN
if (state_count == 2)
{
const double t0 = sync_profile_enabled() ? MPI_Wtime() : 0.0;
bool ok = false;
double *views[2];
double soa_flat[6];
const bool have_views = cuda_build_escalar_host_views(src->Bg, VarLists, views);
const bool have_soa = cuda_build_state_soa(VarLists, state_count, soa_flat);
if (!have_views)
return false;
if (type == 1)
{
const int i0 = cuda_seg_begin(dst, src->Bg, 0);
const int j0 = cuda_seg_begin(dst, src->Bg, 1);
const int k0 = cuda_seg_begin(dst, src->Bg, 2);
ok = bssn_cuda_pack_escalar_batch_to_device_buffer(
src->Bg, views, buffer, src->Bg->shape,
i0, j0, k0,
dst->shape[0], dst->shape[1], dst->shape[2]) == 0;
}
else if (type == 2)
{
int first_fine[3];
if (!cuda_cell_gw3_restrict_params(src, dst, first_fine))
return false;
ok = bssn_cuda_restrict_escalar_batch_to_device_buffer(
src->Bg, views, buffer, src->Bg->shape,
dst->shape[0], dst->shape[1], dst->shape[2],
first_fine[0], first_fine[1], first_fine[2],
have_soa ? soa_flat : 0) == 0;
}
else if (type == 3)
{
int first_fine_ii[3], coarse_lb[3];
if (!cuda_cell_gw3_prolong_params(src, dst, first_fine_ii, coarse_lb))
return false;
ok = bssn_cuda_prolong_escalar_batch_to_device_buffer(
src->Bg, views, buffer, src->Bg->shape,
dst->shape[0], dst->shape[1], dst->shape[2],
first_fine_ii[0], first_fine_ii[1], first_fine_ii[2],
coarse_lb[0], coarse_lb[1], coarse_lb[2],
have_soa ? soa_flat : 0) == 0;
}
if (sync_profile_enabled())
sync_profile_stats().direct_pack_sec += MPI_Wtime() - t0;
return ok;
}
if (state_count != BSSN_CUDA_STATE_COUNT)
return false;
const double t0 = sync_profile_enabled() ? MPI_Wtime() : 0.0;
@@ -1073,6 +1270,23 @@ bool cuda_direct_unpack_segment_from_device(double *buffer,
}
#endif
#if USE_CUDA_BSSN
if (state_count == 2)
{
const double t0 = sync_profile_enabled() ? MPI_Wtime() : 0.0;
const int i0 = cuda_seg_begin(dst, dst->Bg, 0);
const int j0 = cuda_seg_begin(dst, dst->Bg, 1);
const int k0 = cuda_seg_begin(dst, dst->Bg, 2);
double *views[2];
if (!cuda_build_escalar_host_views(dst->Bg, VarListd, views))
return false;
const bool ok = bssn_cuda_unpack_escalar_batch_from_device_buffer(
dst->Bg, views, buffer, dst->Bg->shape,
i0, j0, k0,
dst->shape[0], dst->shape[1], dst->shape[2]) == 0;
if (sync_profile_enabled())
sync_profile_stats().direct_unpack_sec += MPI_Wtime() - t0;
return ok;
}
if (state_count != BSSN_CUDA_STATE_COUNT)
return false;
const double t0 = sync_profile_enabled() ? MPI_Wtime() : 0.0;
@@ -1127,8 +1341,15 @@ bool cuda_download_resident_subset_to_host(Block *block,
}
#endif
#if USE_CUDA_BSSN
if (!block || state_count != BSSN_CUDA_STATE_COUNT)
if (!block || !block->fgfs || state_count != BSSN_CUDA_STATE_COUNT)
{
if (getenv("AMSS_CUDA_FALLBACK_DIAG"))
fprintf(stderr,
"[AMSS-CUDA-FALLBACK] invalid subset request block=%p fgfs=%p state_count=%d expected=%d\n",
(void *)block, block ? (void *)block->fgfs : 0,
state_count, BSSN_CUDA_STATE_COUNT);
return false;
}
if (bssn_cuda_has_resident_state(block) == 0)
return true;
int indices[BSSN_CUDA_STATE_COUNT];
@@ -1137,14 +1358,24 @@ bool cuda_download_resident_subset_to_host(Block *block,
for (int i = 0; i < state_count; ++i)
{
if (!v)
return false;
return true;
indices[i] = i;
views[i] = block->fgfs[v->data->sgfn];
if (!views[i])
return true;
v = v->next;
}
if (bssn_cuda_resident_state_matches(block, views) == 0)
return false;
return bssn_cuda_download_state_subset(block, block->shape, state_count, indices, views) == 0;
return true;
const int rc = bssn_cuda_download_state_subset(block, block->shape, state_count, indices, views);
if (rc != 0 && getenv("AMSS_CUDA_FALLBACK_DIAG"))
fprintf(stderr,
"[AMSS-CUDA-FALLBACK] subset download rc=%d block=%p lev=%d shape=[%d,%d,%d] first_var=%s sgfn=%d\n",
rc, (void *)block, block->lev,
block->shape[0], block->shape[1], block->shape[2],
(vars && vars->data) ? vars->data->name : "(null)",
(vars && vars->data) ? vars->data->sgfn : -1);
return rc == 0;
#else
(void)block; (void)vars; (void)state_count;
return false;
@@ -1197,7 +1428,8 @@ bool cuda_device_state_count_supported(int state_count)
return true;
#endif
#if USE_CUDA_BSSN
return state_count == BSSN_CUDA_STATE_COUNT;
return state_count == BSSN_CUDA_STATE_COUNT || state_count == 2 ||
state_count == BSSN_CUDA_STATE_COUNT + 2;
#else
(void)state_count;
return false;
@@ -1316,6 +1548,12 @@ int cuda_data_packer_device_batched(double *data,
const int state_count = cuda_state_var_count(VarLists, VarListd);
if (!cuda_device_state_count_supported(state_count))
return -1;
#if USE_CUDA_BSSN
if (state_count == 2)
return -1;
if (state_count > BSSN_CUDA_STATE_COUNT)
return -1;
#endif
int size_out = 0;
Block *batch_block = 0;
int batch_type = 0;
@@ -1620,6 +1858,109 @@ int data_packer_with_device_buffer(double *data,
}
#endif
#if USE_CUDA_BSSN || USE_CUDA_Z4C
std::vector<double> &cuda_host_stage_buffer()
{
static thread_local std::vector<double> buffer;
return buffer;
}
bool cuda_pack_one_host_field_to_device(double *device_buffer,
Parallel::gridseg *src,
Parallel::gridseg *dst,
int type,
var *src_var,
int Symmetry)
{
if (!device_buffer || !src || !dst || !src->Bg || !dst->Bg ||
!src->Bg->fgfs || !src_var || !src->Bg->fgfs[src_var->sgfn])
return false;
const int region_all = dst->shape[0] * dst->shape[1] * dst->shape[2];
if (region_all <= 0)
return false;
std::vector<double> &stage = cuda_host_stage_buffer();
stage.resize((size_t)region_all);
int DIM = dim;
switch (type)
{
case 1:
f_copy(DIM, dst->llb, dst->uub, dst->shape, stage.data(),
src->Bg->bbox, src->Bg->bbox + dim, src->Bg->shape,
src->Bg->fgfs[src_var->sgfn],
dst->llb, dst->uub);
break;
case 2:
f_restrict3(DIM, dst->llb, dst->uub, dst->shape, stage.data(),
src->Bg->bbox, src->Bg->bbox + dim, src->Bg->shape,
src->Bg->fgfs[src_var->sgfn],
dst->llb, dst->uub, src_var->SoA, Symmetry);
break;
case 3:
f_prolong3(DIM, src->Bg->bbox, src->Bg->bbox + dim, src->Bg->shape,
src->Bg->fgfs[src_var->sgfn],
dst->llb, dst->uub, dst->shape, stage.data(),
dst->llb, dst->uub, src_var->SoA, Symmetry);
break;
default:
return false;
}
cudaError_t err = cudaMemcpy(device_buffer, stage.data(),
(size_t)region_all * sizeof(double),
cudaMemcpyHostToDevice);
if (err != cudaSuccess)
{
fprintf(stderr, "Parallel: host tail cudaMemcpy H2D failed, err=%d\n", (int)err);
return false;
}
return true;
}
bool cuda_unpack_one_device_field_to_host(double *device_buffer,
Parallel::gridseg *dst,
var *dst_var)
{
if (!device_buffer || !dst || !dst->Bg || !dst->Bg->fgfs ||
!dst_var || !dst->Bg->fgfs[dst_var->sgfn])
return false;
const int region_all = dst->shape[0] * dst->shape[1] * dst->shape[2];
if (region_all <= 0)
return false;
std::vector<double> &stage = cuda_host_stage_buffer();
stage.resize((size_t)region_all);
cudaError_t err = cudaMemcpy(stage.data(), device_buffer,
(size_t)region_all * sizeof(double),
cudaMemcpyDeviceToHost);
if (err != cudaSuccess)
{
fprintf(stderr, "Parallel: host tail cudaMemcpy D2H failed, err=%d\n", (int)err);
return false;
}
int DIM = dim;
f_copy(DIM,
dst->Bg->bbox, dst->Bg->bbox + dim, dst->Bg->shape,
dst->Bg->fgfs[dst_var->sgfn],
dst->llb, dst->uub, dst->shape, stage.data(),
dst->llb, dst->uub);
return true;
}
#if USE_CUDA_BSSN
void cuda_download_escalar_tail_if_present(Block *block, MyList<var> *tail)
{
if (!block || !block->fgfs || !tail || !tail->next)
return;
bssn_cuda_escalar_download_fields_if_present(
block, block->shape,
block->fgfs[tail->data->sgfn],
block->fgfs[tail->next->data->sgfn]);
}
#endif
#endif
} // namespace
int Parallel::partition1(int &nx, int split_size, int min_width, int cpusize, int shape) // special for 1 diemnsion
@@ -5377,6 +5718,15 @@ int Parallel::data_packer(double *data, MyList<Parallel::gridseg> *src, MyList<P
if (data)
{
#if USE_CUDA_BSSN || USE_CUDA_Z4C
#if USE_CUDA_BSSN
if (s_cuda_aware_pack_active &&
state_count == BSSN_CUDA_STATE_COUNT + 2 &&
state_idx == BSSN_CUDA_STATE_COUNT &&
dir == PACK && src->data && src->data->Bg)
{
cuda_download_escalar_tail_if_present(src->data->Bg, varls);
}
#endif
bool handled_by_cuda = false;
int cuda_handled_count = state_count;
if (dir == PACK && (type == 1 || s_cuda_aware_pack_active) &&
@@ -5409,10 +5759,32 @@ int Parallel::data_packer(double *data, MyList<Parallel::gridseg> *src, MyList<P
MPI_Abort(MPI_COMM_WORLD, 1);
}
}
#if USE_CUDA_BSSN
else if (s_cuda_aware_pack_active &&
state_idx == 0 &&
state_count == BSSN_CUDA_STATE_COUNT + 2 &&
dir == PACK &&
cuda_direct_pack_segment_to_device(data + size_out, src->data, dst->data,
BSSN_CUDA_STATE_COUNT, type, VarLists, Symmetry))
{
handled_by_cuda = true;
cuda_handled_count = BSSN_CUDA_STATE_COUNT;
}
else if (s_cuda_aware_pack_active &&
state_idx == 0 &&
state_count == BSSN_CUDA_STATE_COUNT + 2 &&
dir == UNPACK &&
cuda_direct_unpack_segment_from_device(data + size_out, dst->data,
BSSN_CUDA_STATE_COUNT, VarListd))
{
handled_by_cuda = true;
cuda_handled_count = BSSN_CUDA_STATE_COUNT;
}
#endif
#if USE_CUDA_BSSN
else if (!s_cuda_aware_pack_active &&
state_idx == 0 &&
state_count > BSSN_CUDA_STATE_COUNT &&
state_count >= BSSN_CUDA_STATE_COUNT &&
dir == PACK &&
cuda_direct_pack_bssn_prefix_to_host(data + size_out, src->data, dst->data,
type, VarLists, Symmetry))
@@ -5422,7 +5794,7 @@ int Parallel::data_packer(double *data, MyList<Parallel::gridseg> *src, MyList<P
}
else if (!s_cuda_aware_pack_active &&
state_idx == 0 &&
state_count > BSSN_CUDA_STATE_COUNT &&
state_count >= BSSN_CUDA_STATE_COUNT &&
dir == UNPACK &&
cuda_direct_unpack_bssn_prefix_from_host(data + size_out, dst->data,
type, VarListd))
@@ -5430,6 +5802,27 @@ int Parallel::data_packer(double *data, MyList<Parallel::gridseg> *src, MyList<P
handled_by_cuda = true;
cuda_handled_count = BSSN_CUDA_STATE_COUNT;
}
#endif
#if USE_CUDA_BSSN || USE_CUDA_Z4C
else if (s_cuda_aware_pack_active &&
state_count == BSSN_CUDA_STATE_COUNT + 2 &&
state_idx >= BSSN_CUDA_STATE_COUNT &&
dir == PACK &&
cuda_pack_one_host_field_to_device(data + size_out, src->data, dst->data,
type, varls->data, Symmetry))
{
handled_by_cuda = true;
cuda_handled_count = 1;
}
else if (s_cuda_aware_pack_active &&
state_count == BSSN_CUDA_STATE_COUNT + 2 &&
state_idx >= BSSN_CUDA_STATE_COUNT &&
dir == UNPACK &&
cuda_unpack_one_device_field_to_host(data + size_out, dst->data, varld->data))
{
handled_by_cuda = true;
cuda_handled_count = 1;
}
#endif
if (!handled_by_cuda)
{
@@ -5443,7 +5836,8 @@ int Parallel::data_packer(double *data, MyList<Parallel::gridseg> *src, MyList<P
#endif
#endif
#if USE_CUDA_BSSN || USE_CUDA_Z4C
if (dir == PACK && state_idx == 0 && cuda_state_count_direct_supported(state_count) &&
if (dir == PACK && state_idx == 0 &&
state_count == BSSN_CUDA_STATE_COUNT &&
src->data && src->data->Bg)
{
if (!cuda_download_resident_subset_to_host(src->data->Bg, VarLists, state_count))
@@ -5496,7 +5890,7 @@ int Parallel::data_packer(double *data, MyList<Parallel::gridseg> *src, MyList<P
dst->data->llb, dst->data->uub, dst->data->shape, data + size_out,
dst->data->llb, dst->data->uub);
#if USE_CUDA_BSSN || USE_CUDA_Z4C
if (cuda_state_count_direct_supported(state_count) &&
if (state_count == BSSN_CUDA_STATE_COUNT &&
dst->data && dst->data->Bg)
{
#if USE_CUDA_Z4C && (ABEtype == 2)
@@ -7395,11 +7789,27 @@ void Parallel::prepare_inter_time_level(Patch *Pat,
if (myrank == cg->rank)
{
#if USE_CUDA_BSSN
const int state_count = cuda_state_var_count(VarList1, VarList2);
if (state_count == 2)
{
double *src1_scalar[2], *src2_scalar[2], *dst_scalar[2];
if (cuda_build_escalar_host_views(cg, VarList1, src1_scalar) &&
cuda_build_escalar_host_views(cg, VarList2, src2_scalar) &&
cuda_build_escalar_host_views(cg, VarList3, dst_scalar) &&
bssn_cuda_prepare_escalar_inter_time_level(cg, cg->shape,
src1_scalar, src2_scalar, 0, dst_scalar,
2, tindex) == 0)
{
if (BP == Pat->ble)
break;
BP = BP->next;
continue;
}
}
bool bssn_prefix_done = false;
double *src1_views[BSSN_CUDA_STATE_COUNT];
double *src2_views[BSSN_CUDA_STATE_COUNT];
double *dst_views[BSSN_CUDA_STATE_COUNT];
const int state_count = cuda_state_var_count(VarList1, VarList2);
if (state_count >= BSSN_CUDA_STATE_COUNT &&
cuda_build_bssn_host_views(cg, VarList1, BSSN_CUDA_STATE_COUNT, src1_views) &&
cuda_build_bssn_host_views(cg, VarList2, BSSN_CUDA_STATE_COUNT, src2_views) &&
@@ -7488,12 +7898,29 @@ void Parallel::prepare_inter_time_level(Patch *Pat,
if (myrank == cg->rank)
{
#if USE_CUDA_BSSN
const int state_count = cuda_state_var_count(VarList1, VarList2);
if (state_count == 2)
{
double *src1_scalar[2], *src2_scalar[2], *src3_scalar[2], *dst_scalar[2];
if (cuda_build_escalar_host_views(cg, VarList1, src1_scalar) &&
cuda_build_escalar_host_views(cg, VarList2, src2_scalar) &&
cuda_build_escalar_host_views(cg, VarList3, src3_scalar) &&
cuda_build_escalar_host_views(cg, VarList4, dst_scalar) &&
bssn_cuda_prepare_escalar_inter_time_level(cg, cg->shape,
src1_scalar, src2_scalar, src3_scalar, dst_scalar,
3, tindex) == 0)
{
if (BP == Pat->ble)
break;
BP = BP->next;
continue;
}
}
bool bssn_prefix_done = false;
double *src1_views[BSSN_CUDA_STATE_COUNT];
double *src2_views[BSSN_CUDA_STATE_COUNT];
double *src3_views[BSSN_CUDA_STATE_COUNT];
double *dst_views[BSSN_CUDA_STATE_COUNT];
const int state_count = cuda_state_var_count(VarList1, VarList2);
if (state_count >= BSSN_CUDA_STATE_COUNT &&
cuda_build_bssn_host_views(cg, VarList1, BSSN_CUDA_STATE_COUNT, src1_views) &&
cuda_build_bssn_host_views(cg, VarList2, BSSN_CUDA_STATE_COUNT, src2_views) &&