64-BitBrainstorm_2026/AMSS-NCKU
8
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add keyed BSSN CUDA resident banks

Browse Source
This commit is contained in:
CGH0S7
2026-04-29 19:44:19 +08:00
parent 68eab03bac
commit 1ee229a91f
4 changed files with 910 additions and 89 deletions
Download Patch File Download Diff File Expand all files Collapse all files

194
AMSS_NCKU_source/Parallel.C
View File

@@ -172,6 +172,26 @@ int cuda_state_var_count(MyList<var> *src_vars, MyList<var> *dst_vars)
return (src_vars || dst_vars) ? -1 : count;
}
#if USE_CUDA_BSSN
bool cuda_build_bssn_host_views(Block *block,
MyList<var> *vars,
int state_count,
double **views)
{
if (!block || !vars || !views || state_count != BSSN_CUDA_STATE_COUNT)
return false;
MyList<var> *v = vars;
for (int i = 0; i < BSSN_CUDA_STATE_COUNT; ++i)
{
if (!v)
return false;
views[i] = block->fgfs[v->data->sgfn];
v = v->next;
}
return v == 0;
}
#endif
#if USE_CUDA_BSSN || USE_CUDA_Z4C
int fortran_idint(double x)
{
@@ -318,7 +338,8 @@ bool cuda_can_direct_unpack(const Parallel::gridseg *dst, int type)
bool cuda_direct_pack_segment(double *buffer,
const Parallel::gridseg *src,
const Parallel::gridseg *dst,
int state_count)
int state_count,
MyList<var> *VarLists)
{
#if USE_CUDA_Z4C && (ABEtype == 2)
if (state_count != Z4C_CUDA_STATE_COUNT)
@@ -338,9 +359,17 @@ bool cuda_direct_pack_segment(double *buffer,
i0, j0, k0,
dst->shape[0], dst->shape[1], dst->shape[2]) == 0;
#else
const bool ok = bssn_cuda_pack_state_batch_to_host_buffer(src->Bg, state_count, buffer, src->Bg->shape,
i0, j0, k0,
dst->shape[0], dst->shape[1], dst->shape[2]) == 0;
double *views[BSSN_CUDA_STATE_COUNT];
const bool have_views = cuda_build_bssn_host_views(src->Bg, VarLists, state_count, views);
const bool ok = have_views
? bssn_cuda_pack_state_batch_to_host_buffer_for_host_views(
src->Bg, views, state_count, buffer, src->Bg->shape,
i0, j0, k0,
dst->shape[0], dst->shape[1], dst->shape[2]) == 0
: bssn_cuda_pack_state_batch_to_host_buffer(
src->Bg, state_count, buffer, src->Bg->shape,
i0, j0, k0,
dst->shape[0], dst->shape[1], dst->shape[2]) == 0;
#endif
if (sync_profile_enabled())
sync_profile_stats().direct_pack_sec += MPI_Wtime() - t0;
@@ -349,7 +378,8 @@ bool cuda_direct_pack_segment(double *buffer,
bool cuda_direct_unpack_segment(double *buffer,
const Parallel::gridseg *dst,
int state_count)
int state_count,
MyList<var> *VarListd)
{
#if USE_CUDA_Z4C && (ABEtype == 2)
if (state_count != Z4C_CUDA_STATE_COUNT)
@@ -369,9 +399,17 @@ bool cuda_direct_unpack_segment(double *buffer,
i0, j0, k0,
dst->shape[0], dst->shape[1], dst->shape[2]) == 0;
#else
const bool ok = bssn_cuda_unpack_state_batch_from_host_buffer(dst->Bg, state_count, buffer, dst->Bg->shape,
i0, j0, k0,
dst->shape[0], dst->shape[1], dst->shape[2]) == 0;
double *views[BSSN_CUDA_STATE_COUNT];
const bool have_views = cuda_build_bssn_host_views(dst->Bg, VarListd, state_count, views);
const bool ok = have_views
? bssn_cuda_unpack_state_batch_from_host_buffer_for_host_views(
dst->Bg, views, state_count, buffer, dst->Bg->shape,
i0, j0, k0,
dst->shape[0], dst->shape[1], dst->shape[2]) == 0
: bssn_cuda_unpack_state_batch_from_host_buffer(
dst->Bg, state_count, buffer, dst->Bg->shape,
i0, j0, k0,
dst->shape[0], dst->shape[1], dst->shape[2]) == 0;
#endif
if (sync_profile_enabled())
sync_profile_stats().direct_unpack_sec += MPI_Wtime() - t0;
@@ -441,43 +479,62 @@ bool cuda_direct_pack_segment_to_device(double *buffer,
const Parallel::gridseg *src,
const Parallel::gridseg *dst,
int state_count,
int type)
int type,
MyList<var> *VarLists)
{
#if USE_CUDA_BSSN
if (state_count <= 0 || state_count > BSSN_CUDA_STATE_COUNT)
return false;
const double t0 = sync_profile_enabled() ? MPI_Wtime() : 0.0;
bool ok = false;
double *views[BSSN_CUDA_STATE_COUNT];
const bool have_views = cuda_build_bssn_host_views(src->Bg, VarLists, state_count, views);
if (type == 1)
{
const int i0 = cuda_seg_begin(dst, src->Bg, 0);
const int j0 = cuda_seg_begin(dst, src->Bg, 1);
const int k0 = cuda_seg_begin(dst, src->Bg, 2);
ok = bssn_cuda_pack_state_batch_to_device_buffer(
src->Bg, state_count, buffer, src->Bg->shape,
i0, j0, k0,
dst->shape[0], dst->shape[1], dst->shape[2]) == 0;
ok = have_views
? bssn_cuda_pack_state_batch_to_device_buffer_for_host_views(
src->Bg, views, state_count, buffer, src->Bg->shape,
i0, j0, k0,
dst->shape[0], dst->shape[1], dst->shape[2]) == 0
: bssn_cuda_pack_state_batch_to_device_buffer(
src->Bg, state_count, buffer, src->Bg->shape,
i0, j0, k0,
dst->shape[0], dst->shape[1], dst->shape[2]) == 0;
}
else if (type == 2)
{
int first_fine[3];
if (!cuda_cell_gw3_restrict_params(src, dst, first_fine))
return false;
ok = bssn_cuda_restrict_state_batch_to_device_buffer(
src->Bg, state_count, buffer, src->Bg->shape,
dst->shape[0], dst->shape[1], dst->shape[2],
first_fine[0], first_fine[1], first_fine[2]) == 0;
ok = have_views
? bssn_cuda_restrict_state_batch_to_device_buffer_for_host_views(
src->Bg, views, state_count, buffer, src->Bg->shape,
dst->shape[0], dst->shape[1], dst->shape[2],
first_fine[0], first_fine[1], first_fine[2]) == 0
: bssn_cuda_restrict_state_batch_to_device_buffer(
src->Bg, state_count, buffer, src->Bg->shape,
dst->shape[0], dst->shape[1], dst->shape[2],
first_fine[0], first_fine[1], first_fine[2]) == 0;
}
else if (type == 3)
{
int first_fine_ii[3], coarse_lb[3];
if (!cuda_cell_gw3_prolong_params(src, dst, first_fine_ii, coarse_lb))
return false;
ok = bssn_cuda_prolong_state_batch_to_device_buffer(
src->Bg, state_count, buffer, src->Bg->shape,
dst->shape[0], dst->shape[1], dst->shape[2],
first_fine_ii[0], first_fine_ii[1], first_fine_ii[2],
coarse_lb[0], coarse_lb[1], coarse_lb[2]) == 0;
ok = have_views
? bssn_cuda_prolong_state_batch_to_device_buffer_for_host_views(
src->Bg, views, state_count, buffer, src->Bg->shape,
dst->shape[0], dst->shape[1], dst->shape[2],
first_fine_ii[0], first_fine_ii[1], first_fine_ii[2],
coarse_lb[0], coarse_lb[1], coarse_lb[2]) == 0
: bssn_cuda_prolong_state_batch_to_device_buffer(
src->Bg, state_count, buffer, src->Bg->shape,
dst->shape[0], dst->shape[1], dst->shape[2],
first_fine_ii[0], first_fine_ii[1], first_fine_ii[2],
coarse_lb[0], coarse_lb[1], coarse_lb[2]) == 0;
}
if (sync_profile_enabled())
sync_profile_stats().direct_pack_sec += MPI_Wtime() - t0;
@@ -490,7 +547,8 @@ bool cuda_direct_pack_segment_to_device(double *buffer,
bool cuda_direct_unpack_segment_from_device(double *buffer,
const Parallel::gridseg *dst,
int state_count)
int state_count,
MyList<var> *VarListd)
{
#if USE_CUDA_BSSN
if (state_count <= 0 || state_count > BSSN_CUDA_STATE_COUNT)
@@ -499,10 +557,17 @@ bool cuda_direct_unpack_segment_from_device(double *buffer,
const int i0 = cuda_seg_begin(dst, dst->Bg, 0);
const int j0 = cuda_seg_begin(dst, dst->Bg, 1);
const int k0 = cuda_seg_begin(dst, dst->Bg, 2);
const bool ok = bssn_cuda_unpack_state_batch_from_device_buffer(
dst->Bg, state_count, buffer, dst->Bg->shape,
i0, j0, k0,
dst->shape[0], dst->shape[1], dst->shape[2]) == 0;
double *views[BSSN_CUDA_STATE_COUNT];
const bool have_views = cuda_build_bssn_host_views(dst->Bg, VarListd, state_count, views);
const bool ok = have_views
? bssn_cuda_unpack_state_batch_from_device_buffer_for_host_views(
dst->Bg, views, state_count, buffer, dst->Bg->shape,
i0, j0, k0,
dst->shape[0], dst->shape[1], dst->shape[2]) == 0
: bssn_cuda_unpack_state_batch_from_device_buffer(
dst->Bg, state_count, buffer, dst->Bg->shape,
i0, j0, k0,
dst->shape[0], dst->shape[1], dst->shape[2]) == 0;
if (sync_profile_enabled())
sync_profile_stats().direct_unpack_sec += MPI_Wtime() - t0;
return ok;
@@ -577,16 +642,25 @@ bool cuda_flush_device_segment_batch(Block *block,
double *data,
int state_count,
const std::vector<int> &meta,
int dir)
int dir,
MyList<var> *vars)
{
if (!block || meta.empty())
return true;
const int segment_count = (int)(meta.size() / 8);
double *views[BSSN_CUDA_STATE_COUNT];
const bool have_views = cuda_build_bssn_host_views(block, vars, state_count, views);
if (dir == PACK)
return bssn_cuda_pack_state_segments_to_device_buffer(
block, state_count, data, block->shape, segment_count, meta.data()) == 0;
return bssn_cuda_unpack_state_segments_from_device_buffer(
block, state_count, data, block->shape, segment_count, meta.data()) == 0;
return have_views
? bssn_cuda_pack_state_segments_to_device_buffer_for_host_views(
block, views, state_count, data, block->shape, segment_count, meta.data()) == 0
: bssn_cuda_pack_state_segments_to_device_buffer(
block, state_count, data, block->shape, segment_count, meta.data()) == 0;
return have_views
? bssn_cuda_unpack_state_segments_from_device_buffer_for_host_views(
block, views, state_count, data, block->shape, segment_count, meta.data()) == 0
: bssn_cuda_unpack_state_segments_from_device_buffer(
block, state_count, data, block->shape, segment_count, meta.data()) == 0;
}
int cuda_data_packer_device_batched(double *data,
@@ -638,7 +712,8 @@ int cuda_data_packer_device_batched(double *data,
if (batch_block && batch_block != block)
{
if (!cuda_flush_device_segment_batch(batch_block, data, state_count, batch_meta, dir))
MyList<var> *batch_vars = (dir == PACK) ? VarLists : VarListd;
if (!cuda_flush_device_segment_batch(batch_block, data, state_count, batch_meta, dir, batch_vars))
return -1;
batch_meta.clear();
}
@@ -672,7 +747,8 @@ int cuda_data_packer_device_batched(double *data,
if (batch_block)
{
if (!cuda_flush_device_segment_batch(batch_block, data, state_count, batch_meta, dir))
MyList<var> *batch_vars = (dir == PACK) ? VarLists : VarListd;
if (!cuda_flush_device_segment_batch(batch_block, data, state_count, batch_meta, dir, batch_vars))
return -1;
}
return size_out;
@@ -4542,9 +4618,9 @@ int Parallel::data_packer(double *data, MyList<Parallel::gridseg> *src, MyList<P
cuda_can_direct_pack(src->data, dst->data, type))
{
if (s_cuda_aware_pack_active) {
handled_by_cuda = cuda_direct_pack_segment_to_device(data + size_out, src->data, dst->data, state_count, type);
handled_by_cuda = cuda_direct_pack_segment_to_device(data + size_out, src->data, dst->data, state_count, type, VarLists);
} else {
handled_by_cuda = cuda_direct_pack_segment(data + size_out, src->data, dst->data, state_count);
handled_by_cuda = cuda_direct_pack_segment(data + size_out, src->data, dst->data, state_count, VarLists);
}
if (!handled_by_cuda)
{
@@ -4557,9 +4633,9 @@ int Parallel::data_packer(double *data, MyList<Parallel::gridseg> *src, MyList<P
cuda_can_direct_unpack(dst->data, type))
{
if (s_cuda_aware_pack_active) {
handled_by_cuda = cuda_direct_unpack_segment_from_device(data + size_out, dst->data, state_count);
handled_by_cuda = cuda_direct_unpack_segment_from_device(data + size_out, dst->data, state_count, VarListd);
} else {
handled_by_cuda = cuda_direct_unpack_segment(data + size_out, dst->data, state_count);
handled_by_cuda = cuda_direct_unpack_segment(data + size_out, dst->data, state_count, VarListd);
}
if (!handled_by_cuda)
{
@@ -6476,6 +6552,26 @@ void Parallel::prepare_inter_time_level(Patch *Pat,
Block *cg = BP->data;
if (myrank == cg->rank)
{
#if USE_CUDA_BSSN
double *src1_views[BSSN_CUDA_STATE_COUNT];
double *src2_views[BSSN_CUDA_STATE_COUNT];
double *dst_views[BSSN_CUDA_STATE_COUNT];
const int state_count = cuda_state_var_count(VarList1, VarList2);
if (state_count == BSSN_CUDA_STATE_COUNT &&
cuda_build_bssn_host_views(cg, VarList1, state_count, src1_views) &&
cuda_build_bssn_host_views(cg, VarList2, state_count, src2_views) &&
cuda_build_bssn_host_views(cg, VarList3, state_count, dst_views) &&
bssn_cuda_has_resident_state(cg) &&
bssn_cuda_prepare_inter_time_level(cg, cg->shape,
src1_views, src2_views, 0, dst_views,
2, tindex) == 0)
{
if (BP == Pat->ble)
break;
BP = BP->next;
continue;
}
#endif
varl1 = VarList1;
varl2 = VarList2;
varl3 = VarList3;
@@ -6531,6 +6627,28 @@ void Parallel::prepare_inter_time_level(Patch *Pat,
Block *cg = BP->data;
if (myrank == cg->rank)
{
#if USE_CUDA_BSSN
double *src1_views[BSSN_CUDA_STATE_COUNT];
double *src2_views[BSSN_CUDA_STATE_COUNT];
double *src3_views[BSSN_CUDA_STATE_COUNT];
double *dst_views[BSSN_CUDA_STATE_COUNT];
const int state_count = cuda_state_var_count(VarList1, VarList2);
if (state_count == BSSN_CUDA_STATE_COUNT &&
cuda_build_bssn_host_views(cg, VarList1, state_count, src1_views) &&
cuda_build_bssn_host_views(cg, VarList2, state_count, src2_views) &&
cuda_build_bssn_host_views(cg, VarList3, state_count, src3_views) &&
cuda_build_bssn_host_views(cg, VarList4, state_count, dst_views) &&
bssn_cuda_has_resident_state(cg) &&
bssn_cuda_prepare_inter_time_level(cg, cg->shape,
src1_views, src2_views, src3_views, dst_views,
3, tindex) == 0)
{
if (BP == Pat->ble)
break;
BP = BP->next;
continue;
}
#endif
varl1 = VarList1;
varl2 = VarList2;
varl3 = VarList3;