64-BitBrainstorm_2026/AMSS-NCKU
8
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add keyed BSSN CUDA resident banks

Browse Source
This commit is contained in:
CGH0S7
2026-04-29 19:44:19 +08:00
parent 68eab03bac
commit 1ee229a91f
4 changed files with 910 additions and 89 deletions
Download Patch File Download Diff File Expand all files Collapse all files

194
AMSS_NCKU_source/Parallel.C
View File

@@ -172,6 +172,26 @@ int cuda_state_var_count(MyList<var> *src_vars, MyList<var> *dst_vars)
return (src_vars || dst_vars) ? -1 : count;
}
#if USE_CUDA_BSSN
bool cuda_build_bssn_host_views(Block *block,
MyList<var> *vars,
int state_count,
double **views)
{
if (!block || !vars || !views || state_count != BSSN_CUDA_STATE_COUNT)
return false;
MyList<var> *v = vars;
for (int i = 0; i < BSSN_CUDA_STATE_COUNT; ++i)
{
if (!v)
return false;
views[i] = block->fgfs[v->data->sgfn];
v = v->next;
}
return v == 0;
}
#endif
#if USE_CUDA_BSSN || USE_CUDA_Z4C
int fortran_idint(double x)
{
@@ -318,7 +338,8 @@ bool cuda_can_direct_unpack(const Parallel::gridseg *dst, int type)
bool cuda_direct_pack_segment(double *buffer,
const Parallel::gridseg *src,
const Parallel::gridseg *dst,
int state_count)
int state_count,
MyList<var> *VarLists)
{
#if USE_CUDA_Z4C && (ABEtype == 2)
if (state_count != Z4C_CUDA_STATE_COUNT)
@@ -338,9 +359,17 @@ bool cuda_direct_pack_segment(double *buffer,
i0, j0, k0,
dst->shape[0], dst->shape[1], dst->shape[2]) == 0;
#else
const bool ok = bssn_cuda_pack_state_batch_to_host_buffer(src->Bg, state_count, buffer, src->Bg->shape,
i0, j0, k0,
dst->shape[0], dst->shape[1], dst->shape[2]) == 0;
double *views[BSSN_CUDA_STATE_COUNT];
const bool have_views = cuda_build_bssn_host_views(src->Bg, VarLists, state_count, views);
const bool ok = have_views
? bssn_cuda_pack_state_batch_to_host_buffer_for_host_views(
src->Bg, views, state_count, buffer, src->Bg->shape,
i0, j0, k0,
dst->shape[0], dst->shape[1], dst->shape[2]) == 0
: bssn_cuda_pack_state_batch_to_host_buffer(
src->Bg, state_count, buffer, src->Bg->shape,
i0, j0, k0,
dst->shape[0], dst->shape[1], dst->shape[2]) == 0;
#endif
if (sync_profile_enabled())
sync_profile_stats().direct_pack_sec += MPI_Wtime() - t0;
@@ -349,7 +378,8 @@ bool cuda_direct_pack_segment(double *buffer,
bool cuda_direct_unpack_segment(double *buffer,
const Parallel::gridseg *dst,
int state_count)
int state_count,
MyList<var> *VarListd)
{
#if USE_CUDA_Z4C && (ABEtype == 2)
if (state_count != Z4C_CUDA_STATE_COUNT)
@@ -369,9 +399,17 @@ bool cuda_direct_unpack_segment(double *buffer,
i0, j0, k0,
dst->shape[0], dst->shape[1], dst->shape[2]) == 0;
#else
const bool ok = bssn_cuda_unpack_state_batch_from_host_buffer(dst->Bg, state_count, buffer, dst->Bg->shape,
i0, j0, k0,
dst->shape[0], dst->shape[1], dst->shape[2]) == 0;
double *views[BSSN_CUDA_STATE_COUNT];
const bool have_views = cuda_build_bssn_host_views(dst->Bg, VarListd, state_count, views);
const bool ok = have_views
? bssn_cuda_unpack_state_batch_from_host_buffer_for_host_views(
dst->Bg, views, state_count, buffer, dst->Bg->shape,
i0, j0, k0,
dst->shape[0], dst->shape[1], dst->shape[2]) == 0
: bssn_cuda_unpack_state_batch_from_host_buffer(
dst->Bg, state_count, buffer, dst->Bg->shape,
i0, j0, k0,
dst->shape[0], dst->shape[1], dst->shape[2]) == 0;
#endif
if (sync_profile_enabled())
sync_profile_stats().direct_unpack_sec += MPI_Wtime() - t0;
@@ -441,43 +479,62 @@ bool cuda_direct_pack_segment_to_device(double *buffer,
const Parallel::gridseg *src,
const Parallel::gridseg *dst,
int state_count,
int type)
int type,
MyList<var> *VarLists)
{
#if USE_CUDA_BSSN
if (state_count <= 0 || state_count > BSSN_CUDA_STATE_COUNT)
return false;
const double t0 = sync_profile_enabled() ? MPI_Wtime() : 0.0;
bool ok = false;
double *views[BSSN_CUDA_STATE_COUNT];
const bool have_views = cuda_build_bssn_host_views(src->Bg, VarLists, state_count, views);
if (type == 1)
{
const int i0 = cuda_seg_begin(dst, src->Bg, 0);
const int j0 = cuda_seg_begin(dst, src->Bg, 1);
const int k0 = cuda_seg_begin(dst, src->Bg, 2);
ok = bssn_cuda_pack_state_batch_to_device_buffer(
src->Bg, state_count, buffer, src->Bg->shape,
i0, j0, k0,
dst->shape[0], dst->shape[1], dst->shape[2]) == 0;
ok = have_views
? bssn_cuda_pack_state_batch_to_device_buffer_for_host_views(
src->Bg, views, state_count, buffer, src->Bg->shape,
i0, j0, k0,
dst->shape[0], dst->shape[1], dst->shape[2]) == 0
: bssn_cuda_pack_state_batch_to_device_buffer(
src->Bg, state_count, buffer, src->Bg->shape,
i0, j0, k0,
dst->shape[0], dst->shape[1], dst->shape[2]) == 0;
}
else if (type == 2)
{
int first_fine[3];
if (!cuda_cell_gw3_restrict_params(src, dst, first_fine))
return false;
ok = bssn_cuda_restrict_state_batch_to_device_buffer(
src->Bg, state_count, buffer, src->Bg->shape,
dst->shape[0], dst->shape[1], dst->shape[2],
first_fine[0], first_fine[1], first_fine[2]) == 0;
ok = have_views
? bssn_cuda_restrict_state_batch_to_device_buffer_for_host_views(
src->Bg, views, state_count, buffer, src->Bg->shape,
dst->shape[0], dst->shape[1], dst->shape[2],
first_fine[0], first_fine[1], first_fine[2]) == 0
: bssn_cuda_restrict_state_batch_to_device_buffer(
src->Bg, state_count, buffer, src->Bg->shape,
dst->shape[0], dst->shape[1], dst->shape[2],
first_fine[0], first_fine[1], first_fine[2]) == 0;
}
else if (type == 3)
{
int first_fine_ii[3], coarse_lb[3];
if (!cuda_cell_gw3_prolong_params(src, dst, first_fine_ii, coarse_lb))
return false;
ok = bssn_cuda_prolong_state_batch_to_device_buffer(
src->Bg, state_count, buffer, src->Bg->shape,
dst->shape[0], dst->shape[1], dst->shape[2],
first_fine_ii[0], first_fine_ii[1], first_fine_ii[2],
coarse_lb[0], coarse_lb[1], coarse_lb[2]) == 0;
ok = have_views
? bssn_cuda_prolong_state_batch_to_device_buffer_for_host_views(
src->Bg, views, state_count, buffer, src->Bg->shape,
dst->shape[0], dst->shape[1], dst->shape[2],
first_fine_ii[0], first_fine_ii[1], first_fine_ii[2],
coarse_lb[0], coarse_lb[1], coarse_lb[2]) == 0
: bssn_cuda_prolong_state_batch_to_device_buffer(
src->Bg, state_count, buffer, src->Bg->shape,
dst->shape[0], dst->shape[1], dst->shape[2],
first_fine_ii[0], first_fine_ii[1], first_fine_ii[2],
coarse_lb[0], coarse_lb[1], coarse_lb[2]) == 0;
}
if (sync_profile_enabled())
sync_profile_stats().direct_pack_sec += MPI_Wtime() - t0;
@@ -490,7 +547,8 @@ bool cuda_direct_pack_segment_to_device(double *buffer,
bool cuda_direct_unpack_segment_from_device(double *buffer,
const Parallel::gridseg *dst,
int state_count)
int state_count,
MyList<var> *VarListd)
{
#if USE_CUDA_BSSN
if (state_count <= 0 || state_count > BSSN_CUDA_STATE_COUNT)
@@ -499,10 +557,17 @@ bool cuda_direct_unpack_segment_from_device(double *buffer,
const int i0 = cuda_seg_begin(dst, dst->Bg, 0);
const int j0 = cuda_seg_begin(dst, dst->Bg, 1);
const int k0 = cuda_seg_begin(dst, dst->Bg, 2);
const bool ok = bssn_cuda_unpack_state_batch_from_device_buffer(
dst->Bg, state_count, buffer, dst->Bg->shape,
i0, j0, k0,
dst->shape[0], dst->shape[1], dst->shape[2]) == 0;
double *views[BSSN_CUDA_STATE_COUNT];
const bool have_views = cuda_build_bssn_host_views(dst->Bg, VarListd, state_count, views);
const bool ok = have_views
? bssn_cuda_unpack_state_batch_from_device_buffer_for_host_views(
dst->Bg, views, state_count, buffer, dst->Bg->shape,
i0, j0, k0,
dst->shape[0], dst->shape[1], dst->shape[2]) == 0
: bssn_cuda_unpack_state_batch_from_device_buffer(
dst->Bg, state_count, buffer, dst->Bg->shape,
i0, j0, k0,
dst->shape[0], dst->shape[1], dst->shape[2]) == 0;
if (sync_profile_enabled())
sync_profile_stats().direct_unpack_sec += MPI_Wtime() - t0;
return ok;
@@ -577,16 +642,25 @@ bool cuda_flush_device_segment_batch(Block *block,
double *data,
int state_count,
const std::vector<int> &meta,
int dir)
int dir,
MyList<var> *vars)
{
if (!block || meta.empty())
return true;
const int segment_count = (int)(meta.size() / 8);
double *views[BSSN_CUDA_STATE_COUNT];
const bool have_views = cuda_build_bssn_host_views(block, vars, state_count, views);
if (dir == PACK)
return bssn_cuda_pack_state_segments_to_device_buffer(
block, state_count, data, block->shape, segment_count, meta.data()) == 0;
return bssn_cuda_unpack_state_segments_from_device_buffer(
block, state_count, data, block->shape, segment_count, meta.data()) == 0;
return have_views
? bssn_cuda_pack_state_segments_to_device_buffer_for_host_views(
block, views, state_count, data, block->shape, segment_count, meta.data()) == 0
: bssn_cuda_pack_state_segments_to_device_buffer(
block, state_count, data, block->shape, segment_count, meta.data()) == 0;
return have_views
? bssn_cuda_unpack_state_segments_from_device_buffer_for_host_views(
block, views, state_count, data, block->shape, segment_count, meta.data()) == 0
: bssn_cuda_unpack_state_segments_from_device_buffer(
block, state_count, data, block->shape, segment_count, meta.data()) == 0;
}
int cuda_data_packer_device_batched(double *data,
@@ -638,7 +712,8 @@ int cuda_data_packer_device_batched(double *data,
if (batch_block && batch_block != block)
{
if (!cuda_flush_device_segment_batch(batch_block, data, state_count, batch_meta, dir))
MyList<var> *batch_vars = (dir == PACK) ? VarLists : VarListd;
if (!cuda_flush_device_segment_batch(batch_block, data, state_count, batch_meta, dir, batch_vars))
return -1;
batch_meta.clear();
}
@@ -672,7 +747,8 @@ int cuda_data_packer_device_batched(double *data,
if (batch_block)
{
if (!cuda_flush_device_segment_batch(batch_block, data, state_count, batch_meta, dir))
MyList<var> *batch_vars = (dir == PACK) ? VarLists : VarListd;
if (!cuda_flush_device_segment_batch(batch_block, data, state_count, batch_meta, dir, batch_vars))
return -1;
}
return size_out;
@@ -4542,9 +4618,9 @@ int Parallel::data_packer(double *data, MyList<Parallel::gridseg> *src, MyList<P
cuda_can_direct_pack(src->data, dst->data, type))
{
if (s_cuda_aware_pack_active) {
handled_by_cuda = cuda_direct_pack_segment_to_device(data + size_out, src->data, dst->data, state_count, type);
handled_by_cuda = cuda_direct_pack_segment_to_device(data + size_out, src->data, dst->data, state_count, type, VarLists);
} else {
handled_by_cuda = cuda_direct_pack_segment(data + size_out, src->data, dst->data, state_count);
handled_by_cuda = cuda_direct_pack_segment(data + size_out, src->data, dst->data, state_count, VarLists);
}
if (!handled_by_cuda)
{
@@ -4557,9 +4633,9 @@ int Parallel::data_packer(double *data, MyList<Parallel::gridseg> *src, MyList<P
cuda_can_direct_unpack(dst->data, type))
{
if (s_cuda_aware_pack_active) {
handled_by_cuda = cuda_direct_unpack_segment_from_device(data + size_out, dst->data, state_count);
handled_by_cuda = cuda_direct_unpack_segment_from_device(data + size_out, dst->data, state_count, VarListd);
} else {
handled_by_cuda = cuda_direct_unpack_segment(data + size_out, dst->data, state_count);
handled_by_cuda = cuda_direct_unpack_segment(data + size_out, dst->data, state_count, VarListd);
}
if (!handled_by_cuda)
{
@@ -6476,6 +6552,26 @@ void Parallel::prepare_inter_time_level(Patch *Pat,
Block *cg = BP->data;
if (myrank == cg->rank)
{
#if USE_CUDA_BSSN
double *src1_views[BSSN_CUDA_STATE_COUNT];
double *src2_views[BSSN_CUDA_STATE_COUNT];
double *dst_views[BSSN_CUDA_STATE_COUNT];
const int state_count = cuda_state_var_count(VarList1, VarList2);
if (state_count == BSSN_CUDA_STATE_COUNT &&
cuda_build_bssn_host_views(cg, VarList1, state_count, src1_views) &&
cuda_build_bssn_host_views(cg, VarList2, state_count, src2_views) &&
cuda_build_bssn_host_views(cg, VarList3, state_count, dst_views) &&
bssn_cuda_has_resident_state(cg) &&
bssn_cuda_prepare_inter_time_level(cg, cg->shape,
src1_views, src2_views, 0, dst_views,
2, tindex) == 0)
{
if (BP == Pat->ble)
break;
BP = BP->next;
continue;
}
#endif
varl1 = VarList1;
varl2 = VarList2;
varl3 = VarList3;
@@ -6531,6 +6627,28 @@ void Parallel::prepare_inter_time_level(Patch *Pat,
Block *cg = BP->data;
if (myrank == cg->rank)
{
#if USE_CUDA_BSSN
double *src1_views[BSSN_CUDA_STATE_COUNT];
double *src2_views[BSSN_CUDA_STATE_COUNT];
double *src3_views[BSSN_CUDA_STATE_COUNT];
double *dst_views[BSSN_CUDA_STATE_COUNT];
const int state_count = cuda_state_var_count(VarList1, VarList2);
if (state_count == BSSN_CUDA_STATE_COUNT &&
cuda_build_bssn_host_views(cg, VarList1, state_count, src1_views) &&
cuda_build_bssn_host_views(cg, VarList2, state_count, src2_views) &&
cuda_build_bssn_host_views(cg, VarList3, state_count, src3_views) &&
cuda_build_bssn_host_views(cg, VarList4, state_count, dst_views) &&
bssn_cuda_has_resident_state(cg) &&
bssn_cuda_prepare_inter_time_level(cg, cg->shape,
src1_views, src2_views, src3_views, dst_views,
3, tindex) == 0)
{
if (BP == Pat->ble)
break;
BP = BP->next;
continue;
}
#endif
varl1 = VarList1;
varl2 = VarList2;
varl3 = VarList3;

13
AMSS_NCKU_source/bssn_class.C
View File

@@ -273,6 +273,12 @@ bool bssn_cuda_use_resident_sync(int lev)
bool bssn_cuda_keep_resident_after_step(int lev, int trfls_in, int analysis_lev)
{
static int keep_all_levels = -1;
if (keep_all_levels < 0)
{
const char *env = getenv("AMSS_CUDA_KEEP_ALL_LEVELS");
keep_all_levels = (env && atoi(env) != 0) ? 1 : 0;
}
static int enabled = -1;
if (enabled < 0)
{
@@ -281,11 +287,12 @@ bool bssn_cuda_keep_resident_after_step(int lev, int trfls_in, int analysis_lev)
}
if (!enabled)
return false;
// Levels at and above trfls can be read by CPU time interpolation through
// State/Old/corrector lists. Keep those conservative until multi-time-level
// resident storage is implemented.
if (lev == analysis_lev)
return false;
if (keep_all_levels)
return true;
// Conservative default: high time-refinement levels still have scattered
// CPU consumers outside the RK4/AMR exchange path.
return lev < trfls_in;
}

718
AMSS_NCKU_source/bssn_rhs_cuda.cu
View File

File diff suppressed because it is too large Load Diff

74
AMSS_NCKU_source/bssn_rhs_cuda.h
View File

@@ -97,6 +97,14 @@ int bssn_cuda_pack_state_batch_to_host_buffer(void *block_tag,
int i0, int j0, int k0,
int sx, int sy, int sz);
int bssn_cuda_pack_state_batch_to_host_buffer_for_host_views(void *block_tag,
double **state_host_key,
int state_count,
double *host_buffer,
int *ex,
int i0, int j0, int k0,
int sx, int sy, int sz);
int bssn_cuda_unpack_state_batch_from_host_buffer(void *block_tag,
int state_count,
double *host_buffer,
@@ -104,6 +112,14 @@ int bssn_cuda_unpack_state_batch_from_host_buffer(void *block_tag,
int i0, int j0, int k0,
int sx, int sy, int sz);
int bssn_cuda_unpack_state_batch_from_host_buffer_for_host_views(void *block_tag,
double **state_host_key,
int state_count,
double *host_buffer,
int *ex,
int i0, int j0, int k0,
int sx, int sy, int sz);
int bssn_cuda_pack_state_batch_to_device_buffer(void *block_tag,
int state_count,
double *device_buffer,
@@ -111,6 +127,14 @@ int bssn_cuda_pack_state_batch_to_device_buffer(void *block_tag,
int i0, int j0, int k0,
int sx, int sy, int sz);
int bssn_cuda_pack_state_batch_to_device_buffer_for_host_views(void *block_tag,
double **state_host_key,
int state_count,
double *device_buffer,
int *ex,
int i0, int j0, int k0,
int sx, int sy, int sz);
int bssn_cuda_unpack_state_batch_from_device_buffer(void *block_tag,
int state_count,
double *device_buffer,
@@ -118,6 +142,14 @@ int bssn_cuda_unpack_state_batch_from_device_buffer(void *block_tag,
int i0, int j0, int k0,
int sx, int sy, int sz);
int bssn_cuda_unpack_state_batch_from_device_buffer_for_host_views(void *block_tag,
double **state_host_key,
int state_count,
double *device_buffer,
int *ex,
int i0, int j0, int k0,
int sx, int sy, int sz);
int bssn_cuda_pack_state_segments_to_device_buffer(void *block_tag,
int state_count,
double *device_buffer,
@@ -125,6 +157,14 @@ int bssn_cuda_pack_state_segments_to_device_buffer(void *block_tag,
int segment_count,
const int *segment_meta);
int bssn_cuda_pack_state_segments_to_device_buffer_for_host_views(void *block_tag,
double **state_host_key,
int state_count,
double *device_buffer,
int *ex,
int segment_count,
const int *segment_meta);
int bssn_cuda_unpack_state_segments_from_device_buffer(void *block_tag,
int state_count,
double *device_buffer,
@@ -132,6 +172,14 @@ int bssn_cuda_unpack_state_segments_from_device_buffer(void *block_tag,
int segment_count,
const int *segment_meta);
int bssn_cuda_unpack_state_segments_from_device_buffer_for_host_views(void *block_tag,
double **state_host_key,
int state_count,
double *device_buffer,
int *ex,
int segment_count,
const int *segment_meta);
int bssn_cuda_restrict_state_batch_to_device_buffer(void *block_tag,
int state_count,
double *device_buffer,
@@ -139,6 +187,14 @@ int bssn_cuda_restrict_state_batch_to_device_buffer(void *block_tag,
int sx, int sy, int sz,
int fi0, int fj0, int fk0);
int bssn_cuda_restrict_state_batch_to_device_buffer_for_host_views(void *block_tag,
double **state_host_key,
int state_count,
double *device_buffer,
int *ex,
int sx, int sy, int sz,
int fi0, int fj0, int fk0);
int bssn_cuda_prolong_state_batch_to_device_buffer(void *block_tag,
int state_count,
double *device_buffer,
@@ -147,6 +203,15 @@ int bssn_cuda_prolong_state_batch_to_device_buffer(void *block_tag,
int ii0, int jj0, int kk0,
int lbc_i, int lbc_j, int lbc_k);
int bssn_cuda_prolong_state_batch_to_device_buffer_for_host_views(void *block_tag,
double **state_host_key,
int state_count,
double *device_buffer,
int *ex,
int sx, int sy, int sz,
int ii0, int jj0, int kk0,
int lbc_i, int lbc_j, int lbc_k);
int bssn_cuda_download_state_subset(void *block_tag,
int *ex,
int subset_count,
@@ -159,6 +224,15 @@ int bssn_cuda_upload_state_subset(void *block_tag,
const int *state_indices,
double **state_host_in);
int bssn_cuda_prepare_inter_time_level(void *block_tag,
int *ex,
double **src1_host_key,
double **src2_host_key,
double **src3_host_key,
double **dst_host_key,
int source_count,
int tindex);
int bssn_cuda_has_resident_state(void *block_tag);
void bssn_cuda_release_step_ctx(void *block_tag);