1. Pass 1(357-395行):遍历所有 Patch,对每个 block 计算含ghost zone 的实际体积,存入 block_volumes

2. Greedy LPT(397-414行):按体积从大到小排序,依次分配给当前负载最小的 rank
  3. Pass 2(416-555行):原来的 block创建循环,但用 assigned_ranks[block_idx++] 替代 n_rank++,Block
  构造时直接拿到正确的 rank,内存分配在对的进程上
This commit is contained in:
2026-02-12 03:22:46 +08:00
parent 93362baee5
commit b32675ba99

View File

@@ -4,6 +4,8 @@
#include "prolongrestrict.h" #include "prolongrestrict.h"
#include "misc.h" #include "misc.h"
#include "parameters.h" #include "parameters.h"
#include <vector>
#include <algorithm>
int Parallel::partition1(int &nx, int split_size, int min_width, int cpusize, int shape) // special for 1 diemnsion int Parallel::partition1(int &nx, int split_size, int min_width, int cpusize, int shape) // special for 1 diemnsion
{ {
@@ -352,14 +354,73 @@ MyList<Block> *Parallel::distribute(MyList<Patch> *PatchLIST, int cpusize, int i
split_size = Mymax(min_size, block_size / nodes); split_size = Mymax(min_size, block_size / nodes);
split_size = Mymax(1, split_size); split_size = Mymax(1, split_size);
int n_rank = 0; // Pass 1: compute block volumes for greedy rank assignment
std::vector<long> block_volumes;
PLi = PatchLIST; PLi = PatchLIST;
int reacpu = 0; int reacpu = 0;
while (PLi) while (PLi)
{ {
Patch *PP = PLi->data; Patch *PP = PLi->data;
reacpu += partition3(nxyz, split_size, mmin_width, nodes, PP->shape); reacpu += partition3(nxyz, split_size, mmin_width, nodes, PP->shape);
int ibbox_here[2 * dim];
for (int i = 0; i < nxyz[0]; i++)
for (int j = 0; j < nxyz[1]; j++)
for (int k = 0; k < nxyz[2]; k++)
{
ibbox_here[0] = (PP->shape[0] * i) / nxyz[0];
ibbox_here[3] = (PP->shape[0] * (i + 1)) / nxyz[0] - 1;
ibbox_here[1] = (PP->shape[1] * j) / nxyz[1];
ibbox_here[4] = (PP->shape[1] * (j + 1)) / nxyz[1] - 1;
ibbox_here[2] = (PP->shape[2] * k) / nxyz[2];
ibbox_here[5] = (PP->shape[2] * (k + 1)) / nxyz[2] - 1;
if (periodic)
{
for (int d = 0; d < dim; d++) { ibbox_here[d] -= ghost_width; ibbox_here[dim + d] += ghost_width; }
}
else
{
ibbox_here[0] = Mymax(0, ibbox_here[0] - ghost_width);
ibbox_here[3] = Mymin(PP->shape[0] - 1, ibbox_here[3] + ghost_width);
ibbox_here[1] = Mymax(0, ibbox_here[1] - ghost_width);
ibbox_here[4] = Mymin(PP->shape[1] - 1, ibbox_here[4] + ghost_width);
ibbox_here[2] = Mymax(0, ibbox_here[2] - ghost_width);
ibbox_here[5] = Mymin(PP->shape[2] - 1, ibbox_here[5] + ghost_width);
}
long vol = 1;
for (int d = 0; d < dim; d++)
vol *= (ibbox_here[dim + d] - ibbox_here[d] + 1);
block_volumes.push_back(vol);
}
PLi = PLi->next;
}
// Greedy LPT: sort by volume descending, assign each to least-loaded rank
std::vector<int> assigned_ranks(block_volumes.size());
{
std::vector<int> order(block_volumes.size());
for (int i = 0; i < (int)order.size(); i++) order[i] = i;
std::sort(order.begin(), order.end(), [&](int a, int b) {
return block_volumes[a] > block_volumes[b];
});
std::vector<long> load(cpusize, 0);
for (int idx : order)
{
int min_r = 0;
for (int r = 1; r < cpusize; r++)
if (load[r] < load[min_r]) min_r = r;
assigned_ranks[idx] = min_r;
load[min_r] += block_volumes[idx];
}
}
// Pass 2: create blocks with pre-assigned ranks
int block_idx = 0;
PLi = PatchLIST;
while (PLi)
{
Patch *PP = PLi->data;
partition3(nxyz, split_size, mmin_width, nodes, PP->shape);
Block *ng0, *ng; Block *ng0, *ng;
int shape_here[dim], ibbox_here[2 * dim]; int shape_here[dim], ibbox_here[2 * dim];
@@ -443,10 +504,7 @@ MyList<Block> *Parallel::distribute(MyList<Patch> *PatchLIST, int cpusize, int i
int shape_res[dim * pices]; int shape_res[dim * pices];
double bbox_res[2 * dim * pices]; double bbox_res[2 * dim * pices];
misc::dividBlock(dim, shape_here, bbox_here, pices, picef, shape_res, bbox_res, min_width); misc::dividBlock(dim, shape_here, bbox_here, pices, picef, shape_res, bbox_res, min_width);
ng = ng0 = new Block(dim, shape_res, bbox_res, n_rank++, ingfsi, fngfsi, PP->lev, 0); // delete through KillBlocks ng = ng0 = new Block(dim, shape_res, bbox_res, assigned_ranks[block_idx++], ingfsi, fngfsi, PP->lev, 0); // delete through KillBlocks
// if(n_rank==cpusize) {n_rank=0; cerr<<"place one!!"<<endl;}
// ng->checkBlock(); // ng->checkBlock();
if (BlL) if (BlL)
BlL->insert(ng); BlL->insert(ng);
@@ -455,22 +513,19 @@ MyList<Block> *Parallel::distribute(MyList<Patch> *PatchLIST, int cpusize, int i
for (int i = 1; i < pices; i++) for (int i = 1; i < pices; i++)
{ {
ng = new Block(dim, shape_res + i * dim, bbox_res + i * 2 * dim, n_rank++, ingfsi, fngfsi, PP->lev, i); // delete through KillBlocks ng = new Block(dim, shape_res + i * dim, bbox_res + i * 2 * dim, assigned_ranks[block_idx++], ingfsi, fngfsi, PP->lev, i); // delete through KillBlocks
// if(n_rank==cpusize) {n_rank=0; cerr<<"place two!! "<<i<<endl;}
// ng->checkBlock(); // ng->checkBlock();
BlL->insert(ng); BlL->insert(ng);
} }
} }
#else #else
ng = ng0 = new Block(dim, shape_here, bbox_here, n_rank++, ingfsi, fngfsi, PP->lev); // delete through KillBlocks ng = ng0 = new Block(dim, shape_here, bbox_here, assigned_ranks[block_idx++], ingfsi, fngfsi, PP->lev); // delete through KillBlocks
// ng->checkBlock(); // ng->checkBlock();
if (BlL) if (BlL)
BlL->insert(ng); BlL->insert(ng);
else else
BlL = new MyList<Block>(ng); // delete through KillBlocks BlL = new MyList<Block>(ng); // delete through KillBlocks
#endif #endif
if (n_rank == cpusize)
n_rank = 0;
// set PP->blb // set PP->blb
if (i == 0 && j == 0 && k == 0) if (i == 0 && j == 0 && k == 0)
@@ -3524,10 +3579,8 @@ void Parallel::transfer(MyList<Parallel::gridseg> **src, MyList<Parallel::gridse
*/ */
int node; int node;
MPI_Request *reqs; MPI_Request *reqs = new MPI_Request[2 * cpusize];
MPI_Status *stats; MPI_Status *stats = new MPI_Status[2 * cpusize];
reqs = new MPI_Request[2 * cpusize];
stats = new MPI_Status[2 * cpusize];
int req_no = 0; int req_no = 0;
double **send_data, **rec_data; double **send_data, **rec_data;
@@ -3538,7 +3591,7 @@ void Parallel::transfer(MyList<Parallel::gridseg> **src, MyList<Parallel::gridse
for (node = 0; node < cpusize; node++) for (node = 0; node < cpusize; node++)
send_data[node] = rec_data[node] = 0; send_data[node] = rec_data[node] = 0;
// 第1: 本地拷贝 + 所有 Irecv(先 post 接收,减少 unexpected message buffering // 第1: 本地拷贝 + 所有 Irecv
for (node = 0; node < cpusize; node++) for (node = 0; node < cpusize; node++)
{ {
if (node == myrank) if (node == myrank)
@@ -3551,7 +3604,6 @@ void Parallel::transfer(MyList<Parallel::gridseg> **src, MyList<Parallel::gridse
} }
else else
{ {
// receive from cpu#node to this cpu
if (length = data_packer(0, src[node], dst[node], node, UNPACK, VarList1, VarList2, Symmetry)) if (length = data_packer(0, src[node], dst[node], node, UNPACK, VarList1, VarList2, Symmetry))
{ {
rec_data[node] = new double[length]; rec_data[node] = new double[length];
@@ -3560,11 +3612,10 @@ void Parallel::transfer(MyList<Parallel::gridseg> **src, MyList<Parallel::gridse
} }
} }
// 第2: 所有 Isend // 第2: pack + Isend
for (node = 0; node < cpusize; node++) for (node = 0; node < cpusize; node++)
{ {
if (node == myrank) continue; if (node == myrank) continue;
// send from this cpu to cpu#node
if (length = data_packer(0, src[myrank], dst[myrank], node, PACK, VarList1, VarList2, Symmetry)) if (length = data_packer(0, src[myrank], dst[myrank], node, PACK, VarList1, VarList2, Symmetry))
{ {
send_data[node] = new double[length]; send_data[node] = new double[length];
@@ -3573,9 +3624,6 @@ void Parallel::transfer(MyList<Parallel::gridseg> **src, MyList<Parallel::gridse
} }
} }
// 一次性等待所有通信完成
//for (int i = 0; i < req_no; i++)
//MPI_Wait(&reqs[i], &stats[i]);
MPI_Waitall(req_no, reqs, stats); MPI_Waitall(req_no, reqs, stats);
for (node = 0; node < cpusize; node++) for (node = 0; node < cpusize; node++)