8abac8dd88
Rank [ 20]: Calc 0.000003 s Rank [ 35]: Calc 0.000003 s Rank [ 10]: Calc 0.000010 s Rank [ 17]: Calc 0.000005 s Rank [ 32]: Calc 0.000003 s，而且rank不固定，一般就是rank 10 和 rank 52； 但尽管有很多，比前者时间还是少很多 对于第二个Surf_Wave 调用 Interp_points，我发现前四个rank时间最长，比较固定，就是下面四个rank Rank [ 27]: Calc 0.331978 s Rank [ 35]: Calc 0.242219 s Rank [ 28]: Calc 0.242132 s Rank [ 36]: Calc 0.197024 s 因此下面surf_wave是核心
1762 lines
47 KiB
C
1762 lines
47 KiB
C
|
||
#include <iostream>
|
||
#include <iomanip>
|
||
#include <fstream>
|
||
#include <cstdlib>
|
||
#include <cstdio>
|
||
#include <string>
|
||
#include <cmath>
|
||
#include <new>
|
||
using namespace std;
|
||
|
||
#include "misc.h"
|
||
#include "MPatch.h"
|
||
#include "Parallel.h"
|
||
#include "fmisc.h"
|
||
|
||
Patch::Patch(int DIM, int *shapei, double *bboxi, int levi, bool buflog, int Symmetry) : lev(levi)
|
||
{
|
||
|
||
int hbuffer_width = buffer_width;
|
||
if (lev == 0)
|
||
hbuffer_width = CS_width; // specific for shell-box coulping
|
||
|
||
if (DIM != dim)
|
||
{
|
||
cout << "dimension is not consistent in Patch construction" << endl;
|
||
MPI_Abort(MPI_COMM_WORLD, 1);
|
||
}
|
||
for (int i = 0; i < dim; i++)
|
||
{
|
||
shape[i] = shapei[i];
|
||
bbox[i] = bboxi[i];
|
||
bbox[dim + i] = bboxi[dim + i];
|
||
lli[i] = uui[i] = 0;
|
||
if (buflog)
|
||
{
|
||
double DH;
|
||
#ifdef Vertex
|
||
#ifdef Cell
|
||
#error Both Cell and Vertex are defined
|
||
#endif
|
||
DH = (bbox[dim + i] - bbox[i]) / (shape[i] - 1);
|
||
#else
|
||
#ifdef Cell
|
||
DH = (bbox[dim + i] - bbox[i]) / shape[i];
|
||
#else
|
||
#error Not define Vertex nor Cell
|
||
#endif
|
||
#endif
|
||
uui[i] = hbuffer_width;
|
||
bbox[dim + i] = bbox[dim + i] + uui[i] * DH;
|
||
shape[i] = shape[i] + uui[i];
|
||
}
|
||
}
|
||
|
||
if (buflog)
|
||
{
|
||
if (DIM != 3)
|
||
{
|
||
cout << "Symmetry in Patch construction only support 3 yet but dim = " << DIM << endl;
|
||
MPI_Abort(MPI_COMM_WORLD, 1);
|
||
}
|
||
double tmpb, DH;
|
||
if (Symmetry > 0)
|
||
{
|
||
#ifdef Vertex
|
||
#ifdef Cell
|
||
#error Both Cell and Vertex are defined
|
||
#endif
|
||
DH = (bbox[5] - bbox[2]) / (shape[2] - 1);
|
||
#else
|
||
#ifdef Cell
|
||
DH = (bbox[5] - bbox[2]) / shape[2];
|
||
#else
|
||
#error Not define Vertex nor Cell
|
||
#endif
|
||
#endif
|
||
tmpb = Mymax(0, bbox[2] - hbuffer_width * DH);
|
||
lli[2] = int((bbox[2] - tmpb) / DH + 0.4);
|
||
bbox[2] = bbox[2] - lli[2] * DH;
|
||
shape[2] = shape[2] + lli[2];
|
||
if (lli[2] < hbuffer_width)
|
||
{
|
||
if (feq(bbox[2], 0, DH / 2))
|
||
lli[2] = 0;
|
||
else
|
||
{
|
||
cout << "Code mistake for lli[2] = " << lli[2] << ", bbox[2] = " << bbox[2] << endl;
|
||
MPI_Abort(MPI_COMM_WORLD, 1);
|
||
}
|
||
}
|
||
if (Symmetry > 1)
|
||
{
|
||
#ifdef Vertex
|
||
#ifdef Cell
|
||
#error Both Cell and Vertex are defined
|
||
#endif
|
||
DH = (bbox[3] - bbox[0]) / (shape[0] - 1);
|
||
#else
|
||
#ifdef Cell
|
||
DH = (bbox[3] - bbox[0]) / shape[0];
|
||
#else
|
||
#error Not define Vertex nor Cell
|
||
#endif
|
||
#endif
|
||
tmpb = Mymax(0, bbox[0] - hbuffer_width * DH);
|
||
lli[0] = int((bbox[0] - tmpb) / DH + 0.4);
|
||
bbox[0] = bbox[0] - lli[0] * DH;
|
||
shape[0] = shape[0] + lli[0];
|
||
if (lli[0] < hbuffer_width)
|
||
{
|
||
if (feq(bbox[0], 0, DH / 2))
|
||
lli[0] = 0;
|
||
else
|
||
{
|
||
cout << "Code mistake for lli[0] = " << lli[0] << ", bbox[0] = " << bbox[0] << endl;
|
||
MPI_Abort(MPI_COMM_WORLD, 1);
|
||
}
|
||
}
|
||
#ifdef Vertex
|
||
#ifdef Cell
|
||
#error Both Cell and Vertex are defined
|
||
#endif
|
||
DH = (bbox[4] - bbox[1]) / (shape[1] - 1);
|
||
#else
|
||
#ifdef Cell
|
||
DH = (bbox[4] - bbox[1]) / shape[1];
|
||
#else
|
||
#error Not define Vertex nor Cell
|
||
#endif
|
||
#endif
|
||
tmpb = Mymax(0, bbox[1] - hbuffer_width * DH);
|
||
lli[1] = int((bbox[1] - tmpb) / DH + 0.4);
|
||
bbox[1] = bbox[1] - lli[1] * DH;
|
||
shape[1] = shape[1] + lli[1];
|
||
if (lli[1] < hbuffer_width)
|
||
{
|
||
if (feq(bbox[1], 0, DH / 2))
|
||
lli[1] = 0;
|
||
else
|
||
{
|
||
cout << "Code mistake for lli[1] = " << lli[1] << ", bbox[1] = " << bbox[1] << endl;
|
||
MPI_Abort(MPI_COMM_WORLD, 1);
|
||
}
|
||
}
|
||
}
|
||
else
|
||
{
|
||
for (int i = 0; i < 2; i++)
|
||
{
|
||
#ifdef Vertex
|
||
#ifdef Cell
|
||
#error Both Cell and Vertex are defined
|
||
#endif
|
||
DH = (bbox[dim + i] - bbox[i]) / (shape[i] - 1);
|
||
#else
|
||
#ifdef Cell
|
||
DH = (bbox[dim + i] - bbox[i]) / shape[i];
|
||
#else
|
||
#error Not define Vertex nor Cell
|
||
#endif
|
||
#endif
|
||
lli[i] = hbuffer_width;
|
||
bbox[i] = bbox[i] - lli[i] * DH;
|
||
shape[i] = shape[i] + lli[i];
|
||
}
|
||
}
|
||
}
|
||
else
|
||
{
|
||
for (int i = 0; i < dim; i++)
|
||
{
|
||
#ifdef Vertex
|
||
#ifdef Cell
|
||
#error Both Cell and Vertex are defined
|
||
#endif
|
||
DH = (bbox[dim + i] - bbox[i]) / (shape[i] - 1);
|
||
#else
|
||
#ifdef Cell
|
||
DH = (bbox[dim + i] - bbox[i]) / shape[i];
|
||
#else
|
||
#error Not define Vertex nor Cell
|
||
#endif
|
||
#endif
|
||
lli[i] = hbuffer_width;
|
||
bbox[i] = bbox[i] - lli[i] * DH;
|
||
shape[i] = shape[i] + lli[i];
|
||
}
|
||
}
|
||
}
|
||
|
||
blb = ble = 0;
|
||
}
|
||
Patch::~Patch()
|
||
{
|
||
}
|
||
// buflog 1: with buffer points; 0 without
|
||
void Patch::checkPatch(bool buflog)
|
||
{
|
||
int myrank;
|
||
MPI_Comm_rank(MPI_COMM_WORLD, &myrank);
|
||
if (myrank == 0)
|
||
{
|
||
if (buflog)
|
||
{
|
||
cout << " belong to level " << lev << endl;
|
||
cout << " shape: [";
|
||
for (int i = 0; i < dim; i++)
|
||
{
|
||
cout << shape[i];
|
||
if (i < dim - 1)
|
||
cout << ",";
|
||
else
|
||
cout << "]";
|
||
}
|
||
cout << " resolution: [";
|
||
for (int i = 0; i < dim; i++)
|
||
{
|
||
cout << getdX(i);
|
||
if (i < dim - 1)
|
||
cout << ",";
|
||
else
|
||
cout << "]" << endl;
|
||
}
|
||
cout << " range:" << "(";
|
||
for (int i = 0; i < dim; i++)
|
||
{
|
||
cout << bbox[i] << ":" << bbox[dim + i];
|
||
if (i < dim - 1)
|
||
cout << ",";
|
||
else
|
||
cout << ")" << endl;
|
||
}
|
||
}
|
||
else
|
||
{
|
||
cout << " belong to level " << lev << endl;
|
||
cout << " shape: [";
|
||
for (int i = 0; i < dim; i++)
|
||
{
|
||
cout << shape[i] - lli[i] - uui[i];
|
||
if (i < dim - 1)
|
||
cout << ",";
|
||
else
|
||
cout << "]";
|
||
}
|
||
cout << " resolution: [";
|
||
for (int i = 0; i < dim; i++)
|
||
{
|
||
cout << getdX(i);
|
||
if (i < dim - 1)
|
||
cout << ",";
|
||
else
|
||
cout << "]" << endl;
|
||
}
|
||
cout << " range:" << "(";
|
||
for (int i = 0; i < dim; i++)
|
||
{
|
||
cout << bbox[i] + lli[i] * getdX(i) << ":" << bbox[dim + i] - uui[i] * getdX(i);
|
||
if (i < dim - 1)
|
||
cout << ",";
|
||
else
|
||
cout << ")" << endl;
|
||
}
|
||
}
|
||
}
|
||
}
|
||
void Patch::checkPatch(bool buflog, const int out_rank)
|
||
{
|
||
int myrank;
|
||
MPI_Comm_rank(MPI_COMM_WORLD, &myrank);
|
||
if (myrank == out_rank)
|
||
{
|
||
cout << " out_rank = " << out_rank << endl;
|
||
if (buflog)
|
||
{
|
||
cout << " belong to level " << lev << endl;
|
||
cout << " shape: [";
|
||
for (int i = 0; i < dim; i++)
|
||
{
|
||
cout << shape[i];
|
||
if (i < dim - 1)
|
||
cout << ",";
|
||
else
|
||
cout << "]";
|
||
}
|
||
cout << " resolution: [";
|
||
for (int i = 0; i < dim; i++)
|
||
{
|
||
cout << getdX(i);
|
||
if (i < dim - 1)
|
||
cout << ",";
|
||
else
|
||
cout << "]" << endl;
|
||
}
|
||
cout << " range:" << "(";
|
||
for (int i = 0; i < dim; i++)
|
||
{
|
||
cout << bbox[i] << ":" << bbox[dim + i];
|
||
if (i < dim - 1)
|
||
cout << ",";
|
||
else
|
||
cout << ")" << endl;
|
||
}
|
||
}
|
||
else
|
||
{
|
||
cout << " belong to level " << lev << endl;
|
||
cout << " shape: [";
|
||
for (int i = 0; i < dim; i++)
|
||
{
|
||
cout << shape[i] - lli[i] - uui[i];
|
||
if (i < dim - 1)
|
||
cout << ",";
|
||
else
|
||
cout << "]";
|
||
}
|
||
cout << " resolution: [";
|
||
for (int i = 0; i < dim; i++)
|
||
{
|
||
cout << getdX(i);
|
||
if (i < dim - 1)
|
||
cout << ",";
|
||
else
|
||
cout << "]" << endl;
|
||
}
|
||
cout << " range:" << "(";
|
||
for (int i = 0; i < dim; i++)
|
||
{
|
||
cout << bbox[i] + lli[i] * getdX(i) << ":" << bbox[dim + i] - uui[i] * getdX(i);
|
||
if (i < dim - 1)
|
||
cout << ",";
|
||
else
|
||
cout << ")" << endl;
|
||
}
|
||
}
|
||
}
|
||
}
|
||
void Patch::Interp_Points(MyList<var> *VarList,
|
||
int NN, double **XX,
|
||
double *Shellf, int Symmetry)
|
||
{
|
||
// NOTE: we do not Synchnize variables here, make sure of that before calling this routine
|
||
double t_calc_end, t_calc_total = 0;
|
||
double t_calc_start = MPI_Wtime();
|
||
int myrank, nprocs;
|
||
MPI_Comm_rank(MPI_COMM_WORLD, &myrank);
|
||
MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
|
||
|
||
int ordn = 2 * ghost_width;
|
||
MyList<var> *varl;
|
||
int num_var = 0;
|
||
varl = VarList;
|
||
while (varl)
|
||
{
|
||
num_var++;
|
||
varl = varl->next;
|
||
}
|
||
|
||
memset(Shellf, 0, sizeof(double) * NN * num_var);
|
||
|
||
// owner_rank[j] records which MPI rank owns point j
|
||
// All ranks traverse the same block list so they all agree on ownership
|
||
int *owner_rank;
|
||
owner_rank = new int[NN];
|
||
for (int j = 0; j < NN; j++)
|
||
owner_rank[j] = -1;
|
||
|
||
double DH[dim], llb[dim], uub[dim];
|
||
for (int i = 0; i < dim; i++)
|
||
DH[i] = getdX(i);
|
||
|
||
for (int j = 0; j < NN; j++) // run along points
|
||
{
|
||
double pox[dim];
|
||
for (int i = 0; i < dim; i++)
|
||
{
|
||
pox[i] = XX[i][j];
|
||
if (myrank == 0 && (XX[i][j] < bbox[i] + lli[i] * DH[i] || XX[i][j] > bbox[dim + i] - uui[i] * DH[i]))
|
||
{
|
||
cout << "Patch::Interp_Points: point (";
|
||
for (int k = 0; k < dim; k++)
|
||
{
|
||
cout << XX[k][j];
|
||
if (k < dim - 1)
|
||
cout << ",";
|
||
else
|
||
cout << ") is out of current Patch." << endl;
|
||
}
|
||
MPI_Abort(MPI_COMM_WORLD, 1);
|
||
}
|
||
}
|
||
|
||
MyList<Block> *Bp = blb;
|
||
bool notfind = true;
|
||
while (notfind && Bp) // run along Blocks
|
||
{
|
||
Block *BP = Bp->data;
|
||
|
||
bool flag = true;
|
||
for (int i = 0; i < dim; i++)
|
||
{
|
||
#ifdef Vertex
|
||
#ifdef Cell
|
||
#error Both Cell and Vertex are defined
|
||
#endif
|
||
llb[i] = (feq(BP->bbox[i], bbox[i], DH[i] / 2)) ? BP->bbox[i] + lli[i] * DH[i] : BP->bbox[i] + (ghost_width - 0.5) * DH[i];
|
||
uub[i] = (feq(BP->bbox[dim + i], bbox[dim + i], DH[i] / 2)) ? BP->bbox[dim + i] - uui[i] * DH[i] : BP->bbox[dim + i] - (ghost_width - 0.5) * DH[i];
|
||
#else
|
||
#ifdef Cell
|
||
llb[i] = (feq(BP->bbox[i], bbox[i], DH[i] / 2)) ? BP->bbox[i] + lli[i] * DH[i] : BP->bbox[i] + ghost_width * DH[i];
|
||
uub[i] = (feq(BP->bbox[dim + i], bbox[dim + i], DH[i] / 2)) ? BP->bbox[dim + i] - uui[i] * DH[i] : BP->bbox[dim + i] - ghost_width * DH[i];
|
||
#else
|
||
#error Not define Vertex nor Cell
|
||
#endif
|
||
#endif
|
||
if (XX[i][j] - llb[i] < -DH[i] / 2 || XX[i][j] - uub[i] > DH[i] / 2)
|
||
{
|
||
flag = false;
|
||
break;
|
||
}
|
||
}
|
||
|
||
if (flag)
|
||
{
|
||
notfind = false;
|
||
owner_rank[j] = BP->rank;
|
||
if (myrank == BP->rank)
|
||
{
|
||
//---> interpolation
|
||
varl = VarList;
|
||
int k = 0;
|
||
while (varl) // run along variables
|
||
{
|
||
f_global_interp(BP->shape, BP->X[0], BP->X[1], BP->X[2], BP->fgfs[varl->data->sgfn], Shellf[j * num_var + k],
|
||
pox[0], pox[1], pox[2], ordn, varl->data->SoA, Symmetry);
|
||
varl = varl->next;
|
||
k++;
|
||
}
|
||
}
|
||
}
|
||
if (Bp == ble)
|
||
break;
|
||
Bp = Bp->next;
|
||
}
|
||
}
|
||
t_calc_end = MPI_Wtime();
|
||
t_calc_total = t_calc_end - t_calc_start;
|
||
// Replace MPI_Allreduce with per-owner MPI_Bcast:
|
||
// Group consecutive points by owner rank and broadcast each group.
|
||
// Since each point's data is non-zero only on the owner rank,
|
||
// Bcast from owner is equivalent to Allreduce(MPI_SUM) but much cheaper.
|
||
{
|
||
int j = 0;
|
||
while (j < NN)
|
||
{
|
||
int cur_owner = owner_rank[j];
|
||
if (cur_owner < 0)
|
||
{
|
||
if (myrank == 0)
|
||
{
|
||
cout << "ERROR: Patch::Interp_Points fails to find point (";
|
||
for (int d = 0; d < dim; d++)
|
||
{
|
||
cout << XX[d][j];
|
||
if (d < dim - 1)
|
||
cout << ",";
|
||
else
|
||
cout << ")";
|
||
}
|
||
cout << " on Patch (";
|
||
for (int d = 0; d < dim; d++)
|
||
{
|
||
cout << bbox[d] << "+" << lli[d] * DH[d];
|
||
if (d < dim - 1)
|
||
cout << ",";
|
||
else
|
||
cout << ")--";
|
||
}
|
||
cout << "(";
|
||
for (int d = 0; d < dim; d++)
|
||
{
|
||
cout << bbox[dim + d] << "-" << uui[d] * DH[d];
|
||
if (d < dim - 1)
|
||
cout << ",";
|
||
else
|
||
cout << ")" << endl;
|
||
}
|
||
MPI_Abort(MPI_COMM_WORLD, 1);
|
||
}
|
||
j++;
|
||
continue;
|
||
}
|
||
// Find contiguous run of points with the same owner
|
||
int jstart = j;
|
||
while (j < NN && owner_rank[j] == cur_owner)
|
||
j++;
|
||
int count = (j - jstart) * num_var;
|
||
MPI_Bcast(Shellf + jstart * num_var, count, MPI_DOUBLE, cur_owner, MPI_COMM_WORLD);
|
||
}
|
||
}
|
||
|
||
delete[] owner_rank;
|
||
|
||
|
||
|
||
// 4. 汇总并输出真正干活最慢的 Top 10
|
||
struct RankStats {
|
||
int rank;
|
||
double calc_time; // 净计算时间
|
||
double comm_time; // 等待时间
|
||
};
|
||
|
||
// 创建当前进程的统计数据
|
||
RankStats local_stat;
|
||
local_stat.rank = myrank;
|
||
local_stat.calc_time = t_calc_total;
|
||
local_stat.comm_time = 0; // 此函数中未跟踪通信时间
|
||
|
||
// 为所有进程的统计数据分配内存
|
||
RankStats *all_stats = nullptr;
|
||
if (myrank == 0) {
|
||
all_stats = new RankStats[nprocs];
|
||
}
|
||
|
||
// 使用MPI_Gather收集所有进程的数据到rank 0
|
||
MPI_Gather(&local_stat, sizeof(RankStats), MPI_BYTE,
|
||
all_stats, sizeof(RankStats), MPI_BYTE,
|
||
0, MPI_COMM_WORLD);
|
||
|
||
if (myrank == 0) {
|
||
// 按 calc_time（净计算时间）排序
|
||
std::sort(all_stats, all_stats + nprocs, [](const RankStats& a, const RankStats& b) {
|
||
return a.calc_time > b.calc_time;
|
||
});
|
||
|
||
printf("\n--- Top 10 Ranks by ACTIVE COMPUTATION (CPU Time) ---\n");
|
||
int display_count = (nprocs < 10) ? nprocs : 10;
|
||
for (int i = 0; i < display_count; i++) {
|
||
printf("Rank [%4d]: Calc %.6f s\n",
|
||
all_stats[i].rank, all_stats[i].calc_time);
|
||
}
|
||
|
||
// 清理分配的内存
|
||
delete[] all_stats;
|
||
}
|
||
}
|
||
void Patch::Interp_Points(MyList<var> *VarList,
|
||
int NN, double **XX,
|
||
double *Shellf, int Symmetry,
|
||
int Nmin_consumer, int Nmax_consumer)
|
||
{
|
||
// Targeted point-to-point overload: each owner sends each point only to
|
||
// the one rank that needs it for integration (consumer), reducing
|
||
// communication volume by ~nprocs times compared to the Bcast version.
|
||
double t_calc_end, t_calc_total = 0;
|
||
double t_calc_start = MPI_Wtime();
|
||
int myrank, nprocs;
|
||
MPI_Comm_rank(MPI_COMM_WORLD, &myrank);
|
||
MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
|
||
|
||
int ordn = 2 * ghost_width;
|
||
MyList<var> *varl;
|
||
int num_var = 0;
|
||
varl = VarList;
|
||
while (varl)
|
||
{
|
||
num_var++;
|
||
varl = varl->next;
|
||
}
|
||
|
||
memset(Shellf, 0, sizeof(double) * NN * num_var);
|
||
|
||
// owner_rank[j] records which MPI rank owns point j
|
||
int *owner_rank;
|
||
owner_rank = new int[NN];
|
||
for (int j = 0; j < NN; j++)
|
||
owner_rank[j] = -1;
|
||
|
||
double DH[dim], llb[dim], uub[dim];
|
||
for (int i = 0; i < dim; i++)
|
||
DH[i] = getdX(i);
|
||
|
||
// --- Interpolation phase (identical to original) ---
|
||
for (int j = 0; j < NN; j++)
|
||
{
|
||
double pox[dim];
|
||
for (int i = 0; i < dim; i++)
|
||
{
|
||
pox[i] = XX[i][j];
|
||
if (myrank == 0 && (XX[i][j] < bbox[i] + lli[i] * DH[i] || XX[i][j] > bbox[dim + i] - uui[i] * DH[i]))
|
||
{
|
||
cout << "Patch::Interp_Points: point (";
|
||
for (int k = 0; k < dim; k++)
|
||
{
|
||
cout << XX[k][j];
|
||
if (k < dim - 1)
|
||
cout << ",";
|
||
else
|
||
cout << ") is out of current Patch." << endl;
|
||
}
|
||
MPI_Abort(MPI_COMM_WORLD, 1);
|
||
}
|
||
}
|
||
|
||
MyList<Block> *Bp = blb;
|
||
bool notfind = true;
|
||
while (notfind && Bp)
|
||
{
|
||
Block *BP = Bp->data;
|
||
|
||
bool flag = true;
|
||
for (int i = 0; i < dim; i++)
|
||
{
|
||
#ifdef Vertex
|
||
#ifdef Cell
|
||
#error Both Cell and Vertex are defined
|
||
#endif
|
||
llb[i] = (feq(BP->bbox[i], bbox[i], DH[i] / 2)) ? BP->bbox[i] + lli[i] * DH[i] : BP->bbox[i] + (ghost_width - 0.5) * DH[i];
|
||
uub[i] = (feq(BP->bbox[dim + i], bbox[dim + i], DH[i] / 2)) ? BP->bbox[dim + i] - uui[i] * DH[i] : BP->bbox[dim + i] - (ghost_width - 0.5) * DH[i];
|
||
#else
|
||
#ifdef Cell
|
||
llb[i] = (feq(BP->bbox[i], bbox[i], DH[i] / 2)) ? BP->bbox[i] + lli[i] * DH[i] : BP->bbox[i] + ghost_width * DH[i];
|
||
uub[i] = (feq(BP->bbox[dim + i], bbox[dim + i], DH[i] / 2)) ? BP->bbox[dim + i] - uui[i] * DH[i] : BP->bbox[dim + i] - ghost_width * DH[i];
|
||
#else
|
||
#error Not define Vertex nor Cell
|
||
#endif
|
||
#endif
|
||
if (XX[i][j] - llb[i] < -DH[i] / 2 || XX[i][j] - uub[i] > DH[i] / 2)
|
||
{
|
||
flag = false;
|
||
break;
|
||
}
|
||
}
|
||
|
||
if (flag)
|
||
{
|
||
notfind = false;
|
||
owner_rank[j] = BP->rank;
|
||
if (myrank == BP->rank)
|
||
{
|
||
varl = VarList;
|
||
int k = 0;
|
||
while (varl)
|
||
{
|
||
f_global_interp(BP->shape, BP->X[0], BP->X[1], BP->X[2], BP->fgfs[varl->data->sgfn], Shellf[j * num_var + k],
|
||
pox[0], pox[1], pox[2], ordn, varl->data->SoA, Symmetry);
|
||
varl = varl->next;
|
||
k++;
|
||
}
|
||
}
|
||
}
|
||
if (Bp == ble)
|
||
break;
|
||
Bp = Bp->next;
|
||
}
|
||
}
|
||
t_calc_end = MPI_Wtime();
|
||
t_calc_total = t_calc_end - t_calc_start;
|
||
// --- Error check for unfound points ---
|
||
for (int j = 0; j < NN; j++)
|
||
{
|
||
if (owner_rank[j] < 0 && myrank == 0)
|
||
{
|
||
cout << "ERROR: Patch::Interp_Points fails to find point (";
|
||
for (int d = 0; d < dim; d++)
|
||
{
|
||
cout << XX[d][j];
|
||
if (d < dim - 1)
|
||
cout << ",";
|
||
else
|
||
cout << ")";
|
||
}
|
||
cout << " on Patch (";
|
||
for (int d = 0; d < dim; d++)
|
||
{
|
||
cout << bbox[d] << "+" << lli[d] * DH[d];
|
||
if (d < dim - 1)
|
||
cout << ",";
|
||
else
|
||
cout << ")--";
|
||
}
|
||
cout << "(";
|
||
for (int d = 0; d < dim; d++)
|
||
{
|
||
cout << bbox[dim + d] << "-" << uui[d] * DH[d];
|
||
if (d < dim - 1)
|
||
cout << ",";
|
||
else
|
||
cout << ")" << endl;
|
||
}
|
||
MPI_Abort(MPI_COMM_WORLD, 1);
|
||
}
|
||
}
|
||
|
||
// --- Targeted point-to-point communication phase ---
|
||
// Compute consumer_rank[j] using the same deterministic formula as surface_integral
|
||
int *consumer_rank = new int[NN];
|
||
{
|
||
int mp = NN / nprocs;
|
||
int Lp = NN - nprocs * mp;
|
||
for (int j = 0; j < NN; j++)
|
||
{
|
||
if (j < Lp * (mp + 1))
|
||
consumer_rank[j] = j / (mp + 1);
|
||
else
|
||
consumer_rank[j] = Lp + (j - Lp * (mp + 1)) / mp;
|
||
}
|
||
}
|
||
|
||
// Count sends and recvs per rank
|
||
int *send_count = new int[nprocs];
|
||
int *recv_count = new int[nprocs];
|
||
memset(send_count, 0, sizeof(int) * nprocs);
|
||
memset(recv_count, 0, sizeof(int) * nprocs);
|
||
|
||
for (int j = 0; j < NN; j++)
|
||
{
|
||
int own = owner_rank[j];
|
||
int con = consumer_rank[j];
|
||
if (own == con)
|
||
continue; // local — no communication needed
|
||
if (own == myrank)
|
||
send_count[con]++;
|
||
if (con == myrank)
|
||
recv_count[own]++;
|
||
}
|
||
|
||
// Build send buffers: for each destination rank, pack (index, data) pairs
|
||
// Each entry: 1 int (point index j) + num_var doubles
|
||
int total_send = 0, total_recv = 0;
|
||
int *send_offset = new int[nprocs];
|
||
int *recv_offset = new int[nprocs];
|
||
for (int r = 0; r < nprocs; r++)
|
||
{
|
||
send_offset[r] = total_send;
|
||
total_send += send_count[r];
|
||
recv_offset[r] = total_recv;
|
||
total_recv += recv_count[r];
|
||
}
|
||
|
||
// Pack send buffers: each message contains (j, data[0..num_var-1]) per point
|
||
int stride = 1 + num_var; // 1 double for index + num_var doubles for data
|
||
double *sendbuf = new double[total_send * stride];
|
||
double *recvbuf = new double[total_recv * stride];
|
||
|
||
// Temporary counters for packing
|
||
int *pack_pos = new int[nprocs];
|
||
memset(pack_pos, 0, sizeof(int) * nprocs);
|
||
|
||
for (int j = 0; j < NN; j++)
|
||
{
|
||
int own = owner_rank[j];
|
||
int con = consumer_rank[j];
|
||
if (own != myrank || con == myrank)
|
||
continue;
|
||
int pos = (send_offset[con] + pack_pos[con]) * stride;
|
||
sendbuf[pos] = (double)j; // point index
|
||
for (int v = 0; v < num_var; v++)
|
||
sendbuf[pos + 1 + v] = Shellf[j * num_var + v];
|
||
pack_pos[con]++;
|
||
}
|
||
|
||
// Post non-blocking recvs and sends
|
||
int n_req = 0;
|
||
for (int r = 0; r < nprocs; r++)
|
||
{
|
||
if (recv_count[r] > 0) n_req++;
|
||
if (send_count[r] > 0) n_req++;
|
||
}
|
||
|
||
MPI_Request *reqs = new MPI_Request[n_req];
|
||
int req_idx = 0;
|
||
|
||
for (int r = 0; r < nprocs; r++)
|
||
{
|
||
if (recv_count[r] > 0)
|
||
{
|
||
MPI_Irecv(recvbuf + recv_offset[r] * stride,
|
||
recv_count[r] * stride, MPI_DOUBLE,
|
||
r, 0, MPI_COMM_WORLD, &reqs[req_idx++]);
|
||
}
|
||
}
|
||
for (int r = 0; r < nprocs; r++)
|
||
{
|
||
if (send_count[r] > 0)
|
||
{
|
||
MPI_Isend(sendbuf + send_offset[r] * stride,
|
||
send_count[r] * stride, MPI_DOUBLE,
|
||
r, 0, MPI_COMM_WORLD, &reqs[req_idx++]);
|
||
}
|
||
}
|
||
|
||
if (n_req > 0)
|
||
MPI_Waitall(n_req, reqs, MPI_STATUSES_IGNORE);
|
||
|
||
// Unpack recv buffers into Shellf
|
||
for (int i = 0; i < total_recv; i++)
|
||
{
|
||
int pos = i * stride;
|
||
int j = (int)recvbuf[pos];
|
||
for (int v = 0; v < num_var; v++)
|
||
Shellf[j * num_var + v] = recvbuf[pos + 1 + v];
|
||
}
|
||
|
||
delete[] reqs;
|
||
delete[] sendbuf;
|
||
delete[] recvbuf;
|
||
delete[] pack_pos;
|
||
delete[] send_offset;
|
||
delete[] recv_offset;
|
||
delete[] send_count;
|
||
delete[] recv_count;
|
||
delete[] consumer_rank;
|
||
delete[] owner_rank;
|
||
|
||
// 4. 汇总并输出真正干活最慢的 Top 10
|
||
struct RankStats {
|
||
int rank;
|
||
double calc_time; // 净计算时间
|
||
double comm_time; // 等待时间
|
||
};
|
||
|
||
// 创建当前进程的统计数据
|
||
RankStats local_stat;
|
||
local_stat.rank = myrank;
|
||
local_stat.calc_time = t_calc_total;
|
||
local_stat.comm_time = 0; // 此函数中未跟踪通信时间
|
||
|
||
// 为所有进程的统计数据分配内存
|
||
RankStats *all_stats = nullptr;
|
||
if (myrank == 0) {
|
||
all_stats = new RankStats[nprocs];
|
||
}
|
||
|
||
// 使用MPI_Gather收集所有进程的数据到rank 0
|
||
MPI_Gather(&local_stat, sizeof(RankStats), MPI_BYTE,
|
||
all_stats, sizeof(RankStats), MPI_BYTE,
|
||
0, MPI_COMM_WORLD);
|
||
|
||
if (myrank == 0) {
|
||
// 按 calc_time（净计算时间）排序
|
||
std::sort(all_stats, all_stats + nprocs, [](const RankStats& a, const RankStats& b) {
|
||
return a.calc_time > b.calc_time;
|
||
});
|
||
/*
|
||
printf("\n--- Top 10 Ranks by ACTIVE COMPUTATION (CPU Time) ---\n");
|
||
int display_count = (nprocs < 10) ? nprocs : 10;
|
||
for (int i = 0; i < display_count; i++) {
|
||
printf("Rank [%4d]: Calc %.6f s\n",
|
||
all_stats[i].rank, all_stats[i].calc_time);
|
||
}*/
|
||
|
||
// 清理分配的内存
|
||
delete[] all_stats;
|
||
}
|
||
|
||
}
|
||
void Patch::Interp_Points(MyList<var> *VarList,
|
||
int NN, double **XX,
|
||
double *Shellf, int Symmetry, MPI_Comm Comm_here)
|
||
{
|
||
// NOTE: we do not Synchnize variables here, make sure of that before calling this routine
|
||
int myrank, lmyrank;
|
||
MPI_Comm_rank(MPI_COMM_WORLD, &myrank);
|
||
MPI_Comm_rank(Comm_here, &lmyrank);
|
||
|
||
int ordn = 2 * ghost_width;
|
||
MyList<var> *varl;
|
||
int num_var = 0;
|
||
varl = VarList;
|
||
while (varl)
|
||
{
|
||
num_var++;
|
||
varl = varl->next;
|
||
}
|
||
|
||
memset(Shellf, 0, sizeof(double) * NN * num_var);
|
||
|
||
// owner_rank[j] stores the global rank that owns point j
|
||
int *owner_rank;
|
||
owner_rank = new int[NN];
|
||
for (int j = 0; j < NN; j++)
|
||
owner_rank[j] = -1;
|
||
|
||
// Build global-to-local rank translation for Comm_here
|
||
MPI_Group world_group, local_group;
|
||
MPI_Comm_group(MPI_COMM_WORLD, &world_group);
|
||
MPI_Comm_group(Comm_here, &local_group);
|
||
|
||
double DH[dim], llb[dim], uub[dim];
|
||
for (int i = 0; i < dim; i++)
|
||
DH[i] = getdX(i);
|
||
|
||
for (int j = 0; j < NN; j++) // run along points
|
||
{
|
||
double pox[dim];
|
||
for (int i = 0; i < dim; i++)
|
||
{
|
||
pox[i] = XX[i][j];
|
||
if (lmyrank == 0 && (XX[i][j] < bbox[i] + lli[i] * DH[i] || XX[i][j] > bbox[dim + i] - uui[i] * DH[i]))
|
||
{
|
||
cout << "Patch::Interp_Points: point (";
|
||
for (int k = 0; k < dim; k++)
|
||
{
|
||
cout << XX[k][j];
|
||
if (k < dim - 1)
|
||
cout << ",";
|
||
else
|
||
cout << ") is out of current Patch." << endl;
|
||
}
|
||
MPI_Abort(MPI_COMM_WORLD, 1);
|
||
}
|
||
}
|
||
|
||
MyList<Block> *Bp = blb;
|
||
bool notfind = true;
|
||
while (notfind && Bp) // run along Blocks
|
||
{
|
||
Block *BP = Bp->data;
|
||
|
||
bool flag = true;
|
||
for (int i = 0; i < dim; i++)
|
||
{
|
||
#ifdef Vertex
|
||
#ifdef Cell
|
||
#error Both Cell and Vertex are defined
|
||
#endif
|
||
llb[i] = (feq(BP->bbox[i], bbox[i], DH[i] / 2)) ? BP->bbox[i] + lli[i] * DH[i] : BP->bbox[i] + (ghost_width - 0.5) * DH[i];
|
||
uub[i] = (feq(BP->bbox[dim + i], bbox[dim + i], DH[i] / 2)) ? BP->bbox[dim + i] - uui[i] * DH[i] : BP->bbox[dim + i] - (ghost_width - 0.5) * DH[i];
|
||
#else
|
||
#ifdef Cell
|
||
llb[i] = (feq(BP->bbox[i], bbox[i], DH[i] / 2)) ? BP->bbox[i] + lli[i] * DH[i] : BP->bbox[i] + ghost_width * DH[i];
|
||
uub[i] = (feq(BP->bbox[dim + i], bbox[dim + i], DH[i] / 2)) ? BP->bbox[dim + i] - uui[i] * DH[i] : BP->bbox[dim + i] - ghost_width * DH[i];
|
||
#else
|
||
#error Not define Vertex nor Cell
|
||
#endif
|
||
#endif
|
||
if (XX[i][j] - llb[i] < -DH[i] / 2 || XX[i][j] - uub[i] > DH[i] / 2)
|
||
{
|
||
flag = false;
|
||
break;
|
||
}
|
||
}
|
||
|
||
if (flag)
|
||
{
|
||
notfind = false;
|
||
owner_rank[j] = BP->rank;
|
||
if (myrank == BP->rank)
|
||
{
|
||
//---> interpolation
|
||
varl = VarList;
|
||
int k = 0;
|
||
while (varl) // run along variables
|
||
{
|
||
f_global_interp(BP->shape, BP->X[0], BP->X[1], BP->X[2], BP->fgfs[varl->data->sgfn], Shellf[j * num_var + k],
|
||
pox[0], pox[1], pox[2], ordn, varl->data->SoA, Symmetry);
|
||
varl = varl->next;
|
||
k++;
|
||
}
|
||
}
|
||
}
|
||
if (Bp == ble)
|
||
break;
|
||
Bp = Bp->next;
|
||
}
|
||
}
|
||
|
||
// Collect unique global owner ranks and translate to local ranks in Comm_here
|
||
// Then broadcast each owner's points via MPI_Bcast on Comm_here
|
||
{
|
||
int j = 0;
|
||
while (j < NN)
|
||
{
|
||
int cur_owner_global = owner_rank[j];
|
||
if (cur_owner_global < 0)
|
||
{
|
||
// Point not found — skip (error check disabled for sub-communicator levels)
|
||
j++;
|
||
continue;
|
||
}
|
||
// Translate global rank to local rank in Comm_here
|
||
int cur_owner_local;
|
||
MPI_Group_translate_ranks(world_group, 1, &cur_owner_global, local_group, &cur_owner_local);
|
||
|
||
// Find contiguous run of points with the same owner
|
||
int jstart = j;
|
||
while (j < NN && owner_rank[j] == cur_owner_global)
|
||
j++;
|
||
int count = (j - jstart) * num_var;
|
||
MPI_Bcast(Shellf + jstart * num_var, count, MPI_DOUBLE, cur_owner_local, Comm_here);
|
||
}
|
||
}
|
||
|
||
MPI_Group_free(&world_group);
|
||
MPI_Group_free(&local_group);
|
||
delete[] owner_rank;
|
||
}
|
||
void Patch::checkBlock()
|
||
{
|
||
int myrank;
|
||
MPI_Comm_rank(MPI_COMM_WORLD, &myrank);
|
||
if (myrank == 0)
|
||
{
|
||
MyList<Block> *BP = blb;
|
||
while (BP)
|
||
{
|
||
BP->data->checkBlock();
|
||
if (BP == ble)
|
||
break;
|
||
BP = BP->next;
|
||
}
|
||
}
|
||
}
|
||
double Patch::getdX(int dir)
|
||
{
|
||
if (dir < 0 || dir >= dim)
|
||
{
|
||
cout << "Patch::getdX: error input dir = " << dir << ", this Patch has direction (0," << dim - 1 << ")" << endl;
|
||
MPI_Abort(MPI_COMM_WORLD, 1);
|
||
}
|
||
double h;
|
||
#ifdef Vertex
|
||
#ifdef Cell
|
||
#error Both Cell and Vertex are defined
|
||
#endif
|
||
if (shape[dir] == 1)
|
||
{
|
||
cout << "Patch::getdX: for direction " << dir << ", this Patch has only one point. Can not determine dX for vertex center grid." << endl;
|
||
MPI_Abort(MPI_COMM_WORLD, 1);
|
||
}
|
||
h = (bbox[dim + dir] - bbox[dir]) / (shape[dir] - 1);
|
||
#else
|
||
#ifdef Cell
|
||
h = (bbox[dim + dir] - bbox[dir]) / shape[dir];
|
||
#else
|
||
#error Not define Vertex nor Cell
|
||
#endif
|
||
#endif
|
||
return h;
|
||
}
|
||
bool Patch::Interp_ONE_Point(MyList<var> *VarList, double *XX,
|
||
double *Shellf, int Symmetry)
|
||
{
|
||
// NOTE: we do not Synchnize variables here, make sure of that before calling this routine
|
||
int myrank;
|
||
MPI_Comm_rank(MPI_COMM_WORLD, &myrank);
|
||
|
||
int ordn = 2 * ghost_width;
|
||
MyList<var> *varl;
|
||
int num_var = 0;
|
||
varl = VarList;
|
||
while (varl)
|
||
{
|
||
num_var++;
|
||
varl = varl->next;
|
||
}
|
||
|
||
double *shellf;
|
||
shellf = new double[num_var];
|
||
memset(shellf, 0, sizeof(double) * num_var);
|
||
|
||
double *DH, *llb, *uub;
|
||
DH = new double[dim];
|
||
|
||
for (int i = 0; i < dim; i++)
|
||
{
|
||
DH[i] = getdX(i);
|
||
}
|
||
llb = new double[dim];
|
||
uub = new double[dim];
|
||
|
||
double pox[dim];
|
||
for (int i = 0; i < dim; i++)
|
||
{
|
||
pox[i] = XX[i];
|
||
// has excluded the buffer points
|
||
if (XX[i] < bbox[i] + lli[i] * DH[i] - DH[i] / 100 || XX[i] > bbox[dim + i] - uui[i] * DH[i] + DH[i] / 100)
|
||
{
|
||
delete[] shellf;
|
||
delete[] DH;
|
||
delete[] llb;
|
||
delete[] uub;
|
||
return false; // out of current patch,
|
||
// remember to delete the allocated arrays before return!!!
|
||
}
|
||
}
|
||
|
||
MyList<Block> *Bp = blb;
|
||
bool notfind = true;
|
||
while (notfind && Bp) // run along Blocks
|
||
{
|
||
Block *BP = Bp->data;
|
||
|
||
bool flag = true;
|
||
for (int i = 0; i < dim; i++)
|
||
{
|
||
// NOTE: our dividing structure is (exclude ghost)
|
||
// -1 0
|
||
// 1 2
|
||
// so (0,1) does not belong to any part for vertex structure
|
||
// here we put (0,0.5) to left part and (0.5,1) to right part
|
||
// BUT for cell structure the bbox is (-1.5,0.5) and (0.5,2.5), there is no missing region at all
|
||
#ifdef Vertex
|
||
#ifdef Cell
|
||
#error Both Cell and Vertex are defined
|
||
#endif
|
||
llb[i] = (feq(BP->bbox[i], bbox[i], DH[i] / 2)) ? BP->bbox[i] + lli[i] * DH[i] : BP->bbox[i] + (ghost_width - 0.5) * DH[i];
|
||
uub[i] = (feq(BP->bbox[dim + i], bbox[dim + i], DH[i] / 2)) ? BP->bbox[dim + i] - uui[i] * DH[i] : BP->bbox[dim + i] - (ghost_width - 0.5) * DH[i];
|
||
#else
|
||
#ifdef Cell
|
||
llb[i] = (feq(BP->bbox[i], bbox[i], DH[i] / 2)) ? BP->bbox[i] + lli[i] * DH[i] : BP->bbox[i] + ghost_width * DH[i];
|
||
uub[i] = (feq(BP->bbox[dim + i], bbox[dim + i], DH[i] / 2)) ? BP->bbox[dim + i] - uui[i] * DH[i] : BP->bbox[dim + i] - ghost_width * DH[i];
|
||
#else
|
||
#error Not define Vertex nor Cell
|
||
#endif
|
||
#endif
|
||
if (XX[i] - llb[i] < -DH[i] / 2 || XX[i] - uub[i] > DH[i] / 2)
|
||
{
|
||
flag = false;
|
||
break;
|
||
}
|
||
}
|
||
|
||
if (flag)
|
||
{
|
||
notfind = false;
|
||
if (myrank == BP->rank)
|
||
{
|
||
// test old code
|
||
#if 0
|
||
#define floorint(a) ((a) < 0 ? int(a) - 1 : int(a))
|
||
//---> interpolation
|
||
int ixl,iyl,izl,ixu,iyu,izu;
|
||
double Delx,Dely,Delz;
|
||
|
||
ixl = 1+floorint((pox[0]-BP->X[0][0])/DH[0]);
|
||
iyl = 1+floorint((pox[1]-BP->X[1][0])/DH[1]);
|
||
izl = 1+floorint((pox[2]-BP->X[2][0])/DH[2]);
|
||
|
||
int nn=ordn/2;
|
||
|
||
ixl = ixl-nn;
|
||
iyl = iyl-nn;
|
||
izl = izl-nn;
|
||
|
||
int tmi;
|
||
tmi = (Symmetry==2)?-1:0;
|
||
if(ixl<tmi) ixl=tmi;
|
||
if(iyl<tmi) iyl=tmi;
|
||
tmi = (Symmetry>0)?-1:0;
|
||
if(izl<tmi) izl=tmi;
|
||
|
||
if(ixl+ordn>BP->shape[0]) ixl=BP->shape[0]-ordn;
|
||
if(iyl+ordn>BP->shape[1]) iyl=BP->shape[1]-ordn;
|
||
if(izl+ordn>BP->shape[2]) izl=BP->shape[2]-ordn;
|
||
// support cell center
|
||
if(ixl>=0) Delx = ( pox[0] - BP->X[0][ixl] )/ DH[0];
|
||
else Delx = ( pox[0] + BP->X[0][0] )/ DH[0];
|
||
if(iyl>=0) Dely = ( pox[1] - BP->X[1][iyl] )/ DH[1];
|
||
else Dely = ( pox[1] + BP->X[1][0] )/ DH[1];
|
||
if(izl>=0) Delz = ( pox[2] - BP->X[2][izl] )/ DH[2];
|
||
else Delz = ( pox[2] + BP->X[2][0] )/ DH[2];
|
||
//change to fortran index
|
||
ixl++;
|
||
iyl++;
|
||
izl++;
|
||
ixu = ixl + ordn - 1;
|
||
iyu = iyl + ordn - 1;
|
||
izu = izl + ordn - 1;
|
||
varl=VarList;
|
||
int j=0;
|
||
while(varl)
|
||
{
|
||
f_interp_2(BP->shape,BP->fgfs[varl->data->sgfn],shellf[j],ixl,ixu,iyl,iyu,izl,izu,Delx,Dely,Delz,
|
||
ordn,varl->data->SoA,Symmetry);
|
||
varl=varl->next;
|
||
j++;
|
||
} //varl
|
||
#else
|
||
//---> interpolation
|
||
varl = VarList;
|
||
int k = 0;
|
||
while (varl) // run along variables
|
||
{
|
||
// shellf[j*num_var+k] = Parallel::global_interp(dim,BP->shape,BP->X,BP->fgfs[varl->data->sgfn],
|
||
// pox,ordn,varl->data->SoA,Symmetry);
|
||
f_global_interp(BP->shape, BP->X[0], BP->X[1], BP->X[2], BP->fgfs[varl->data->sgfn], shellf[k],
|
||
pox[0], pox[1], pox[2], ordn, varl->data->SoA, Symmetry);
|
||
varl = varl->next;
|
||
k++;
|
||
}
|
||
#endif
|
||
}
|
||
}
|
||
if (Bp == ble)
|
||
break;
|
||
Bp = Bp->next;
|
||
}
|
||
|
||
if (notfind && myrank == 0)
|
||
{
|
||
cout << "ERROR: Patch::Interp_Points fails to find point (";
|
||
for (int j = 0; j < dim; j++)
|
||
{
|
||
cout << XX[j];
|
||
if (j < dim - 1)
|
||
cout << ",";
|
||
else
|
||
cout << ")";
|
||
}
|
||
cout << " on Patch (";
|
||
for (int j = 0; j < dim; j++)
|
||
{
|
||
cout << bbox[j] << "+" << lli[j] * getdX(j);
|
||
if (j < dim - 1)
|
||
cout << ",";
|
||
else
|
||
cout << ")--";
|
||
}
|
||
cout << "(";
|
||
for (int j = 0; j < dim; j++)
|
||
{
|
||
cout << bbox[dim + j] << "-" << uui[j] * getdX(j);
|
||
if (j < dim - 1)
|
||
cout << ",";
|
||
else
|
||
cout << ")" << endl;
|
||
}
|
||
#if 0
|
||
checkBlock();
|
||
#else
|
||
cout << "splited domains:" << endl;
|
||
{
|
||
MyList<Block> *Bp = blb;
|
||
while (Bp)
|
||
{
|
||
Block *BP = Bp->data;
|
||
|
||
for (int i = 0; i < dim; i++)
|
||
{
|
||
#ifdef Vertex
|
||
#ifdef Cell
|
||
#error Both Cell and Vertex are defined
|
||
#endif
|
||
llb[i] = (feq(BP->bbox[i], bbox[i], DH[i] / 2)) ? BP->bbox[i] + lli[i] * DH[i] : BP->bbox[i] + (ghost_width - 0.5) * DH[i];
|
||
uub[i] = (feq(BP->bbox[dim + i], bbox[dim + i], DH[i] / 2)) ? BP->bbox[dim + i] - uui[i] * DH[i] : BP->bbox[dim + i] - (ghost_width - 0.5) * DH[i];
|
||
#else
|
||
#ifdef Cell
|
||
llb[i] = (feq(BP->bbox[i], bbox[i], DH[i] / 2)) ? BP->bbox[i] + lli[i] * DH[i] : BP->bbox[i] + ghost_width * DH[i];
|
||
uub[i] = (feq(BP->bbox[dim + i], bbox[dim + i], DH[i] / 2)) ? BP->bbox[dim + i] - uui[i] * DH[i] : BP->bbox[dim + i] - ghost_width * DH[i];
|
||
#else
|
||
#error Not define Vertex nor Cell
|
||
#endif
|
||
#endif
|
||
}
|
||
cout << "(";
|
||
for (int j = 0; j < dim; j++)
|
||
{
|
||
cout << llb[j] << ":" << uub[j];
|
||
if (j < dim - 1)
|
||
cout << ",";
|
||
else
|
||
cout << ")" << endl;
|
||
}
|
||
if (Bp == ble)
|
||
break;
|
||
Bp = Bp->next;
|
||
}
|
||
}
|
||
#endif
|
||
MPI_Abort(MPI_COMM_WORLD, 1);
|
||
}
|
||
|
||
MPI_Allreduce(shellf, Shellf, num_var, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD);
|
||
|
||
delete[] shellf;
|
||
delete[] DH;
|
||
delete[] llb;
|
||
delete[] uub;
|
||
|
||
return true;
|
||
}
|
||
bool Patch::Interp_ONE_Point(MyList<var> *VarList, double *XX,
|
||
double *Shellf, int Symmetry, MPI_Comm Comm_here)
|
||
{
|
||
// NOTE: we do not Synchnize variables here, make sure of that before calling this routine
|
||
int myrank;
|
||
MPI_Comm_rank(MPI_COMM_WORLD, &myrank);
|
||
|
||
int ordn = 2 * ghost_width;
|
||
MyList<var> *varl;
|
||
int num_var = 0;
|
||
varl = VarList;
|
||
while (varl)
|
||
{
|
||
num_var++;
|
||
varl = varl->next;
|
||
}
|
||
|
||
double *shellf;
|
||
shellf = new double[num_var];
|
||
memset(shellf, 0, sizeof(double) * num_var);
|
||
|
||
double *DH, *llb, *uub;
|
||
DH = new double[dim];
|
||
|
||
for (int i = 0; i < dim; i++)
|
||
{
|
||
DH[i] = getdX(i);
|
||
}
|
||
llb = new double[dim];
|
||
uub = new double[dim];
|
||
|
||
double pox[dim];
|
||
for (int i = 0; i < dim; i++)
|
||
{
|
||
pox[i] = XX[i];
|
||
// has excluded the buffer points
|
||
if (XX[i] < bbox[i] + lli[i] * DH[i] - DH[i] / 100 || XX[i] > bbox[dim + i] - uui[i] * DH[i] + DH[i] / 100)
|
||
{
|
||
delete[] shellf;
|
||
delete[] DH;
|
||
delete[] llb;
|
||
delete[] uub;
|
||
return false; // out of current patch,
|
||
// remember to delete the allocated arrays before return!!!
|
||
}
|
||
}
|
||
|
||
MyList<Block> *Bp = blb;
|
||
bool notfind = true;
|
||
while (notfind && Bp) // run along Blocks
|
||
{
|
||
Block *BP = Bp->data;
|
||
|
||
bool flag = true;
|
||
for (int i = 0; i < dim; i++)
|
||
{
|
||
// NOTE: our dividing structure is (exclude ghost)
|
||
// -1 0
|
||
// 1 2
|
||
// so (0,1) does not belong to any part for vertex structure
|
||
// here we put (0,0.5) to left part and (0.5,1) to right part
|
||
// BUT for cell structure the bbox is (-1.5,0.5) and (0.5,2.5), there is no missing region at all
|
||
#ifdef Vertex
|
||
#ifdef Cell
|
||
#error Both Cell and Vertex are defined
|
||
#endif
|
||
llb[i] = (feq(BP->bbox[i], bbox[i], DH[i] / 2)) ? BP->bbox[i] + lli[i] * DH[i] : BP->bbox[i] + (ghost_width - 0.5) * DH[i];
|
||
uub[i] = (feq(BP->bbox[dim + i], bbox[dim + i], DH[i] / 2)) ? BP->bbox[dim + i] - uui[i] * DH[i] : BP->bbox[dim + i] - (ghost_width - 0.5) * DH[i];
|
||
#else
|
||
#ifdef Cell
|
||
llb[i] = (feq(BP->bbox[i], bbox[i], DH[i] / 2)) ? BP->bbox[i] + lli[i] * DH[i] : BP->bbox[i] + ghost_width * DH[i];
|
||
uub[i] = (feq(BP->bbox[dim + i], bbox[dim + i], DH[i] / 2)) ? BP->bbox[dim + i] - uui[i] * DH[i] : BP->bbox[dim + i] - ghost_width * DH[i];
|
||
#else
|
||
#error Not define Vertex nor Cell
|
||
#endif
|
||
#endif
|
||
if (XX[i] - llb[i] < -DH[i] / 2 || XX[i] - uub[i] > DH[i] / 2)
|
||
{
|
||
flag = false;
|
||
break;
|
||
}
|
||
}
|
||
|
||
if (flag)
|
||
{
|
||
notfind = false;
|
||
if (myrank == BP->rank)
|
||
{
|
||
// test old code
|
||
#if 0
|
||
#define floorint(a) ((a) < 0 ? int(a) - 1 : int(a))
|
||
//---> interpolation
|
||
int ixl,iyl,izl,ixu,iyu,izu;
|
||
double Delx,Dely,Delz;
|
||
|
||
ixl = 1+floorint((pox[0]-BP->X[0][0])/DH[0]);
|
||
iyl = 1+floorint((pox[1]-BP->X[1][0])/DH[1]);
|
||
izl = 1+floorint((pox[2]-BP->X[2][0])/DH[2]);
|
||
|
||
int nn=ordn/2;
|
||
|
||
ixl = ixl-nn;
|
||
iyl = iyl-nn;
|
||
izl = izl-nn;
|
||
|
||
int tmi;
|
||
tmi = (Symmetry==2)?-1:0;
|
||
if(ixl<tmi) ixl=tmi;
|
||
if(iyl<tmi) iyl=tmi;
|
||
tmi = (Symmetry>0)?-1:0;
|
||
if(izl<tmi) izl=tmi;
|
||
|
||
if(ixl+ordn>BP->shape[0]) ixl=BP->shape[0]-ordn;
|
||
if(iyl+ordn>BP->shape[1]) iyl=BP->shape[1]-ordn;
|
||
if(izl+ordn>BP->shape[2]) izl=BP->shape[2]-ordn;
|
||
// support cell center
|
||
if(ixl>=0) Delx = ( pox[0] - BP->X[0][ixl] )/ DH[0];
|
||
else Delx = ( pox[0] + BP->X[0][0] )/ DH[0];
|
||
if(iyl>=0) Dely = ( pox[1] - BP->X[1][iyl] )/ DH[1];
|
||
else Dely = ( pox[1] + BP->X[1][0] )/ DH[1];
|
||
if(izl>=0) Delz = ( pox[2] - BP->X[2][izl] )/ DH[2];
|
||
else Delz = ( pox[2] + BP->X[2][0] )/ DH[2];
|
||
//change to fortran index
|
||
ixl++;
|
||
iyl++;
|
||
izl++;
|
||
ixu = ixl + ordn - 1;
|
||
iyu = iyl + ordn - 1;
|
||
izu = izl + ordn - 1;
|
||
varl=VarList;
|
||
int j=0;
|
||
while(varl)
|
||
{
|
||
f_interp_2(BP->shape,BP->fgfs[varl->data->sgfn],shellf[j],ixl,ixu,iyl,iyu,izl,izu,Delx,Dely,Delz,
|
||
ordn,varl->data->SoA,Symmetry);
|
||
varl=varl->next;
|
||
j++;
|
||
} //varl
|
||
#else
|
||
//---> interpolation
|
||
varl = VarList;
|
||
int k = 0;
|
||
while (varl) // run along variables
|
||
{
|
||
// shellf[j*num_var+k] = Parallel::global_interp(dim,BP->shape,BP->X,BP->fgfs[varl->data->sgfn],
|
||
// pox,ordn,varl->data->SoA,Symmetry);
|
||
f_global_interp(BP->shape, BP->X[0], BP->X[1], BP->X[2], BP->fgfs[varl->data->sgfn], shellf[k],
|
||
pox[0], pox[1], pox[2], ordn, varl->data->SoA, Symmetry);
|
||
varl = varl->next;
|
||
k++;
|
||
}
|
||
#endif
|
||
}
|
||
}
|
||
if (Bp == ble)
|
||
break;
|
||
Bp = Bp->next;
|
||
}
|
||
|
||
if (notfind && myrank == 0)
|
||
{
|
||
cout << "ERROR: Patch::Interp_Points fails to find point (";
|
||
for (int j = 0; j < dim; j++)
|
||
{
|
||
cout << XX[j];
|
||
if (j < dim - 1)
|
||
cout << ",";
|
||
else
|
||
cout << ")";
|
||
}
|
||
cout << " on Patch (";
|
||
for (int j = 0; j < dim; j++)
|
||
{
|
||
cout << bbox[j] << "+" << lli[j] * getdX(j);
|
||
if (j < dim - 1)
|
||
cout << ",";
|
||
else
|
||
cout << ")--";
|
||
}
|
||
cout << "(";
|
||
for (int j = 0; j < dim; j++)
|
||
{
|
||
cout << bbox[dim + j] << "-" << uui[j] * getdX(j);
|
||
if (j < dim - 1)
|
||
cout << ",";
|
||
else
|
||
cout << ")" << endl;
|
||
}
|
||
#if 0
|
||
checkBlock();
|
||
#else
|
||
cout << "splited domains:" << endl;
|
||
{
|
||
MyList<Block> *Bp = blb;
|
||
while (Bp)
|
||
{
|
||
Block *BP = Bp->data;
|
||
|
||
for (int i = 0; i < dim; i++)
|
||
{
|
||
#ifdef Vertex
|
||
#ifdef Cell
|
||
#error Both Cell and Vertex are defined
|
||
#endif
|
||
llb[i] = (feq(BP->bbox[i], bbox[i], DH[i] / 2)) ? BP->bbox[i] + lli[i] * DH[i] : BP->bbox[i] + (ghost_width - 0.5) * DH[i];
|
||
uub[i] = (feq(BP->bbox[dim + i], bbox[dim + i], DH[i] / 2)) ? BP->bbox[dim + i] - uui[i] * DH[i] : BP->bbox[dim + i] - (ghost_width - 0.5) * DH[i];
|
||
#else
|
||
#ifdef Cell
|
||
llb[i] = (feq(BP->bbox[i], bbox[i], DH[i] / 2)) ? BP->bbox[i] + lli[i] * DH[i] : BP->bbox[i] + ghost_width * DH[i];
|
||
uub[i] = (feq(BP->bbox[dim + i], bbox[dim + i], DH[i] / 2)) ? BP->bbox[dim + i] - uui[i] * DH[i] : BP->bbox[dim + i] - ghost_width * DH[i];
|
||
#else
|
||
#error Not define Vertex nor Cell
|
||
#endif
|
||
#endif
|
||
}
|
||
cout << "(";
|
||
for (int j = 0; j < dim; j++)
|
||
{
|
||
cout << llb[j] << ":" << uub[j];
|
||
if (j < dim - 1)
|
||
cout << ",";
|
||
else
|
||
cout << ")" << endl;
|
||
}
|
||
if (Bp == ble)
|
||
break;
|
||
Bp = Bp->next;
|
||
}
|
||
}
|
||
#endif
|
||
MPI_Abort(MPI_COMM_WORLD, 1);
|
||
}
|
||
|
||
MPI_Allreduce(shellf, Shellf, num_var, MPI_DOUBLE, MPI_SUM, Comm_here);
|
||
|
||
delete[] shellf;
|
||
delete[] DH;
|
||
delete[] llb;
|
||
delete[] uub;
|
||
|
||
return true;
|
||
}
|
||
// find maximum of abstract value, XX store position for maximum, Shellf store maximum themselvs
|
||
void Patch::Find_Maximum(MyList<var> *VarList, double *XX,
|
||
double *Shellf)
|
||
{
|
||
int myrank;
|
||
MPI_Comm_rank(MPI_COMM_WORLD, &myrank);
|
||
|
||
MyList<var> *varl;
|
||
int num_var = 0;
|
||
varl = VarList;
|
||
while (varl)
|
||
{
|
||
num_var++;
|
||
varl = varl->next;
|
||
}
|
||
|
||
double *shellf, *xx;
|
||
shellf = new double[num_var];
|
||
xx = new double[dim * num_var];
|
||
memset(shellf, 0, sizeof(double) * num_var);
|
||
memset(xx, 0, sizeof(double) * dim * num_var);
|
||
|
||
double *DH;
|
||
int *llb, *uub;
|
||
DH = new double[dim];
|
||
|
||
for (int i = 0; i < dim; i++)
|
||
{
|
||
DH[i] = getdX(i);
|
||
}
|
||
|
||
llb = new int[dim];
|
||
uub = new int[dim];
|
||
|
||
MyList<Block> *Bp = blb;
|
||
while (Bp) // run along Blocks
|
||
{
|
||
Block *BP = Bp->data;
|
||
|
||
if (myrank == BP->rank)
|
||
{
|
||
|
||
for (int i = 0; i < dim; i++)
|
||
{
|
||
llb[i] = (feq(BP->bbox[i], bbox[i], DH[i] / 2)) ? lli[i] : ghost_width;
|
||
uub[i] = (feq(BP->bbox[dim + i], bbox[dim + i], DH[i] / 2)) ? uui[i] : ghost_width;
|
||
}
|
||
|
||
varl = VarList;
|
||
int k = 0;
|
||
double tmp, tmpx[dim];
|
||
while (varl) // run along variables
|
||
{
|
||
f_find_maximum(BP->shape, BP->X[0], BP->X[1], BP->X[2], BP->fgfs[varl->data->sgfn], tmp, tmpx, llb, uub);
|
||
if (tmp > shellf[k])
|
||
{
|
||
shellf[k] = tmp;
|
||
for (int i = 0; i < dim; i++)
|
||
xx[dim * k + i] = tmpx[i];
|
||
}
|
||
varl = varl->next;
|
||
k++;
|
||
}
|
||
}
|
||
|
||
if (Bp == ble)
|
||
break;
|
||
Bp = Bp->next;
|
||
}
|
||
|
||
struct mloc
|
||
{
|
||
double val;
|
||
int rank;
|
||
};
|
||
|
||
mloc *IN, *OUT;
|
||
IN = new mloc[num_var];
|
||
OUT = new mloc[num_var];
|
||
for (int i = 0; i < num_var; i++)
|
||
{
|
||
IN[i].val = shellf[i];
|
||
IN[i].rank = myrank;
|
||
}
|
||
|
||
MPI_Allreduce(IN, OUT, num_var, MPI_DOUBLE_INT, MPI_MAXLOC, MPI_COMM_WORLD);
|
||
|
||
for (int i = 0; i < num_var; i++)
|
||
{
|
||
Shellf[i] = OUT[i].val;
|
||
if (myrank != OUT[i].rank)
|
||
for (int k = 0; k < 3; k++)
|
||
xx[3 * i + k] = 0;
|
||
}
|
||
|
||
MPI_Allreduce(xx, XX, dim * num_var, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD);
|
||
|
||
delete[] IN;
|
||
delete[] OUT;
|
||
delete[] shellf;
|
||
delete[] xx;
|
||
delete[] DH;
|
||
delete[] llb;
|
||
delete[] uub;
|
||
}
|
||
void Patch::Find_Maximum(MyList<var> *VarList, double *XX,
|
||
double *Shellf, MPI_Comm Comm_here)
|
||
{
|
||
int myrank;
|
||
MPI_Comm_rank(MPI_COMM_WORLD, &myrank);
|
||
|
||
MyList<var> *varl;
|
||
int num_var = 0;
|
||
varl = VarList;
|
||
while (varl)
|
||
{
|
||
num_var++;
|
||
varl = varl->next;
|
||
}
|
||
|
||
double *shellf, *xx;
|
||
shellf = new double[num_var];
|
||
xx = new double[dim * num_var];
|
||
memset(shellf, 0, sizeof(double) * num_var);
|
||
memset(xx, 0, sizeof(double) * dim * num_var);
|
||
|
||
double *DH;
|
||
int *llb, *uub;
|
||
DH = new double[dim];
|
||
|
||
for (int i = 0; i < dim; i++)
|
||
{
|
||
DH[i] = getdX(i);
|
||
}
|
||
|
||
llb = new int[dim];
|
||
uub = new int[dim];
|
||
|
||
MyList<Block> *Bp = blb;
|
||
while (Bp) // run along Blocks
|
||
{
|
||
Block *BP = Bp->data;
|
||
|
||
if (myrank == BP->rank)
|
||
{
|
||
|
||
for (int i = 0; i < dim; i++)
|
||
{
|
||
llb[i] = (feq(BP->bbox[i], bbox[i], DH[i] / 2)) ? lli[i] : ghost_width;
|
||
uub[i] = (feq(BP->bbox[dim + i], bbox[dim + i], DH[i] / 2)) ? uui[i] : ghost_width;
|
||
}
|
||
|
||
varl = VarList;
|
||
int k = 0;
|
||
double tmp, tmpx[dim];
|
||
while (varl) // run along variables
|
||
{
|
||
f_find_maximum(BP->shape, BP->X[0], BP->X[1], BP->X[2], BP->fgfs[varl->data->sgfn], tmp, tmpx, llb, uub);
|
||
if (tmp > shellf[k])
|
||
{
|
||
shellf[k] = tmp;
|
||
for (int i = 0; i < dim; i++)
|
||
xx[dim * k + i] = tmpx[i];
|
||
}
|
||
varl = varl->next;
|
||
k++;
|
||
}
|
||
}
|
||
|
||
if (Bp == ble)
|
||
break;
|
||
Bp = Bp->next;
|
||
}
|
||
|
||
struct mloc
|
||
{
|
||
double val;
|
||
int rank;
|
||
};
|
||
|
||
mloc *IN, *OUT;
|
||
IN = new mloc[num_var];
|
||
OUT = new mloc[num_var];
|
||
for (int i = 0; i < num_var; i++)
|
||
{
|
||
IN[i].val = shellf[i];
|
||
IN[i].rank = myrank;
|
||
}
|
||
|
||
MPI_Allreduce(IN, OUT, num_var, MPI_DOUBLE_INT, MPI_MAXLOC, Comm_here);
|
||
|
||
for (int i = 0; i < num_var; i++)
|
||
{
|
||
Shellf[i] = OUT[i].val;
|
||
if (myrank != OUT[i].rank)
|
||
for (int k = 0; k < 3; k++)
|
||
xx[3 * i + k] = 0;
|
||
}
|
||
|
||
MPI_Allreduce(xx, XX, dim * num_var, MPI_DOUBLE, MPI_SUM, Comm_here);
|
||
|
||
delete[] IN;
|
||
delete[] OUT;
|
||
delete[] shellf;
|
||
delete[] xx;
|
||
delete[] DH;
|
||
delete[] llb;
|
||
delete[] uub;
|
||
}
|
||
// if the given point locates in the present Patch return true
|
||
// otherwise return false
|
||
bool Patch::Find_Point(double *XX)
|
||
{
|
||
double *DH;
|
||
DH = new double[dim];
|
||
|
||
for (int i = 0; i < dim; i++)
|
||
{
|
||
DH[i] = getdX(i);
|
||
}
|
||
|
||
for (int i = 0; i < dim; i++)
|
||
{
|
||
// has excluded the buffer points
|
||
if (XX[i] < bbox[i] + lli[i] * DH[i] - DH[i] / 100 || XX[i] > bbox[dim + i] - uui[i] * DH[i] + DH[i] / 100)
|
||
{
|
||
delete[] DH;
|
||
return false; // out of current patch,
|
||
// remember to delete the allocated arrays before return!!!
|
||
}
|
||
}
|
||
|
||
delete[] DH;
|
||
|
||
return true;
|
||
} |