1682 lines
45 KiB
C
1682 lines
45 KiB
C
|
|
#include <iostream>
|
|
#include <iomanip>
|
|
#include <fstream>
|
|
#include <cstdlib>
|
|
#include <cstdio>
|
|
#include <string>
|
|
#include <cmath>
|
|
#include <new>
|
|
using namespace std;
|
|
|
|
#include "misc.h"
|
|
#include "MPatch.h"
|
|
#include "Parallel.h"
|
|
#include "fmisc.h"
|
|
#include "xh_global_interp.h"
|
|
Patch::Patch(int DIM, int *shapei, double *bboxi, int levi, bool buflog, int Symmetry) : lev(levi)
|
|
{
|
|
|
|
int hbuffer_width = buffer_width;
|
|
if (lev == 0)
|
|
hbuffer_width = CS_width; // specific for shell-box coulping
|
|
|
|
if (DIM != dim)
|
|
{
|
|
cout << "dimension is not consistent in Patch construction" << endl;
|
|
MPI_Abort(MPI_COMM_WORLD, 1);
|
|
}
|
|
for (int i = 0; i < dim; i++)
|
|
{
|
|
shape[i] = shapei[i];
|
|
bbox[i] = bboxi[i];
|
|
bbox[dim + i] = bboxi[dim + i];
|
|
lli[i] = uui[i] = 0;
|
|
if (buflog)
|
|
{
|
|
double DH;
|
|
#ifdef Vertex
|
|
#ifdef Cell
|
|
#error Both Cell and Vertex are defined
|
|
#endif
|
|
DH = (bbox[dim + i] - bbox[i]) / (shape[i] - 1);
|
|
#else
|
|
#ifdef Cell
|
|
DH = (bbox[dim + i] - bbox[i]) / shape[i];
|
|
#else
|
|
#error Not define Vertex nor Cell
|
|
#endif
|
|
#endif
|
|
uui[i] = hbuffer_width;
|
|
bbox[dim + i] = bbox[dim + i] + uui[i] * DH;
|
|
shape[i] = shape[i] + uui[i];
|
|
}
|
|
}
|
|
|
|
if (buflog)
|
|
{
|
|
if (DIM != 3)
|
|
{
|
|
cout << "Symmetry in Patch construction only support 3 yet but dim = " << DIM << endl;
|
|
MPI_Abort(MPI_COMM_WORLD, 1);
|
|
}
|
|
double tmpb, DH;
|
|
if (Symmetry > 0)
|
|
{
|
|
#ifdef Vertex
|
|
#ifdef Cell
|
|
#error Both Cell and Vertex are defined
|
|
#endif
|
|
DH = (bbox[5] - bbox[2]) / (shape[2] - 1);
|
|
#else
|
|
#ifdef Cell
|
|
DH = (bbox[5] - bbox[2]) / shape[2];
|
|
#else
|
|
#error Not define Vertex nor Cell
|
|
#endif
|
|
#endif
|
|
tmpb = Mymax(0, bbox[2] - hbuffer_width * DH);
|
|
lli[2] = int((bbox[2] - tmpb) / DH + 0.4);
|
|
bbox[2] = bbox[2] - lli[2] * DH;
|
|
shape[2] = shape[2] + lli[2];
|
|
if (lli[2] < hbuffer_width)
|
|
{
|
|
if (feq(bbox[2], 0, DH / 2))
|
|
lli[2] = 0;
|
|
else
|
|
{
|
|
cout << "Code mistake for lli[2] = " << lli[2] << ", bbox[2] = " << bbox[2] << endl;
|
|
MPI_Abort(MPI_COMM_WORLD, 1);
|
|
}
|
|
}
|
|
if (Symmetry > 1)
|
|
{
|
|
#ifdef Vertex
|
|
#ifdef Cell
|
|
#error Both Cell and Vertex are defined
|
|
#endif
|
|
DH = (bbox[3] - bbox[0]) / (shape[0] - 1);
|
|
#else
|
|
#ifdef Cell
|
|
DH = (bbox[3] - bbox[0]) / shape[0];
|
|
#else
|
|
#error Not define Vertex nor Cell
|
|
#endif
|
|
#endif
|
|
tmpb = Mymax(0, bbox[0] - hbuffer_width * DH);
|
|
lli[0] = int((bbox[0] - tmpb) / DH + 0.4);
|
|
bbox[0] = bbox[0] - lli[0] * DH;
|
|
shape[0] = shape[0] + lli[0];
|
|
if (lli[0] < hbuffer_width)
|
|
{
|
|
if (feq(bbox[0], 0, DH / 2))
|
|
lli[0] = 0;
|
|
else
|
|
{
|
|
cout << "Code mistake for lli[0] = " << lli[0] << ", bbox[0] = " << bbox[0] << endl;
|
|
MPI_Abort(MPI_COMM_WORLD, 1);
|
|
}
|
|
}
|
|
#ifdef Vertex
|
|
#ifdef Cell
|
|
#error Both Cell and Vertex are defined
|
|
#endif
|
|
DH = (bbox[4] - bbox[1]) / (shape[1] - 1);
|
|
#else
|
|
#ifdef Cell
|
|
DH = (bbox[4] - bbox[1]) / shape[1];
|
|
#else
|
|
#error Not define Vertex nor Cell
|
|
#endif
|
|
#endif
|
|
tmpb = Mymax(0, bbox[1] - hbuffer_width * DH);
|
|
lli[1] = int((bbox[1] - tmpb) / DH + 0.4);
|
|
bbox[1] = bbox[1] - lli[1] * DH;
|
|
shape[1] = shape[1] + lli[1];
|
|
if (lli[1] < hbuffer_width)
|
|
{
|
|
if (feq(bbox[1], 0, DH / 2))
|
|
lli[1] = 0;
|
|
else
|
|
{
|
|
cout << "Code mistake for lli[1] = " << lli[1] << ", bbox[1] = " << bbox[1] << endl;
|
|
MPI_Abort(MPI_COMM_WORLD, 1);
|
|
}
|
|
}
|
|
}
|
|
else
|
|
{
|
|
for (int i = 0; i < 2; i++)
|
|
{
|
|
#ifdef Vertex
|
|
#ifdef Cell
|
|
#error Both Cell and Vertex are defined
|
|
#endif
|
|
DH = (bbox[dim + i] - bbox[i]) / (shape[i] - 1);
|
|
#else
|
|
#ifdef Cell
|
|
DH = (bbox[dim + i] - bbox[i]) / shape[i];
|
|
#else
|
|
#error Not define Vertex nor Cell
|
|
#endif
|
|
#endif
|
|
lli[i] = hbuffer_width;
|
|
bbox[i] = bbox[i] - lli[i] * DH;
|
|
shape[i] = shape[i] + lli[i];
|
|
}
|
|
}
|
|
}
|
|
else
|
|
{
|
|
for (int i = 0; i < dim; i++)
|
|
{
|
|
#ifdef Vertex
|
|
#ifdef Cell
|
|
#error Both Cell and Vertex are defined
|
|
#endif
|
|
DH = (bbox[dim + i] - bbox[i]) / (shape[i] - 1);
|
|
#else
|
|
#ifdef Cell
|
|
DH = (bbox[dim + i] - bbox[i]) / shape[i];
|
|
#else
|
|
#error Not define Vertex nor Cell
|
|
#endif
|
|
#endif
|
|
lli[i] = hbuffer_width;
|
|
bbox[i] = bbox[i] - lli[i] * DH;
|
|
shape[i] = shape[i] + lli[i];
|
|
}
|
|
}
|
|
}
|
|
|
|
blb = ble = 0;
|
|
}
|
|
Patch::~Patch()
|
|
{
|
|
}
|
|
// buflog 1: with buffer points; 0 without
|
|
void Patch::checkPatch(bool buflog)
|
|
{
|
|
int myrank;
|
|
MPI_Comm_rank(MPI_COMM_WORLD, &myrank);
|
|
if (myrank == 0)
|
|
{
|
|
if (buflog)
|
|
{
|
|
cout << " belong to level " << lev << endl;
|
|
cout << " shape: [";
|
|
for (int i = 0; i < dim; i++)
|
|
{
|
|
cout << shape[i];
|
|
if (i < dim - 1)
|
|
cout << ",";
|
|
else
|
|
cout << "]";
|
|
}
|
|
cout << " resolution: [";
|
|
for (int i = 0; i < dim; i++)
|
|
{
|
|
cout << getdX(i);
|
|
if (i < dim - 1)
|
|
cout << ",";
|
|
else
|
|
cout << "]" << endl;
|
|
}
|
|
cout << " range:" << "(";
|
|
for (int i = 0; i < dim; i++)
|
|
{
|
|
cout << bbox[i] << ":" << bbox[dim + i];
|
|
if (i < dim - 1)
|
|
cout << ",";
|
|
else
|
|
cout << ")" << endl;
|
|
}
|
|
}
|
|
else
|
|
{
|
|
cout << " belong to level " << lev << endl;
|
|
cout << " shape: [";
|
|
for (int i = 0; i < dim; i++)
|
|
{
|
|
cout << shape[i] - lli[i] - uui[i];
|
|
if (i < dim - 1)
|
|
cout << ",";
|
|
else
|
|
cout << "]";
|
|
}
|
|
cout << " resolution: [";
|
|
for (int i = 0; i < dim; i++)
|
|
{
|
|
cout << getdX(i);
|
|
if (i < dim - 1)
|
|
cout << ",";
|
|
else
|
|
cout << "]" << endl;
|
|
}
|
|
cout << " range:" << "(";
|
|
for (int i = 0; i < dim; i++)
|
|
{
|
|
cout << bbox[i] + lli[i] * getdX(i) << ":" << bbox[dim + i] - uui[i] * getdX(i);
|
|
if (i < dim - 1)
|
|
cout << ",";
|
|
else
|
|
cout << ")" << endl;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
void Patch::checkPatch(bool buflog, const int out_rank)
|
|
{
|
|
int myrank;
|
|
MPI_Comm_rank(MPI_COMM_WORLD, &myrank);
|
|
if (myrank == out_rank)
|
|
{
|
|
cout << " out_rank = " << out_rank << endl;
|
|
if (buflog)
|
|
{
|
|
cout << " belong to level " << lev << endl;
|
|
cout << " shape: [";
|
|
for (int i = 0; i < dim; i++)
|
|
{
|
|
cout << shape[i];
|
|
if (i < dim - 1)
|
|
cout << ",";
|
|
else
|
|
cout << "]";
|
|
}
|
|
cout << " resolution: [";
|
|
for (int i = 0; i < dim; i++)
|
|
{
|
|
cout << getdX(i);
|
|
if (i < dim - 1)
|
|
cout << ",";
|
|
else
|
|
cout << "]" << endl;
|
|
}
|
|
cout << " range:" << "(";
|
|
for (int i = 0; i < dim; i++)
|
|
{
|
|
cout << bbox[i] << ":" << bbox[dim + i];
|
|
if (i < dim - 1)
|
|
cout << ",";
|
|
else
|
|
cout << ")" << endl;
|
|
}
|
|
}
|
|
else
|
|
{
|
|
cout << " belong to level " << lev << endl;
|
|
cout << " shape: [";
|
|
for (int i = 0; i < dim; i++)
|
|
{
|
|
cout << shape[i] - lli[i] - uui[i];
|
|
if (i < dim - 1)
|
|
cout << ",";
|
|
else
|
|
cout << "]";
|
|
}
|
|
cout << " resolution: [";
|
|
for (int i = 0; i < dim; i++)
|
|
{
|
|
cout << getdX(i);
|
|
if (i < dim - 1)
|
|
cout << ",";
|
|
else
|
|
cout << "]" << endl;
|
|
}
|
|
cout << " range:" << "(";
|
|
for (int i = 0; i < dim; i++)
|
|
{
|
|
cout << bbox[i] + lli[i] * getdX(i) << ":" << bbox[dim + i] - uui[i] * getdX(i);
|
|
if (i < dim - 1)
|
|
cout << ",";
|
|
else
|
|
cout << ")" << endl;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
void Patch::Interp_Points(MyList<var> *VarList,
|
|
int NN, double **XX,
|
|
double *Shellf, int Symmetry)
|
|
{
|
|
// NOTE: we do not Synchnize variables here, make sure of that before calling this routine
|
|
int myrank, nprocs;
|
|
MPI_Comm_rank(MPI_COMM_WORLD, &myrank);
|
|
MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
|
|
|
|
int ordn = 2 * ghost_width;
|
|
MyList<var> *varl;
|
|
int num_var = 0;
|
|
varl = VarList;
|
|
while (varl)
|
|
{
|
|
num_var++;
|
|
varl = varl->next;
|
|
}
|
|
|
|
memset(Shellf, 0, sizeof(double) * NN * num_var);
|
|
|
|
// owner_rank[j] records which MPI rank owns point j
|
|
// All ranks traverse the same block list so they all agree on ownership
|
|
int *owner_rank;
|
|
owner_rank = new int[NN];
|
|
for (int j = 0; j < NN; j++)
|
|
owner_rank[j] = -1;
|
|
|
|
double DH[dim], llb[dim], uub[dim];
|
|
for (int i = 0; i < dim; i++)
|
|
DH[i] = getdX(i);
|
|
|
|
for (int j = 0; j < NN; j++) // run along points
|
|
{
|
|
double pox[dim];
|
|
for (int i = 0; i < dim; i++)
|
|
{
|
|
pox[i] = XX[i][j];
|
|
if (myrank == 0 && (XX[i][j] < bbox[i] + lli[i] * DH[i] || XX[i][j] > bbox[dim + i] - uui[i] * DH[i]))
|
|
{
|
|
cout << "Patch::Interp_Points: point (";
|
|
for (int k = 0; k < dim; k++)
|
|
{
|
|
cout << XX[k][j];
|
|
if (k < dim - 1)
|
|
cout << ",";
|
|
else
|
|
cout << ") is out of current Patch." << endl;
|
|
}
|
|
MPI_Abort(MPI_COMM_WORLD, 1);
|
|
}
|
|
}
|
|
|
|
MyList<Block> *Bp = blb;
|
|
bool notfind = true;
|
|
while (notfind && Bp) // run along Blocks
|
|
{
|
|
Block *BP = Bp->data;
|
|
bool flag = true;
|
|
for (int i = 0; i < dim; i++)
|
|
{
|
|
#ifdef Vertex
|
|
#ifdef Cell
|
|
#error Both Cell and Vertex are defined
|
|
#endif
|
|
llb[i] = (feq(BP->bbox[i], bbox[i], DH[i] / 2)) ? BP->bbox[i] + lli[i] * DH[i] : BP->bbox[i] + (ghost_width - 0.5) * DH[i];
|
|
uub[i] = (feq(BP->bbox[dim + i], bbox[dim + i], DH[i] / 2)) ? BP->bbox[dim + i] - uui[i] * DH[i] : BP->bbox[dim + i] - (ghost_width - 0.5) * DH[i];
|
|
#else
|
|
#ifdef Cell
|
|
llb[i] = (feq(BP->bbox[i], bbox[i], DH[i] / 2)) ? BP->bbox[i] + lli[i] * DH[i] : BP->bbox[i] + ghost_width * DH[i];
|
|
uub[i] = (feq(BP->bbox[dim + i], bbox[dim + i], DH[i] / 2)) ? BP->bbox[dim + i] - uui[i] * DH[i] : BP->bbox[dim + i] - ghost_width * DH[i];
|
|
#else
|
|
#error Not define Vertex nor Cell
|
|
#endif
|
|
#endif
|
|
if (XX[i][j] - llb[i] < -DH[i] / 2 || XX[i][j] - uub[i] > DH[i] / 2)
|
|
{
|
|
flag = false;
|
|
break;
|
|
}
|
|
}
|
|
|
|
if (flag)
|
|
{
|
|
notfind = false;
|
|
owner_rank[j] = BP->rank;
|
|
if (myrank == BP->rank)
|
|
{
|
|
//---> interpolation
|
|
varl = VarList;
|
|
int k = 0;
|
|
while (varl) // run along variables
|
|
{
|
|
|
|
xh_global_interp(BP->shape, BP->X[0], BP->X[1], BP->X[2], BP->fgfs[varl->data->sgfn], Shellf[j * num_var + k],
|
|
pox[0], pox[1], pox[2], ordn, varl->data->SoA, Symmetry);
|
|
|
|
varl = varl->next;
|
|
k++;
|
|
}
|
|
}
|
|
}
|
|
if (Bp == ble)
|
|
break;
|
|
Bp = Bp->next;
|
|
}
|
|
|
|
}
|
|
|
|
// Replace MPI_Allreduce with per-owner MPI_Bcast:
|
|
// Group consecutive points by owner rank and broadcast each group.
|
|
// Since each point's data is non-zero only on the owner rank,
|
|
// Bcast from owner is equivalent to Allreduce(MPI_SUM) but much cheaper.
|
|
{
|
|
int j = 0;
|
|
while (j < NN)
|
|
{
|
|
int cur_owner = owner_rank[j];
|
|
if (cur_owner < 0)
|
|
{
|
|
if (myrank == 0)
|
|
{
|
|
cout << "ERROR: Patch::Interp_Points fails to find point (";
|
|
for (int d = 0; d < dim; d++)
|
|
{
|
|
cout << XX[d][j];
|
|
if (d < dim - 1)
|
|
cout << ",";
|
|
else
|
|
cout << ")";
|
|
}
|
|
cout << " on Patch (";
|
|
for (int d = 0; d < dim; d++)
|
|
{
|
|
cout << bbox[d] << "+" << lli[d] * DH[d];
|
|
if (d < dim - 1)
|
|
cout << ",";
|
|
else
|
|
cout << ")--";
|
|
}
|
|
cout << "(";
|
|
for (int d = 0; d < dim; d++)
|
|
{
|
|
cout << bbox[dim + d] << "-" << uui[d] * DH[d];
|
|
if (d < dim - 1)
|
|
cout << ",";
|
|
else
|
|
cout << ")" << endl;
|
|
}
|
|
MPI_Abort(MPI_COMM_WORLD, 1);
|
|
}
|
|
j++;
|
|
continue;
|
|
}
|
|
// Find contiguous run of points with the same owner
|
|
int jstart = j;
|
|
while (j < NN && owner_rank[j] == cur_owner)
|
|
j++;
|
|
int count = (j - jstart) * num_var;
|
|
MPI_Bcast(Shellf + jstart * num_var, count, MPI_DOUBLE, cur_owner, MPI_COMM_WORLD);
|
|
}
|
|
}
|
|
|
|
delete[] owner_rank;
|
|
}
|
|
void Patch::Interp_Points(MyList<var> *VarList,
|
|
int NN, double **XX,
|
|
double *Shellf, int Symmetry,
|
|
int Nmin_consumer, int Nmax_consumer)
|
|
{
|
|
// Targeted point-to-point overload: each owner sends each point only to
|
|
// the one rank that needs it for integration (consumer), reducing
|
|
// communication volume by ~nprocs times compared to the Bcast version.
|
|
int myrank, nprocs;
|
|
MPI_Comm_rank(MPI_COMM_WORLD, &myrank);
|
|
MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
|
|
// printf("here----\n");
|
|
// int zzz = 0;
|
|
int ordn = 2 * ghost_width;
|
|
MyList<var> *varl;
|
|
int num_var = 0;
|
|
varl = VarList;
|
|
while (varl)
|
|
{
|
|
num_var++;
|
|
varl = varl->next;
|
|
}
|
|
|
|
memset(Shellf, 0, sizeof(double) * NN * num_var);
|
|
|
|
// owner_rank[j] records which MPI rank owns point j
|
|
int *owner_rank;
|
|
owner_rank = new int[NN];
|
|
for (int j = 0; j < NN; j++)
|
|
owner_rank[j] = -1;
|
|
|
|
double DH[dim];
|
|
for (int i = 0; i < dim; i++)
|
|
DH[i] = getdX(i);
|
|
|
|
// --- Interpolation phase (identical to original) ---
|
|
// printf("NN: %d, num_var = %d\n", NN, num_var);
|
|
#pragma omp parallel
|
|
{
|
|
#pragma omp for
|
|
for (int j = 0; j < NN; j++)
|
|
{
|
|
double pox[dim], llb[dim], uub[dim];
|
|
MyList<var> *varl1;
|
|
for (int i = 0; i < dim; i++)
|
|
{
|
|
pox[i] = XX[i][j];
|
|
// if (myrank == 0 && (XX[i][j] < bbox[i] + lli[i] * DH[i] || XX[i][j] > bbox[dim + i] - uui[i] * DH[i]))
|
|
// {
|
|
// cout << "Patch::Interp_Points: point (";
|
|
// for (int k = 0; k < dim; k++)
|
|
// {
|
|
// cout << XX[k][j];
|
|
// if (k < dim - 1)
|
|
// cout << ",";
|
|
// else
|
|
// cout << ") is out of current Patch." << endl;
|
|
// }
|
|
// MPI_Abort(MPI_COMM_WORLD, 1);
|
|
// }
|
|
}
|
|
|
|
MyList<Block> *Bp = blb;
|
|
bool notfind = true;
|
|
while (notfind && Bp)
|
|
{
|
|
Block *BP = Bp->data;
|
|
|
|
bool flag = true;
|
|
for (int i = 0; i < dim; i++)
|
|
{
|
|
#ifdef Vertex
|
|
#ifdef Cell
|
|
#error Both Cell and Vertex are defined
|
|
#endif
|
|
llb[i] = (feq(BP->bbox[i], bbox[i], DH[i] / 2)) ? BP->bbox[i] + lli[i] * DH[i] : BP->bbox[i] + (ghost_width - 0.5) * DH[i];
|
|
uub[i] = (feq(BP->bbox[dim + i], bbox[dim + i], DH[i] / 2)) ? BP->bbox[dim + i] - uui[i] * DH[i] : BP->bbox[dim + i] - (ghost_width - 0.5) * DH[i];
|
|
#else
|
|
#ifdef Cell
|
|
llb[i] = (feq(BP->bbox[i], bbox[i], DH[i] / 2)) ? BP->bbox[i] + lli[i] * DH[i] : BP->bbox[i] + ghost_width * DH[i];
|
|
uub[i] = (feq(BP->bbox[dim + i], bbox[dim + i], DH[i] / 2)) ? BP->bbox[dim + i] - uui[i] * DH[i] : BP->bbox[dim + i] - ghost_width * DH[i];
|
|
#else
|
|
#error Not define Vertex nor Cell
|
|
#endif
|
|
#endif
|
|
if (XX[i][j] - llb[i] < -DH[i] / 2 || XX[i][j] - uub[i] > DH[i] / 2)
|
|
{
|
|
flag = false;
|
|
break;
|
|
}
|
|
}
|
|
// printf("flag = %d\n", flag);
|
|
if (flag)
|
|
{
|
|
notfind = false;
|
|
owner_rank[j] = BP->rank;
|
|
if (myrank == BP->rank)
|
|
{
|
|
varl1 = VarList;
|
|
int k = 0;
|
|
while (varl1)
|
|
{
|
|
|
|
xh_global_interp(BP->shape, BP->X[0], BP->X[1], BP->X[2], BP->fgfs[varl1->data->sgfn], Shellf[j * num_var + k],
|
|
pox[0], pox[1], pox[2], ordn, varl1->data->SoA, Symmetry);
|
|
varl1 = varl1->next;
|
|
k++;
|
|
// zzz += 1;
|
|
}
|
|
}
|
|
}
|
|
if (Bp == ble)
|
|
break;
|
|
Bp = Bp->next;
|
|
}
|
|
}
|
|
}
|
|
// printf("Interpolation done, zzz = %d\n", zzz);
|
|
// --- Error check for unfound points ---
|
|
for (int j = 0; j < NN; j++)
|
|
{
|
|
if (owner_rank[j] < 0 && myrank == 0)
|
|
{
|
|
cout << "ERROR: Patch::Interp_Points fails to find point (";
|
|
for (int d = 0; d < dim; d++)
|
|
{
|
|
cout << XX[d][j];
|
|
if (d < dim - 1)
|
|
cout << ",";
|
|
else
|
|
cout << ")";
|
|
}
|
|
cout << " on Patch (";
|
|
for (int d = 0; d < dim; d++)
|
|
{
|
|
cout << bbox[d] << "+" << lli[d] * DH[d];
|
|
if (d < dim - 1)
|
|
cout << ",";
|
|
else
|
|
cout << ")--";
|
|
}
|
|
cout << "(";
|
|
for (int d = 0; d < dim; d++)
|
|
{
|
|
cout << bbox[dim + d] << "-" << uui[d] * DH[d];
|
|
if (d < dim - 1)
|
|
cout << ",";
|
|
else
|
|
cout << ")" << endl;
|
|
}
|
|
MPI_Abort(MPI_COMM_WORLD, 1);
|
|
}
|
|
}
|
|
|
|
// --- Targeted point-to-point communication phase ---
|
|
// Compute consumer_rank[j] using the same deterministic formula as surface_integral
|
|
int *consumer_rank = new int[NN];
|
|
{
|
|
int mp = NN / nprocs;
|
|
int Lp = NN - nprocs * mp;
|
|
for (int j = 0; j < NN; j++)
|
|
{
|
|
if (j < Lp * (mp + 1))
|
|
consumer_rank[j] = j / (mp + 1);
|
|
else
|
|
consumer_rank[j] = Lp + (j - Lp * (mp + 1)) / mp;
|
|
}
|
|
}
|
|
|
|
// Count sends and recvs per rank
|
|
int *send_count = new int[nprocs];
|
|
int *recv_count = new int[nprocs];
|
|
memset(send_count, 0, sizeof(int) * nprocs);
|
|
memset(recv_count, 0, sizeof(int) * nprocs);
|
|
|
|
for (int j = 0; j < NN; j++)
|
|
{
|
|
int own = owner_rank[j];
|
|
int con = consumer_rank[j];
|
|
if (own == con)
|
|
continue; // local — no communication needed
|
|
if (own == myrank)
|
|
send_count[con]++;
|
|
if (con == myrank)
|
|
recv_count[own]++;
|
|
}
|
|
|
|
// Build send buffers: for each destination rank, pack (index, data) pairs
|
|
// Each entry: 1 int (point index j) + num_var doubles
|
|
int total_send = 0, total_recv = 0;
|
|
int *send_offset = new int[nprocs];
|
|
int *recv_offset = new int[nprocs];
|
|
for (int r = 0; r < nprocs; r++)
|
|
{
|
|
send_offset[r] = total_send;
|
|
total_send += send_count[r];
|
|
recv_offset[r] = total_recv;
|
|
total_recv += recv_count[r];
|
|
}
|
|
|
|
// Pack send buffers: each message contains (j, data[0..num_var-1]) per point
|
|
int stride = 1 + num_var; // 1 double for index + num_var doubles for data
|
|
double *sendbuf = new double[total_send * stride];
|
|
double *recvbuf = new double[total_recv * stride];
|
|
|
|
// Temporary counters for packing
|
|
int *pack_pos = new int[nprocs];
|
|
memset(pack_pos, 0, sizeof(int) * nprocs);
|
|
|
|
for (int j = 0; j < NN; j++)
|
|
{
|
|
int own = owner_rank[j];
|
|
int con = consumer_rank[j];
|
|
if (own != myrank || con == myrank)
|
|
continue;
|
|
int pos = (send_offset[con] + pack_pos[con]) * stride;
|
|
sendbuf[pos] = (double)j; // point index
|
|
for (int v = 0; v < num_var; v++)
|
|
sendbuf[pos + 1 + v] = Shellf[j * num_var + v];
|
|
pack_pos[con]++;
|
|
}
|
|
|
|
// Post non-blocking recvs and sends
|
|
int n_req = 0;
|
|
for (int r = 0; r < nprocs; r++)
|
|
{
|
|
if (recv_count[r] > 0) n_req++;
|
|
if (send_count[r] > 0) n_req++;
|
|
}
|
|
|
|
MPI_Request *reqs = new MPI_Request[n_req];
|
|
int req_idx = 0;
|
|
|
|
for (int r = 0; r < nprocs; r++)
|
|
{
|
|
if (recv_count[r] > 0)
|
|
{
|
|
MPI_Irecv(recvbuf + recv_offset[r] * stride,
|
|
recv_count[r] * stride, MPI_DOUBLE,
|
|
r, 0, MPI_COMM_WORLD, &reqs[req_idx++]);
|
|
}
|
|
}
|
|
for (int r = 0; r < nprocs; r++)
|
|
{
|
|
if (send_count[r] > 0)
|
|
{
|
|
MPI_Isend(sendbuf + send_offset[r] * stride,
|
|
send_count[r] * stride, MPI_DOUBLE,
|
|
r, 0, MPI_COMM_WORLD, &reqs[req_idx++]);
|
|
}
|
|
}
|
|
|
|
if (n_req > 0)
|
|
MPI_Waitall(n_req, reqs, MPI_STATUSES_IGNORE);
|
|
|
|
// Unpack recv buffers into Shellf
|
|
for (int i = 0; i < total_recv; i++)
|
|
{
|
|
int pos = i * stride;
|
|
int j = (int)recvbuf[pos];
|
|
for (int v = 0; v < num_var; v++)
|
|
Shellf[j * num_var + v] = recvbuf[pos + 1 + v];
|
|
}
|
|
|
|
delete[] reqs;
|
|
delete[] sendbuf;
|
|
delete[] recvbuf;
|
|
delete[] pack_pos;
|
|
delete[] send_offset;
|
|
delete[] recv_offset;
|
|
delete[] send_count;
|
|
delete[] recv_count;
|
|
delete[] consumer_rank;
|
|
delete[] owner_rank;
|
|
}
|
|
void Patch::Interp_Points(MyList<var> *VarList,
|
|
int NN, double **XX,
|
|
double *Shellf, int Symmetry, MPI_Comm Comm_here)
|
|
{
|
|
// NOTE: we do not Synchnize variables here, make sure of that before calling this routine
|
|
int myrank, lmyrank;
|
|
MPI_Comm_rank(MPI_COMM_WORLD, &myrank);
|
|
MPI_Comm_rank(Comm_here, &lmyrank);
|
|
int ordn = 2 * ghost_width;
|
|
MyList<var> *varl;
|
|
int num_var = 0;
|
|
varl = VarList;
|
|
while (varl)
|
|
{
|
|
num_var++;
|
|
varl = varl->next;
|
|
}
|
|
|
|
memset(Shellf, 0, sizeof(double) * NN * num_var);
|
|
|
|
// owner_rank[j] stores the global rank that owns point j
|
|
int *owner_rank;
|
|
owner_rank = new int[NN];
|
|
for (int j = 0; j < NN; j++)
|
|
owner_rank[j] = -1;
|
|
|
|
// Build global-to-local rank translation for Comm_here
|
|
MPI_Group world_group, local_group;
|
|
MPI_Comm_group(MPI_COMM_WORLD, &world_group);
|
|
MPI_Comm_group(Comm_here, &local_group);
|
|
|
|
double DH[dim], llb[dim], uub[dim];
|
|
for (int i = 0; i < dim; i++)
|
|
DH[i] = getdX(i);
|
|
|
|
for (int j = 0; j < NN; j++) // run along points
|
|
{
|
|
double pox[dim];
|
|
for (int i = 0; i < dim; i++)
|
|
{
|
|
pox[i] = XX[i][j];
|
|
if (lmyrank == 0 && (XX[i][j] < bbox[i] + lli[i] * DH[i] || XX[i][j] > bbox[dim + i] - uui[i] * DH[i]))
|
|
{
|
|
cout << "Patch::Interp_Points: point (";
|
|
for (int k = 0; k < dim; k++)
|
|
{
|
|
cout << XX[k][j];
|
|
if (k < dim - 1)
|
|
cout << ",";
|
|
else
|
|
cout << ") is out of current Patch." << endl;
|
|
}
|
|
MPI_Abort(MPI_COMM_WORLD, 1);
|
|
}
|
|
}
|
|
|
|
MyList<Block> *Bp = blb;
|
|
bool notfind = true;
|
|
while (notfind && Bp) // run along Blocks
|
|
{
|
|
Block *BP = Bp->data;
|
|
|
|
bool flag = true;
|
|
for (int i = 0; i < dim; i++)
|
|
{
|
|
#ifdef Vertex
|
|
#ifdef Cell
|
|
#error Both Cell and Vertex are defined
|
|
#endif
|
|
llb[i] = (feq(BP->bbox[i], bbox[i], DH[i] / 2)) ? BP->bbox[i] + lli[i] * DH[i] : BP->bbox[i] + (ghost_width - 0.5) * DH[i];
|
|
uub[i] = (feq(BP->bbox[dim + i], bbox[dim + i], DH[i] / 2)) ? BP->bbox[dim + i] - uui[i] * DH[i] : BP->bbox[dim + i] - (ghost_width - 0.5) * DH[i];
|
|
#else
|
|
#ifdef Cell
|
|
llb[i] = (feq(BP->bbox[i], bbox[i], DH[i] / 2)) ? BP->bbox[i] + lli[i] * DH[i] : BP->bbox[i] + ghost_width * DH[i];
|
|
uub[i] = (feq(BP->bbox[dim + i], bbox[dim + i], DH[i] / 2)) ? BP->bbox[dim + i] - uui[i] * DH[i] : BP->bbox[dim + i] - ghost_width * DH[i];
|
|
#else
|
|
#error Not define Vertex nor Cell
|
|
#endif
|
|
#endif
|
|
if (XX[i][j] - llb[i] < -DH[i] / 2 || XX[i][j] - uub[i] > DH[i] / 2)
|
|
{
|
|
flag = false;
|
|
break;
|
|
}
|
|
}
|
|
|
|
if (flag)
|
|
{
|
|
notfind = false;
|
|
owner_rank[j] = BP->rank;
|
|
if (myrank == BP->rank)
|
|
{
|
|
//---> interpolation
|
|
varl = VarList;
|
|
int k = 0;
|
|
while (varl) // run along variables
|
|
{
|
|
xh_global_interp(BP->shape, BP->X[0], BP->X[1], BP->X[2], BP->fgfs[varl->data->sgfn], Shellf[j * num_var + k],
|
|
pox[0], pox[1], pox[2], ordn, varl->data->SoA, Symmetry);
|
|
varl = varl->next;
|
|
k++;
|
|
}
|
|
}
|
|
}
|
|
if (Bp == ble)
|
|
break;
|
|
Bp = Bp->next;
|
|
}
|
|
}
|
|
|
|
// Collect unique global owner ranks and translate to local ranks in Comm_here
|
|
// Then broadcast each owner's points via MPI_Bcast on Comm_here
|
|
{
|
|
int j = 0;
|
|
while (j < NN)
|
|
{
|
|
int cur_owner_global = owner_rank[j];
|
|
if (cur_owner_global < 0)
|
|
{
|
|
// Point not found — skip (error check disabled for sub-communicator levels)
|
|
j++;
|
|
continue;
|
|
}
|
|
// Translate global rank to local rank in Comm_here
|
|
int cur_owner_local;
|
|
MPI_Group_translate_ranks(world_group, 1, &cur_owner_global, local_group, &cur_owner_local);
|
|
|
|
// Find contiguous run of points with the same owner
|
|
int jstart = j;
|
|
while (j < NN && owner_rank[j] == cur_owner_global)
|
|
j++;
|
|
int count = (j - jstart) * num_var;
|
|
MPI_Bcast(Shellf + jstart * num_var, count, MPI_DOUBLE, cur_owner_local, Comm_here);
|
|
}
|
|
}
|
|
|
|
MPI_Group_free(&world_group);
|
|
MPI_Group_free(&local_group);
|
|
delete[] owner_rank;
|
|
}
|
|
void Patch::checkBlock()
|
|
{
|
|
int myrank;
|
|
MPI_Comm_rank(MPI_COMM_WORLD, &myrank);
|
|
if (myrank == 0)
|
|
{
|
|
MyList<Block> *BP = blb;
|
|
while (BP)
|
|
{
|
|
BP->data->checkBlock();
|
|
if (BP == ble)
|
|
break;
|
|
BP = BP->next;
|
|
}
|
|
}
|
|
}
|
|
double Patch::getdX(int dir)
|
|
{
|
|
if (dir < 0 || dir >= dim)
|
|
{
|
|
cout << "Patch::getdX: error input dir = " << dir << ", this Patch has direction (0," << dim - 1 << ")" << endl;
|
|
MPI_Abort(MPI_COMM_WORLD, 1);
|
|
}
|
|
double h;
|
|
#ifdef Vertex
|
|
#ifdef Cell
|
|
#error Both Cell and Vertex are defined
|
|
#endif
|
|
if (shape[dir] == 1)
|
|
{
|
|
cout << "Patch::getdX: for direction " << dir << ", this Patch has only one point. Can not determine dX for vertex center grid." << endl;
|
|
MPI_Abort(MPI_COMM_WORLD, 1);
|
|
}
|
|
h = (bbox[dim + dir] - bbox[dir]) / (shape[dir] - 1);
|
|
#else
|
|
#ifdef Cell
|
|
h = (bbox[dim + dir] - bbox[dir]) / shape[dir];
|
|
#else
|
|
#error Not define Vertex nor Cell
|
|
#endif
|
|
#endif
|
|
return h;
|
|
}
|
|
bool Patch::Interp_ONE_Point(MyList<var> *VarList, double *XX,
|
|
double *Shellf, int Symmetry)
|
|
{
|
|
// NOTE: we do not Synchnize variables here, make sure of that before calling this routine
|
|
int myrank;
|
|
MPI_Comm_rank(MPI_COMM_WORLD, &myrank);
|
|
|
|
int ordn = 2 * ghost_width;
|
|
MyList<var> *varl;
|
|
int num_var = 0;
|
|
varl = VarList;
|
|
while (varl)
|
|
{
|
|
num_var++;
|
|
varl = varl->next;
|
|
}
|
|
|
|
double *shellf;
|
|
shellf = new double[num_var];
|
|
memset(shellf, 0, sizeof(double) * num_var);
|
|
|
|
double *DH, *llb, *uub;
|
|
DH = new double[dim];
|
|
|
|
for (int i = 0; i < dim; i++)
|
|
{
|
|
DH[i] = getdX(i);
|
|
}
|
|
llb = new double[dim];
|
|
uub = new double[dim];
|
|
|
|
double pox[dim];
|
|
for (int i = 0; i < dim; i++)
|
|
{
|
|
pox[i] = XX[i];
|
|
// has excluded the buffer points
|
|
if (XX[i] < bbox[i] + lli[i] * DH[i] - DH[i] / 100 || XX[i] > bbox[dim + i] - uui[i] * DH[i] + DH[i] / 100)
|
|
{
|
|
delete[] shellf;
|
|
delete[] DH;
|
|
delete[] llb;
|
|
delete[] uub;
|
|
return false; // out of current patch,
|
|
// remember to delete the allocated arrays before return!!!
|
|
}
|
|
}
|
|
|
|
MyList<Block> *Bp = blb;
|
|
bool notfind = true;
|
|
while (notfind && Bp) // run along Blocks
|
|
{
|
|
Block *BP = Bp->data;
|
|
|
|
bool flag = true;
|
|
for (int i = 0; i < dim; i++)
|
|
{
|
|
// NOTE: our dividing structure is (exclude ghost)
|
|
// -1 0
|
|
// 1 2
|
|
// so (0,1) does not belong to any part for vertex structure
|
|
// here we put (0,0.5) to left part and (0.5,1) to right part
|
|
// BUT for cell structure the bbox is (-1.5,0.5) and (0.5,2.5), there is no missing region at all
|
|
#ifdef Vertex
|
|
#ifdef Cell
|
|
#error Both Cell and Vertex are defined
|
|
#endif
|
|
llb[i] = (feq(BP->bbox[i], bbox[i], DH[i] / 2)) ? BP->bbox[i] + lli[i] * DH[i] : BP->bbox[i] + (ghost_width - 0.5) * DH[i];
|
|
uub[i] = (feq(BP->bbox[dim + i], bbox[dim + i], DH[i] / 2)) ? BP->bbox[dim + i] - uui[i] * DH[i] : BP->bbox[dim + i] - (ghost_width - 0.5) * DH[i];
|
|
#else
|
|
#ifdef Cell
|
|
llb[i] = (feq(BP->bbox[i], bbox[i], DH[i] / 2)) ? BP->bbox[i] + lli[i] * DH[i] : BP->bbox[i] + ghost_width * DH[i];
|
|
uub[i] = (feq(BP->bbox[dim + i], bbox[dim + i], DH[i] / 2)) ? BP->bbox[dim + i] - uui[i] * DH[i] : BP->bbox[dim + i] - ghost_width * DH[i];
|
|
#else
|
|
#error Not define Vertex nor Cell
|
|
#endif
|
|
#endif
|
|
if (XX[i] - llb[i] < -DH[i] / 2 || XX[i] - uub[i] > DH[i] / 2)
|
|
{
|
|
flag = false;
|
|
break;
|
|
}
|
|
}
|
|
|
|
if (flag)
|
|
{
|
|
notfind = false;
|
|
if (myrank == BP->rank)
|
|
{
|
|
// test old code
|
|
#if 0
|
|
#define floorint(a) ((a) < 0 ? int(a) - 1 : int(a))
|
|
//---> interpolation
|
|
int ixl,iyl,izl,ixu,iyu,izu;
|
|
double Delx,Dely,Delz;
|
|
|
|
ixl = 1+floorint((pox[0]-BP->X[0][0])/DH[0]);
|
|
iyl = 1+floorint((pox[1]-BP->X[1][0])/DH[1]);
|
|
izl = 1+floorint((pox[2]-BP->X[2][0])/DH[2]);
|
|
|
|
int nn=ordn/2;
|
|
|
|
ixl = ixl-nn;
|
|
iyl = iyl-nn;
|
|
izl = izl-nn;
|
|
|
|
int tmi;
|
|
tmi = (Symmetry==2)?-1:0;
|
|
if(ixl<tmi) ixl=tmi;
|
|
if(iyl<tmi) iyl=tmi;
|
|
tmi = (Symmetry>0)?-1:0;
|
|
if(izl<tmi) izl=tmi;
|
|
|
|
if(ixl+ordn>BP->shape[0]) ixl=BP->shape[0]-ordn;
|
|
if(iyl+ordn>BP->shape[1]) iyl=BP->shape[1]-ordn;
|
|
if(izl+ordn>BP->shape[2]) izl=BP->shape[2]-ordn;
|
|
// support cell center
|
|
if(ixl>=0) Delx = ( pox[0] - BP->X[0][ixl] )/ DH[0];
|
|
else Delx = ( pox[0] + BP->X[0][0] )/ DH[0];
|
|
if(iyl>=0) Dely = ( pox[1] - BP->X[1][iyl] )/ DH[1];
|
|
else Dely = ( pox[1] + BP->X[1][0] )/ DH[1];
|
|
if(izl>=0) Delz = ( pox[2] - BP->X[2][izl] )/ DH[2];
|
|
else Delz = ( pox[2] + BP->X[2][0] )/ DH[2];
|
|
//change to fortran index
|
|
ixl++;
|
|
iyl++;
|
|
izl++;
|
|
ixu = ixl + ordn - 1;
|
|
iyu = iyl + ordn - 1;
|
|
izu = izl + ordn - 1;
|
|
varl=VarList;
|
|
int j=0;
|
|
while(varl)
|
|
{
|
|
f_interp_2(BP->shape,BP->fgfs[varl->data->sgfn],shellf[j],ixl,ixu,iyl,iyu,izl,izu,Delx,Dely,Delz,
|
|
ordn,varl->data->SoA,Symmetry);
|
|
varl=varl->next;
|
|
j++;
|
|
} //varl
|
|
#else
|
|
//---> interpolation
|
|
varl = VarList;
|
|
int k = 0;
|
|
while (varl) // run along variables
|
|
{
|
|
// shellf[j*num_var+k] = Parallel::global_interp(dim,BP->shape,BP->X,BP->fgfs[varl->data->sgfn],
|
|
// pox,ordn,varl->data->SoA,Symmetry);
|
|
xh_global_interp(BP->shape, BP->X[0], BP->X[1], BP->X[2], BP->fgfs[varl->data->sgfn], shellf[k],
|
|
pox[0], pox[1], pox[2], ordn, varl->data->SoA, Symmetry);
|
|
varl = varl->next;
|
|
k++;
|
|
}
|
|
#endif
|
|
}
|
|
}
|
|
if (Bp == ble)
|
|
break;
|
|
Bp = Bp->next;
|
|
}
|
|
|
|
if (notfind && myrank == 0)
|
|
{
|
|
cout << "ERROR: Patch::Interp_Points fails to find point (";
|
|
for (int j = 0; j < dim; j++)
|
|
{
|
|
cout << XX[j];
|
|
if (j < dim - 1)
|
|
cout << ",";
|
|
else
|
|
cout << ")";
|
|
}
|
|
cout << " on Patch (";
|
|
for (int j = 0; j < dim; j++)
|
|
{
|
|
cout << bbox[j] << "+" << lli[j] * getdX(j);
|
|
if (j < dim - 1)
|
|
cout << ",";
|
|
else
|
|
cout << ")--";
|
|
}
|
|
cout << "(";
|
|
for (int j = 0; j < dim; j++)
|
|
{
|
|
cout << bbox[dim + j] << "-" << uui[j] * getdX(j);
|
|
if (j < dim - 1)
|
|
cout << ",";
|
|
else
|
|
cout << ")" << endl;
|
|
}
|
|
#if 0
|
|
checkBlock();
|
|
#else
|
|
cout << "splited domains:" << endl;
|
|
{
|
|
MyList<Block> *Bp = blb;
|
|
while (Bp)
|
|
{
|
|
Block *BP = Bp->data;
|
|
|
|
for (int i = 0; i < dim; i++)
|
|
{
|
|
#ifdef Vertex
|
|
#ifdef Cell
|
|
#error Both Cell and Vertex are defined
|
|
#endif
|
|
llb[i] = (feq(BP->bbox[i], bbox[i], DH[i] / 2)) ? BP->bbox[i] + lli[i] * DH[i] : BP->bbox[i] + (ghost_width - 0.5) * DH[i];
|
|
uub[i] = (feq(BP->bbox[dim + i], bbox[dim + i], DH[i] / 2)) ? BP->bbox[dim + i] - uui[i] * DH[i] : BP->bbox[dim + i] - (ghost_width - 0.5) * DH[i];
|
|
#else
|
|
#ifdef Cell
|
|
llb[i] = (feq(BP->bbox[i], bbox[i], DH[i] / 2)) ? BP->bbox[i] + lli[i] * DH[i] : BP->bbox[i] + ghost_width * DH[i];
|
|
uub[i] = (feq(BP->bbox[dim + i], bbox[dim + i], DH[i] / 2)) ? BP->bbox[dim + i] - uui[i] * DH[i] : BP->bbox[dim + i] - ghost_width * DH[i];
|
|
#else
|
|
#error Not define Vertex nor Cell
|
|
#endif
|
|
#endif
|
|
}
|
|
cout << "(";
|
|
for (int j = 0; j < dim; j++)
|
|
{
|
|
cout << llb[j] << ":" << uub[j];
|
|
if (j < dim - 1)
|
|
cout << ",";
|
|
else
|
|
cout << ")" << endl;
|
|
}
|
|
if (Bp == ble)
|
|
break;
|
|
Bp = Bp->next;
|
|
}
|
|
}
|
|
#endif
|
|
MPI_Abort(MPI_COMM_WORLD, 1);
|
|
}
|
|
|
|
MPI_Allreduce(shellf, Shellf, num_var, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD);
|
|
|
|
delete[] shellf;
|
|
delete[] DH;
|
|
delete[] llb;
|
|
delete[] uub;
|
|
|
|
return true;
|
|
}
|
|
bool Patch::Interp_ONE_Point(MyList<var> *VarList, double *XX,
|
|
double *Shellf, int Symmetry, MPI_Comm Comm_here)
|
|
{
|
|
// NOTE: we do not Synchnize variables here, make sure of that before calling this routine
|
|
int myrank;
|
|
MPI_Comm_rank(MPI_COMM_WORLD, &myrank);
|
|
|
|
int ordn = 2 * ghost_width;
|
|
MyList<var> *varl;
|
|
int num_var = 0;
|
|
varl = VarList;
|
|
while (varl)
|
|
{
|
|
num_var++;
|
|
varl = varl->next;
|
|
}
|
|
|
|
double *shellf;
|
|
shellf = new double[num_var];
|
|
memset(shellf, 0, sizeof(double) * num_var);
|
|
|
|
double *DH, *llb, *uub;
|
|
DH = new double[dim];
|
|
|
|
for (int i = 0; i < dim; i++)
|
|
{
|
|
DH[i] = getdX(i);
|
|
}
|
|
llb = new double[dim];
|
|
uub = new double[dim];
|
|
|
|
double pox[dim];
|
|
for (int i = 0; i < dim; i++)
|
|
{
|
|
pox[i] = XX[i];
|
|
// has excluded the buffer points
|
|
if (XX[i] < bbox[i] + lli[i] * DH[i] - DH[i] / 100 || XX[i] > bbox[dim + i] - uui[i] * DH[i] + DH[i] / 100)
|
|
{
|
|
delete[] shellf;
|
|
delete[] DH;
|
|
delete[] llb;
|
|
delete[] uub;
|
|
return false; // out of current patch,
|
|
// remember to delete the allocated arrays before return!!!
|
|
}
|
|
}
|
|
|
|
MyList<Block> *Bp = blb;
|
|
bool notfind = true;
|
|
while (notfind && Bp) // run along Blocks
|
|
{
|
|
Block *BP = Bp->data;
|
|
|
|
bool flag = true;
|
|
for (int i = 0; i < dim; i++)
|
|
{
|
|
// NOTE: our dividing structure is (exclude ghost)
|
|
// -1 0
|
|
// 1 2
|
|
// so (0,1) does not belong to any part for vertex structure
|
|
// here we put (0,0.5) to left part and (0.5,1) to right part
|
|
// BUT for cell structure the bbox is (-1.5,0.5) and (0.5,2.5), there is no missing region at all
|
|
#ifdef Vertex
|
|
#ifdef Cell
|
|
#error Both Cell and Vertex are defined
|
|
#endif
|
|
llb[i] = (feq(BP->bbox[i], bbox[i], DH[i] / 2)) ? BP->bbox[i] + lli[i] * DH[i] : BP->bbox[i] + (ghost_width - 0.5) * DH[i];
|
|
uub[i] = (feq(BP->bbox[dim + i], bbox[dim + i], DH[i] / 2)) ? BP->bbox[dim + i] - uui[i] * DH[i] : BP->bbox[dim + i] - (ghost_width - 0.5) * DH[i];
|
|
#else
|
|
#ifdef Cell
|
|
llb[i] = (feq(BP->bbox[i], bbox[i], DH[i] / 2)) ? BP->bbox[i] + lli[i] * DH[i] : BP->bbox[i] + ghost_width * DH[i];
|
|
uub[i] = (feq(BP->bbox[dim + i], bbox[dim + i], DH[i] / 2)) ? BP->bbox[dim + i] - uui[i] * DH[i] : BP->bbox[dim + i] - ghost_width * DH[i];
|
|
#else
|
|
#error Not define Vertex nor Cell
|
|
#endif
|
|
#endif
|
|
if (XX[i] - llb[i] < -DH[i] / 2 || XX[i] - uub[i] > DH[i] / 2)
|
|
{
|
|
flag = false;
|
|
break;
|
|
}
|
|
}
|
|
|
|
if (flag)
|
|
{
|
|
notfind = false;
|
|
if (myrank == BP->rank)
|
|
{
|
|
// test old code
|
|
#if 0
|
|
#define floorint(a) ((a) < 0 ? int(a) - 1 : int(a))
|
|
//---> interpolation
|
|
int ixl,iyl,izl,ixu,iyu,izu;
|
|
double Delx,Dely,Delz;
|
|
|
|
ixl = 1+floorint((pox[0]-BP->X[0][0])/DH[0]);
|
|
iyl = 1+floorint((pox[1]-BP->X[1][0])/DH[1]);
|
|
izl = 1+floorint((pox[2]-BP->X[2][0])/DH[2]);
|
|
|
|
int nn=ordn/2;
|
|
|
|
ixl = ixl-nn;
|
|
iyl = iyl-nn;
|
|
izl = izl-nn;
|
|
|
|
int tmi;
|
|
tmi = (Symmetry==2)?-1:0;
|
|
if(ixl<tmi) ixl=tmi;
|
|
if(iyl<tmi) iyl=tmi;
|
|
tmi = (Symmetry>0)?-1:0;
|
|
if(izl<tmi) izl=tmi;
|
|
|
|
if(ixl+ordn>BP->shape[0]) ixl=BP->shape[0]-ordn;
|
|
if(iyl+ordn>BP->shape[1]) iyl=BP->shape[1]-ordn;
|
|
if(izl+ordn>BP->shape[2]) izl=BP->shape[2]-ordn;
|
|
// support cell center
|
|
if(ixl>=0) Delx = ( pox[0] - BP->X[0][ixl] )/ DH[0];
|
|
else Delx = ( pox[0] + BP->X[0][0] )/ DH[0];
|
|
if(iyl>=0) Dely = ( pox[1] - BP->X[1][iyl] )/ DH[1];
|
|
else Dely = ( pox[1] + BP->X[1][0] )/ DH[1];
|
|
if(izl>=0) Delz = ( pox[2] - BP->X[2][izl] )/ DH[2];
|
|
else Delz = ( pox[2] + BP->X[2][0] )/ DH[2];
|
|
//change to fortran index
|
|
ixl++;
|
|
iyl++;
|
|
izl++;
|
|
ixu = ixl + ordn - 1;
|
|
iyu = iyl + ordn - 1;
|
|
izu = izl + ordn - 1;
|
|
varl=VarList;
|
|
int j=0;
|
|
while(varl)
|
|
{
|
|
f_interp_2(BP->shape,BP->fgfs[varl->data->sgfn],shellf[j],ixl,ixu,iyl,iyu,izl,izu,Delx,Dely,Delz,
|
|
ordn,varl->data->SoA,Symmetry);
|
|
varl=varl->next;
|
|
j++;
|
|
} //varl
|
|
#else
|
|
//---> interpolation
|
|
varl = VarList;
|
|
int k = 0;
|
|
while (varl) // run along variables
|
|
{
|
|
// shellf[j*num_var+k] = Parallel::global_interp(dim,BP->shape,BP->X,BP->fgfs[varl->data->sgfn],
|
|
// pox,ordn,varl->data->SoA,Symmetry);
|
|
xh_global_interp(BP->shape, BP->X[0], BP->X[1], BP->X[2], BP->fgfs[varl->data->sgfn], shellf[k],
|
|
pox[0], pox[1], pox[2], ordn, varl->data->SoA, Symmetry);
|
|
varl = varl->next;
|
|
k++;
|
|
}
|
|
#endif
|
|
}
|
|
}
|
|
if (Bp == ble)
|
|
break;
|
|
Bp = Bp->next;
|
|
}
|
|
|
|
if (notfind && myrank == 0)
|
|
{
|
|
cout << "ERROR: Patch::Interp_Points fails to find point (";
|
|
for (int j = 0; j < dim; j++)
|
|
{
|
|
cout << XX[j];
|
|
if (j < dim - 1)
|
|
cout << ",";
|
|
else
|
|
cout << ")";
|
|
}
|
|
cout << " on Patch (";
|
|
for (int j = 0; j < dim; j++)
|
|
{
|
|
cout << bbox[j] << "+" << lli[j] * getdX(j);
|
|
if (j < dim - 1)
|
|
cout << ",";
|
|
else
|
|
cout << ")--";
|
|
}
|
|
cout << "(";
|
|
for (int j = 0; j < dim; j++)
|
|
{
|
|
cout << bbox[dim + j] << "-" << uui[j] * getdX(j);
|
|
if (j < dim - 1)
|
|
cout << ",";
|
|
else
|
|
cout << ")" << endl;
|
|
}
|
|
#if 0
|
|
checkBlock();
|
|
#else
|
|
cout << "splited domains:" << endl;
|
|
{
|
|
MyList<Block> *Bp = blb;
|
|
while (Bp)
|
|
{
|
|
Block *BP = Bp->data;
|
|
|
|
for (int i = 0; i < dim; i++)
|
|
{
|
|
#ifdef Vertex
|
|
#ifdef Cell
|
|
#error Both Cell and Vertex are defined
|
|
#endif
|
|
llb[i] = (feq(BP->bbox[i], bbox[i], DH[i] / 2)) ? BP->bbox[i] + lli[i] * DH[i] : BP->bbox[i] + (ghost_width - 0.5) * DH[i];
|
|
uub[i] = (feq(BP->bbox[dim + i], bbox[dim + i], DH[i] / 2)) ? BP->bbox[dim + i] - uui[i] * DH[i] : BP->bbox[dim + i] - (ghost_width - 0.5) * DH[i];
|
|
#else
|
|
#ifdef Cell
|
|
llb[i] = (feq(BP->bbox[i], bbox[i], DH[i] / 2)) ? BP->bbox[i] + lli[i] * DH[i] : BP->bbox[i] + ghost_width * DH[i];
|
|
uub[i] = (feq(BP->bbox[dim + i], bbox[dim + i], DH[i] / 2)) ? BP->bbox[dim + i] - uui[i] * DH[i] : BP->bbox[dim + i] - ghost_width * DH[i];
|
|
#else
|
|
#error Not define Vertex nor Cell
|
|
#endif
|
|
#endif
|
|
}
|
|
cout << "(";
|
|
for (int j = 0; j < dim; j++)
|
|
{
|
|
cout << llb[j] << ":" << uub[j];
|
|
if (j < dim - 1)
|
|
cout << ",";
|
|
else
|
|
cout << ")" << endl;
|
|
}
|
|
if (Bp == ble)
|
|
break;
|
|
Bp = Bp->next;
|
|
}
|
|
}
|
|
#endif
|
|
MPI_Abort(MPI_COMM_WORLD, 1);
|
|
}
|
|
|
|
MPI_Allreduce(shellf, Shellf, num_var, MPI_DOUBLE, MPI_SUM, Comm_here);
|
|
|
|
delete[] shellf;
|
|
delete[] DH;
|
|
delete[] llb;
|
|
delete[] uub;
|
|
|
|
return true;
|
|
}
|
|
// find maximum of abstract value, XX store position for maximum, Shellf store maximum themselvs
|
|
void Patch::Find_Maximum(MyList<var> *VarList, double *XX,
|
|
double *Shellf)
|
|
{
|
|
int myrank;
|
|
MPI_Comm_rank(MPI_COMM_WORLD, &myrank);
|
|
|
|
MyList<var> *varl;
|
|
int num_var = 0;
|
|
varl = VarList;
|
|
while (varl)
|
|
{
|
|
num_var++;
|
|
varl = varl->next;
|
|
}
|
|
|
|
double *shellf, *xx;
|
|
shellf = new double[num_var];
|
|
xx = new double[dim * num_var];
|
|
memset(shellf, 0, sizeof(double) * num_var);
|
|
memset(xx, 0, sizeof(double) * dim * num_var);
|
|
|
|
double *DH;
|
|
int *llb, *uub;
|
|
DH = new double[dim];
|
|
|
|
for (int i = 0; i < dim; i++)
|
|
{
|
|
DH[i] = getdX(i);
|
|
}
|
|
|
|
llb = new int[dim];
|
|
uub = new int[dim];
|
|
|
|
MyList<Block> *Bp = blb;
|
|
while (Bp) // run along Blocks
|
|
{
|
|
Block *BP = Bp->data;
|
|
|
|
if (myrank == BP->rank)
|
|
{
|
|
|
|
for (int i = 0; i < dim; i++)
|
|
{
|
|
llb[i] = (feq(BP->bbox[i], bbox[i], DH[i] / 2)) ? lli[i] : ghost_width;
|
|
uub[i] = (feq(BP->bbox[dim + i], bbox[dim + i], DH[i] / 2)) ? uui[i] : ghost_width;
|
|
}
|
|
|
|
varl = VarList;
|
|
int k = 0;
|
|
double tmp, tmpx[dim];
|
|
while (varl) // run along variables
|
|
{
|
|
f_find_maximum(BP->shape, BP->X[0], BP->X[1], BP->X[2], BP->fgfs[varl->data->sgfn], tmp, tmpx, llb, uub);
|
|
if (tmp > shellf[k])
|
|
{
|
|
shellf[k] = tmp;
|
|
for (int i = 0; i < dim; i++)
|
|
xx[dim * k + i] = tmpx[i];
|
|
}
|
|
varl = varl->next;
|
|
k++;
|
|
}
|
|
}
|
|
|
|
if (Bp == ble)
|
|
break;
|
|
Bp = Bp->next;
|
|
}
|
|
|
|
struct mloc
|
|
{
|
|
double val;
|
|
int rank;
|
|
};
|
|
|
|
mloc *IN, *OUT;
|
|
IN = new mloc[num_var];
|
|
OUT = new mloc[num_var];
|
|
for (int i = 0; i < num_var; i++)
|
|
{
|
|
IN[i].val = shellf[i];
|
|
IN[i].rank = myrank;
|
|
}
|
|
|
|
MPI_Allreduce(IN, OUT, num_var, MPI_DOUBLE_INT, MPI_MAXLOC, MPI_COMM_WORLD);
|
|
|
|
for (int i = 0; i < num_var; i++)
|
|
{
|
|
Shellf[i] = OUT[i].val;
|
|
if (myrank != OUT[i].rank)
|
|
for (int k = 0; k < 3; k++)
|
|
xx[3 * i + k] = 0;
|
|
}
|
|
|
|
MPI_Allreduce(xx, XX, dim * num_var, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD);
|
|
|
|
delete[] IN;
|
|
delete[] OUT;
|
|
delete[] shellf;
|
|
delete[] xx;
|
|
delete[] DH;
|
|
delete[] llb;
|
|
delete[] uub;
|
|
}
|
|
void Patch::Find_Maximum(MyList<var> *VarList, double *XX,
|
|
double *Shellf, MPI_Comm Comm_here)
|
|
{
|
|
int myrank;
|
|
MPI_Comm_rank(MPI_COMM_WORLD, &myrank);
|
|
|
|
MyList<var> *varl;
|
|
int num_var = 0;
|
|
varl = VarList;
|
|
while (varl)
|
|
{
|
|
num_var++;
|
|
varl = varl->next;
|
|
}
|
|
|
|
double *shellf, *xx;
|
|
shellf = new double[num_var];
|
|
xx = new double[dim * num_var];
|
|
memset(shellf, 0, sizeof(double) * num_var);
|
|
memset(xx, 0, sizeof(double) * dim * num_var);
|
|
|
|
double *DH;
|
|
int *llb, *uub;
|
|
DH = new double[dim];
|
|
|
|
for (int i = 0; i < dim; i++)
|
|
{
|
|
DH[i] = getdX(i);
|
|
}
|
|
|
|
llb = new int[dim];
|
|
uub = new int[dim];
|
|
|
|
MyList<Block> *Bp = blb;
|
|
while (Bp) // run along Blocks
|
|
{
|
|
Block *BP = Bp->data;
|
|
|
|
if (myrank == BP->rank)
|
|
{
|
|
|
|
for (int i = 0; i < dim; i++)
|
|
{
|
|
llb[i] = (feq(BP->bbox[i], bbox[i], DH[i] / 2)) ? lli[i] : ghost_width;
|
|
uub[i] = (feq(BP->bbox[dim + i], bbox[dim + i], DH[i] / 2)) ? uui[i] : ghost_width;
|
|
}
|
|
|
|
varl = VarList;
|
|
int k = 0;
|
|
double tmp, tmpx[dim];
|
|
while (varl) // run along variables
|
|
{
|
|
f_find_maximum(BP->shape, BP->X[0], BP->X[1], BP->X[2], BP->fgfs[varl->data->sgfn], tmp, tmpx, llb, uub);
|
|
if (tmp > shellf[k])
|
|
{
|
|
shellf[k] = tmp;
|
|
for (int i = 0; i < dim; i++)
|
|
xx[dim * k + i] = tmpx[i];
|
|
}
|
|
varl = varl->next;
|
|
k++;
|
|
}
|
|
}
|
|
|
|
if (Bp == ble)
|
|
break;
|
|
Bp = Bp->next;
|
|
}
|
|
|
|
struct mloc
|
|
{
|
|
double val;
|
|
int rank;
|
|
};
|
|
|
|
mloc *IN, *OUT;
|
|
IN = new mloc[num_var];
|
|
OUT = new mloc[num_var];
|
|
for (int i = 0; i < num_var; i++)
|
|
{
|
|
IN[i].val = shellf[i];
|
|
IN[i].rank = myrank;
|
|
}
|
|
|
|
MPI_Allreduce(IN, OUT, num_var, MPI_DOUBLE_INT, MPI_MAXLOC, Comm_here);
|
|
|
|
for (int i = 0; i < num_var; i++)
|
|
{
|
|
Shellf[i] = OUT[i].val;
|
|
if (myrank != OUT[i].rank)
|
|
for (int k = 0; k < 3; k++)
|
|
xx[3 * i + k] = 0;
|
|
}
|
|
|
|
MPI_Allreduce(xx, XX, dim * num_var, MPI_DOUBLE, MPI_SUM, Comm_here);
|
|
|
|
delete[] IN;
|
|
delete[] OUT;
|
|
delete[] shellf;
|
|
delete[] xx;
|
|
delete[] DH;
|
|
delete[] llb;
|
|
delete[] uub;
|
|
}
|
|
// if the given point locates in the present Patch return true
|
|
// otherwise return false
|
|
bool Patch::Find_Point(double *XX)
|
|
{
|
|
double *DH;
|
|
DH = new double[dim];
|
|
|
|
for (int i = 0; i < dim; i++)
|
|
{
|
|
DH[i] = getdX(i);
|
|
}
|
|
|
|
for (int i = 0; i < dim; i++)
|
|
{
|
|
// has excluded the buffer points
|
|
if (XX[i] < bbox[i] + lli[i] * DH[i] - DH[i] / 100 || XX[i] > bbox[dim + i] - uui[i] * DH[i] + DH[i] / 100)
|
|
{
|
|
delete[] DH;
|
|
return false; // out of current patch,
|
|
// remember to delete the allocated arrays before return!!!
|
|
}
|
|
}
|
|
|
|
delete[] DH;
|
|
|
|
return true;
|
|
}
|