28 |
{ |
{ |
29 |
index_t min_DOF, max_DOF, *shared=NULL, *offsetInShared=NULL, *locDOFMask=NULL, i, k, myFirstDOF, myLastDOF, *nodeMask=NULL, firstDOF, lastDOF, *globalDOFIndex, *wanted_DOFs=NULL; |
index_t min_DOF, max_DOF, *shared=NULL, *offsetInShared=NULL, *locDOFMask=NULL, i, k, myFirstDOF, myLastDOF, *nodeMask=NULL, firstDOF, lastDOF, *globalDOFIndex, *wanted_DOFs=NULL; |
30 |
dim_t mpiSize, len_loc_dof, numNeighbors, n, lastn, numNodes,*rcv_len=NULL, *snd_len=NULL, count; |
dim_t mpiSize, len_loc_dof, numNeighbors, n, lastn, numNodes,*rcv_len=NULL, *snd_len=NULL, count; |
31 |
Paso_MPI_rank myRank,p,p_min,p_max, *neighbor=NULL; |
Esys_MPI_rank myRank,p,p_min,p_max, *neighbor=NULL; |
32 |
Paso_SharedComponents *rcv_shcomp=NULL, *snd_shcomp=NULL; |
Paso_SharedComponents *rcv_shcomp=NULL, *snd_shcomp=NULL; |
33 |
Finley_NodeMapping *this_mapping=NULL; |
Finley_NodeMapping *this_mapping=NULL; |
34 |
Paso_Connector* this_connector=NULL; |
Paso_Connector* this_connector=NULL; |
35 |
Paso_Distribution* dof_distribution; |
Paso_Distribution* dof_distribution; |
36 |
Paso_MPIInfo *mpi_info = in->MPIInfo; |
Esys_MPIInfo *mpi_info = in->MPIInfo; |
37 |
#ifdef PASO_MPI |
#ifdef ESYS_MPI |
38 |
MPI_Request* mpi_requests=NULL; |
MPI_Request* mpi_requests=NULL; |
39 |
MPI_Status* mpi_stati=NULL; |
MPI_Status* mpi_stati=NULL; |
40 |
#else |
#else |
80 |
} |
} |
81 |
rcv_len=TMPMEMALLOC(mpiSize,dim_t); |
rcv_len=TMPMEMALLOC(mpiSize,dim_t); |
82 |
snd_len=TMPMEMALLOC(mpiSize,dim_t); |
snd_len=TMPMEMALLOC(mpiSize,dim_t); |
83 |
#ifdef PASO_MPI |
#ifdef ESYS_MPI |
84 |
mpi_requests=MEMALLOC(mpiSize*2,MPI_Request); |
mpi_requests=MEMALLOC(mpiSize*2,MPI_Request); |
85 |
mpi_stati=MEMALLOC(mpiSize*2,MPI_Status); |
mpi_stati=MEMALLOC(mpiSize*2,MPI_Status); |
86 |
#else |
#else |
89 |
#endif |
#endif |
90 |
wanted_DOFs=TMPMEMALLOC(numNodes,index_t); |
wanted_DOFs=TMPMEMALLOC(numNodes,index_t); |
91 |
nodeMask=TMPMEMALLOC(numNodes,index_t); |
nodeMask=TMPMEMALLOC(numNodes,index_t); |
92 |
neighbor=TMPMEMALLOC(mpiSize,Paso_MPI_rank); |
neighbor=TMPMEMALLOC(mpiSize,Esys_MPI_rank); |
93 |
shared=TMPMEMALLOC(numNodes*(p_max-p_min+1),index_t); |
shared=TMPMEMALLOC(numNodes*(p_max-p_min+1),index_t); |
94 |
offsetInShared=TMPMEMALLOC(mpiSize+1,index_t); |
offsetInShared=TMPMEMALLOC(mpiSize+1,index_t); |
95 |
locDOFMask=TMPMEMALLOC(len_loc_dof, index_t); |
locDOFMask=TMPMEMALLOC(len_loc_dof, index_t); |
181 |
/* |
/* |
182 |
* now we build the sender |
* now we build the sender |
183 |
*/ |
*/ |
184 |
#ifdef PASO_MPI |
#ifdef ESYS_MPI |
185 |
MPI_Alltoall(rcv_len,1,MPI_INT,snd_len,1,MPI_INT,mpi_info->comm); |
MPI_Alltoall(rcv_len,1,MPI_INT,snd_len,1,MPI_INT,mpi_info->comm); |
186 |
#else |
#else |
187 |
for (p=0;p<mpiSize;++p) snd_len[p]=rcv_len[p]; |
for (p=0;p<mpiSize;++p) snd_len[p]=rcv_len[p]; |
188 |
#endif |
#endif |
189 |
count=0; |
count=0; |
190 |
for (p=0;p<rcv_shcomp->numNeighbors;p++) { |
for (p=0;p<rcv_shcomp->numNeighbors;p++) { |
191 |
#ifdef PASO_MPI |
#ifdef ESYS_MPI |
192 |
MPI_Isend(&(wanted_DOFs[rcv_shcomp->offsetInShared[p]]), rcv_shcomp->offsetInShared[p+1]-rcv_shcomp->offsetInShared[p], |
MPI_Isend(&(wanted_DOFs[rcv_shcomp->offsetInShared[p]]), rcv_shcomp->offsetInShared[p+1]-rcv_shcomp->offsetInShared[p], |
193 |
MPI_INT,rcv_shcomp->neighbor[p],mpi_info->msg_tag_counter+myRank,mpi_info->comm,&mpi_requests[count]); |
MPI_INT,rcv_shcomp->neighbor[p],mpi_info->msg_tag_counter+myRank,mpi_info->comm,&mpi_requests[count]); |
194 |
#endif |
#endif |
198 |
numNeighbors=0; |
numNeighbors=0; |
199 |
for (p=0;p<mpiSize;p++) { |
for (p=0;p<mpiSize;p++) { |
200 |
if (snd_len[p] > 0) { |
if (snd_len[p] > 0) { |
201 |
#ifdef PASO_MPI |
#ifdef ESYS_MPI |
202 |
MPI_Irecv(&(shared[n]),snd_len[p], |
MPI_Irecv(&(shared[n]),snd_len[p], |
203 |
MPI_INT, p, mpi_info->msg_tag_counter+p, mpi_info->comm, &mpi_requests[count]); |
MPI_INT, p, mpi_info->msg_tag_counter+p, mpi_info->comm, &mpi_requests[count]); |
204 |
#endif |
#endif |
211 |
} |
} |
212 |
mpi_info->msg_tag_counter+=mpi_info->size; |
mpi_info->msg_tag_counter+=mpi_info->size; |
213 |
offsetInShared[numNeighbors]=n; |
offsetInShared[numNeighbors]=n; |
214 |
#ifdef PASO_MPI |
#ifdef ESYS_MPI |
215 |
MPI_Waitall(count,mpi_requests,mpi_stati); |
MPI_Waitall(count,mpi_requests,mpi_stati); |
216 |
#endif |
#endif |
217 |
/* map global ids to local id's */ |
/* map global ids to local id's */ |
279 |
index_t myFirstDOF, myLastDOF, myFirstNode, myLastNode, *reduced_dof_first_component=NULL, *nodeMask=NULL, |
index_t myFirstDOF, myLastDOF, myFirstNode, myLastNode, *reduced_dof_first_component=NULL, *nodeMask=NULL, |
280 |
*reduced_nodes_first_component=NULL, k,*maskMyReducedDOF=NULL, *indexMyReducedDOF=NULL, *maskMyReducedNodes=NULL, *indexMyReducedNodes=NULL; |
*reduced_nodes_first_component=NULL, k,*maskMyReducedDOF=NULL, *indexMyReducedDOF=NULL, *maskMyReducedNodes=NULL, *indexMyReducedNodes=NULL; |
281 |
dim_t myNumDOF, myNumNodes, myNumReducedNodes, myNumReducedDOF, globalNumReducedNodes, globalNumReducedDOF,i,mpiSize; |
dim_t myNumDOF, myNumNodes, myNumReducedNodes, myNumReducedDOF, globalNumReducedNodes, globalNumReducedDOF,i,mpiSize; |
282 |
Paso_MPI_rank myRank; |
Esys_MPI_rank myRank; |
283 |
|
|
284 |
mpiSize=in->Nodes->MPIInfo->size; |
mpiSize=in->Nodes->MPIInfo->size; |
285 |
myRank=in->Nodes->MPIInfo->rank; |
myRank=in->Nodes->MPIInfo->rank; |
325 |
myNumReducedNodes=Finley_Util_packMask(myNumNodes,maskMyReducedNodes,indexMyReducedNodes); |
myNumReducedNodes=Finley_Util_packMask(myNumNodes,maskMyReducedNodes,indexMyReducedNodes); |
326 |
myNumReducedDOF=Finley_Util_packMask(myNumDOF,maskMyReducedDOF,indexMyReducedDOF); |
myNumReducedDOF=Finley_Util_packMask(myNumDOF,maskMyReducedDOF,indexMyReducedDOF); |
327 |
|
|
328 |
#ifdef PASO_MPI |
#ifdef ESYS_MPI |
329 |
MPI_Allgather(&myNumReducedNodes,1,MPI_INT,reduced_nodes_first_component,1,MPI_INT,in->Nodes->MPIInfo->comm); |
MPI_Allgather(&myNumReducedNodes,1,MPI_INT,reduced_nodes_first_component,1,MPI_INT,in->Nodes->MPIInfo->comm); |
330 |
MPI_Allgather(&myNumReducedDOF,1,MPI_INT,reduced_dof_first_component,1,MPI_INT,in->Nodes->MPIInfo->comm); |
MPI_Allgather(&myNumReducedDOF,1,MPI_INT,reduced_dof_first_component,1,MPI_INT,in->Nodes->MPIInfo->comm); |
331 |
#else |
#else |