26 |
#ifdef _OPENMP |
#ifdef _OPENMP |
27 |
#include <omp.h> |
#include <omp.h> |
28 |
#endif |
#endif |
29 |
|
#ifdef PARMETIS |
30 |
|
#include "parmetis.h" |
31 |
|
#endif |
32 |
|
|
33 |
/**************************************************************/ |
/**************************************************************/ |
34 |
|
|
43 |
Paso_MPI_rank myRank,dest,source,current_rank, rank; |
Paso_MPI_rank myRank,dest,source,current_rank, rank; |
44 |
Finley_IndexList* index_list=NULL; |
Finley_IndexList* index_list=NULL; |
45 |
float *xyz=NULL; |
float *xyz=NULL; |
46 |
|
int c; |
47 |
|
|
48 |
#ifdef PASO_MPI |
#ifdef PASO_MPI |
49 |
MPI_Status status; |
MPI_Status status; |
71 |
newGlobalDOFID=TMPMEMALLOC(len,index_t); |
newGlobalDOFID=TMPMEMALLOC(len,index_t); |
72 |
setNewDOFId=TMPMEMALLOC(in->Nodes->numNodes,bool_t); |
setNewDOFId=TMPMEMALLOC(in->Nodes->numNodes,bool_t); |
73 |
if (!(Finley_checkPtr(partition) || Finley_checkPtr(xyz) || Finley_checkPtr(partition_count) || Finley_checkPtr(partition_count) || Finley_checkPtr(newGlobalDOFID) || Finley_checkPtr(setNewDOFId))) { |
if (!(Finley_checkPtr(partition) || Finley_checkPtr(xyz) || Finley_checkPtr(partition_count) || Finley_checkPtr(partition_count) || Finley_checkPtr(newGlobalDOFID) || Finley_checkPtr(setNewDOFId))) { |
74 |
|
dim_t *recvbuf=TMPMEMALLOC(mpiSize*mpiSize,dim_t); |
75 |
|
|
76 |
/* set the coordinates: *? |
/* set the coordinates: *? |
77 |
/* it is assumed that at least one node on this processor provides a coordinate */ |
/* it is assumed that at least one node on this processor provides a coordinate */ |
121 |
|
|
122 |
if (Finley_noError()) { |
if (Finley_noError()) { |
123 |
|
|
124 |
/* |
#ifdef PARMETIS |
125 |
|
if (in->MPIInfo->size>1) { |
126 |
ParMETIS_V3_PartGeomKway(distribution, |
int i; |
127 |
|
int wgtflag = 0; |
128 |
|
int numflag = 0; /* pattern->ptr is C style: starting from 0 instead of 1 */ |
129 |
|
int ncon = 1; |
130 |
|
int edgecut; |
131 |
|
int options[2]; |
132 |
|
float *tpwgts = TMPMEMALLOC(ncon*mpiSize,float); |
133 |
|
float *ubvec = TMPMEMALLOC(ncon,float); |
134 |
|
for (i=0; i<ncon*mpiSize; i++) tpwgts[i] = 1.0/(float)mpiSize; |
135 |
|
for (i=0; i<ncon; i++) ubvec[i] = 1.05; |
136 |
|
options[0] = 0; |
137 |
|
options[1] = 15; |
138 |
|
ParMETIS_V3_PartGeomKway(distribution, |
139 |
pattern->ptr, |
pattern->ptr, |
140 |
pattern->index, |
pattern->index, |
141 |
idxtype *vwgt, + |
NULL, |
142 |
idxtype *adjwgt, + |
NULL, |
143 |
int *wgtflag, + |
&wgtflag, |
144 |
int *numflag, + |
&numflag, |
145 |
dim, |
&dim, |
146 |
xyz, |
xyz, |
147 |
int *ncon, + |
&ncon, |
148 |
mpiSize, |
&mpiSize, |
149 |
float *tpwgts, + |
tpwgts, |
150 |
float *ubvec, + |
ubvec, |
151 |
int *options, + |
options, |
152 |
int *edgecut, + |
&edgecut, |
153 |
partition, |
partition, /* new CPU ownership of elements */ |
154 |
in->MPIInfo->comm); |
&(in->MPIInfo->comm)); |
155 |
*/ |
printf("ParMETIS number of edges cut by partitioning: %d\n", edgecut); |
156 |
for (i=0;i<myNumVertices;++i) partition[i]=myRank; /* remove */ |
TMPMEMFREE(ubvec); |
157 |
|
TMPMEMFREE(tpwgts); |
158 |
|
} else { |
159 |
|
for (i=0;i<myNumVertices;++i) partition[i]=0; /* CPU 0 owns it */ |
160 |
|
} |
161 |
|
#else |
162 |
|
for (i=0;i<myNumVertices;++i) partition[i]=myRank; /* CPU 0 owns it */ |
163 |
|
#endif |
164 |
|
|
165 |
} |
} |
166 |
|
|
167 |
Paso_Pattern_free(pattern); |
Paso_Pattern_free(pattern); |
181 |
THREAD_MEMFREE(loc_partition_count); |
THREAD_MEMFREE(loc_partition_count); |
182 |
} |
} |
183 |
#ifdef PASO_MPI |
#ifdef PASO_MPI |
184 |
MPI_Allreduce( new_distribution, partition_count, mpiSize, MPI_INT, MPI_SUM, in->MPIInfo->comm ); |
/* recvbuf will be the concatenation of each CPU's contribution to new_distribution */ |
185 |
|
MPI_Allgather(new_distribution, mpiSize, MPI_INT, recvbuf, mpiSize, MPI_INT, in->MPIInfo->comm); |
186 |
#else |
#else |
187 |
for (i=0;i<mpiSize;++i) partition_count[i]=new_distribution[i]; |
for (i=0;i<mpiSize;++i) recvbuf[i]=new_distribution[i]; |
188 |
#endif |
#endif |
189 |
new_distribution[0]=0; |
new_distribution[0]=0; |
190 |
for (i=0;i<mpiSize;++i) { |
for (rank=0; rank<mpiSize;rank++) { |
191 |
new_distribution[i+1]=new_distribution[i]+partition_count[i]; |
c=0; |
192 |
partition_count[i]=0; |
for (i=0;i<myRank;++i) c+=recvbuf[rank+mpiSize*i]; |
193 |
} |
for (i=0;i<myNumVertices;++i) { |
194 |
for (i=0;i<myNumVertices;++i) { |
if (rank==partition[i]) { |
195 |
rank=partition[i]; |
newGlobalDOFID[i]=new_distribution[rank]+c; |
196 |
newGlobalDOFID[i]=new_distribution[rank]+partition_count[rank]; |
c++; |
197 |
partition_count[rank]++; |
} |
198 |
|
} |
199 |
|
for (i=myRank+1;i<mpiSize;++i) c+=recvbuf[rank+mpiSize*i]; |
200 |
|
new_distribution[rank+1]=new_distribution[rank]+c; |
201 |
} |
} |
202 |
|
TMPMEMFREE(recvbuf); |
203 |
|
|
204 |
/* now the overlap needs to be created by sending the partition around*/ |
/* now the overlap needs to be created by sending the partition around*/ |
205 |
|
|
206 |
dest=Paso_MPIInfo_mod(mpiSize, myRank + 1); |
dest=Paso_MPIInfo_mod(mpiSize, myRank + 1); |