/[escript]/trunk-mpi-branch/finley/src/Mesh_createNodeFileMappings.c
ViewVC logotype

Contents of /trunk-mpi-branch/finley/src/Mesh_createNodeFileMappings.c

Parent Directory Parent Directory | Revision Log Revision Log


Revision 1268 - (show annotations)
Wed Aug 22 04:01:32 2007 UTC (12 years, 9 months ago) by gross
File MIME type: text/plain
File size: 14274 byte(s)
small bug fixed.
1 /*
2 ************************************************************
3 * Copyright 2007 by ACcESS MNRF *
4 * *
5 * http://www.access.edu.au *
6 * Primary Business: Queensland, Australia *
7 * Licensed under the Open Software License version 3.0 *
8 * http://www.opensource.org/licenses/osl-3.0.php *
9 * *
10 ************************************************************
11 */
12
13 /**************************************************************/
14
15 /* Finley: NodeFile : creates the mappings using the indexReducedNodes */
16 /* no distribution is happening */
17
18 /**************************************************************/
19
20 /* Author: gross@access.edu.au */
21 /* Version: $Id:$ */
22
23 /**************************************************************/
24
25 #include "Mesh.h"
26 #define UNUSED -1
27
28 /**************************************************************/
29
30 void Mesh_createDOFMappingAndCoupling(Finley_Mesh* in, bool_t use_reduced_elements)
31 {
32 index_t min_DOF, max_DOF, *shared=NULL, *offsetInShared=NULL, *locDOFMask=NULL, i, k, myFirstDOF, myLastDOF, *nodeMask=NULL, firstDOF, lastDOF, *globalDOFIndex;
33 dim_t mpiSize, len_loc_dof, numNeighbors, n, lastn, numNodes;
34 Paso_MPI_rank myRank,p,p_min,p_max, *neighbor=NULL;
35 Paso_SharedComponents *rcv_shcomp=NULL, *snd_shcomp=NULL;
36 Finley_NodeMapping *this_mapping=NULL;
37 Paso_Coupler* this_coupler=NULL;
38 Paso_Distribution* dof_distribution;
39
40 numNodes=in->Nodes->numNodes;
41 if (use_reduced_elements) {
42 dof_distribution=in->Nodes->reducedDegreesOfFreedomDistribution;
43 globalDOFIndex=in->Nodes->globalReducedDOFIndex;
44 Finley_NodeFile_setReducedDOFRange(&min_DOF, &max_DOF,in->Nodes);
45 } else {
46 dof_distribution=in->Nodes->degreesOfFreedomDistribution;
47 globalDOFIndex=in->Nodes->globalDegreesOfFreedom;
48 Finley_NodeFile_setDOFRange(&min_DOF, &max_DOF,in->Nodes);
49 }
50
51 mpiSize=dof_distribution->mpi_info->size;
52 myRank=dof_distribution->mpi_info->rank;
53
54 min_DOF=Finley_Util_getFlaggedMinInt(1,numNodes,globalDOFIndex,-1);
55 max_DOF=Finley_Util_getFlaggedMaxInt(1,numNodes,globalDOFIndex,-1);
56
57 len_loc_dof=max_DOF-min_DOF+1;
58
59 p_min=mpiSize;
60 p_max=-1;
61
62 for (p=0; p<mpiSize; ++p) {
63 if (dof_distribution->first_component[p]<=min_DOF) p_min=p;
64 if (dof_distribution->first_component[p]<=max_DOF) p_max=p;
65 }
66
67 myFirstDOF=Paso_Distribution_getFirstComponent(dof_distribution);
68 myLastDOF=Paso_Distribution_getLastComponent(dof_distribution);
69
70
71 nodeMask=TMPMEMALLOC(numNodes,index_t);
72 neighbor=TMPMEMALLOC(mpiSize,Paso_MPI_rank);
73 shared=TMPMEMALLOC(numNodes*(p_max-p_min+1),index_t);
74 offsetInShared=TMPMEMALLOC(mpiSize+1,index_t);
75 locDOFMask=TMPMEMALLOC(len_loc_dof, index_t);
76 if (! ( Finley_checkPtr(neighbor) || Finley_checkPtr(shared) || Finley_checkPtr(offsetInShared) || Finley_checkPtr(locDOFMask) || Finley_checkPtr(nodeMask) )) {
77
78
79 #pragma omp parallel for private(i) schedule(static)
80 for (i=0;i<len_loc_dof;++i) locDOFMask[i]=UNUSED;
81 #pragma omp parallel for private(i) schedule(static)
82 for (i=0;i<numNodes;++i) nodeMask[i]=UNUSED;
83
84 for (i=0;i<numNodes;++i) {
85 k=globalDOFIndex[i];
86 if (k>-1) locDOFMask[k-min_DOF]=UNUSED-1;
87 }
88 for (i=myFirstDOF-min_DOF;i<myLastDOF-min_DOF;++i) locDOFMask[i]=i-myFirstDOF+min_DOF;
89
90 numNeighbors=0;
91 n=0;
92 lastn=n;
93 for (p=p_min;p<=p_max;++p) {
94 firstDOF=dof_distribution->first_component[p];
95 lastDOF=dof_distribution->first_component[p+1];
96 if (p != myRank) {
97 for (i=firstDOF-min_DOF;i<lastDOF-min_DOF;++i) {
98 if (locDOFMask[i] == UNUSED-1) {
99 locDOFMask[i]=myLastDOF-myFirstDOF+n;
100 ++n;
101 }
102 }
103 if (n>lastn) {
104 neighbor[numNeighbors]=p;
105 offsetInShared[numNeighbors]=lastn;
106 numNeighbors++;
107 lastn=n;
108 }
109 }
110 }
111 offsetInShared[numNeighbors]=lastn;
112
113 /* assign new DOF labels to nodes */
114 #pragma omp parallel for private(i) schedule(static)
115 for (i=0;i<numNodes;++i) {
116 k=globalDOFIndex[i];
117 if (k>-1) nodeMask[i]=locDOFMask[k-min_DOF];
118 }
119
120 /* now we can set the mapping from nodes to local DOFs */
121 this_mapping=Finley_NodeMapping_alloc(numNodes,nodeMask,UNUSED);
122
123 /* define how to get DOF values for controlled bu other processors */
124 #pragma omp parallel for private(i) schedule(static)
125 for (i=0;i<offsetInShared[numNeighbors];++i) shared[i]=myLastDOF-myFirstDOF+i;
126
127 rcv_shcomp=Paso_SharedComponents_alloc(numNeighbors,neighbor,shared,offsetInShared,1,0,dof_distribution->mpi_info);
128
129 /* now it is determined which DOFs needs to be send off:*/
130 #pragma omp parallel for private(i) schedule(static)
131 for (i=0;i<len_loc_dof;++i) locDOFMask[i]=UNUSED;
132 n=0;
133 numNeighbors=0;
134 lastn=n;
135 for (p=p_min;p<=p_max;++p) {
136 firstDOF=dof_distribution->first_component[p];
137 lastDOF=dof_distribution->first_component[p+1];
138 if (p != myRank) {
139 /* mark a DOF by p if it will be requested by processor p */
140 Finley_Mesh_markDOFsConnectedToRange(locDOFMask,min_DOF,p,firstDOF,lastDOF,in,use_reduced_elements);
141
142 for (i=myFirstDOF-min_DOF;i<myLastDOF-min_DOF;++i) {
143 if (locDOFMask[i] == p) {
144 shared[n]=i-myFirstDOF+min_DOF;
145 ++n;
146 }
147 }
148 if (n>lastn) {
149 neighbor[numNeighbors]=p;
150 offsetInShared[numNeighbors]=lastn;
151 numNeighbors++;
152 lastn=n;
153 }
154 }
155 }
156 offsetInShared[numNeighbors]=lastn;
157 snd_shcomp=Paso_SharedComponents_alloc(numNeighbors,neighbor,shared,offsetInShared,1,0,dof_distribution->mpi_info);
158
159 if (Finley_noError()) this_coupler=Paso_Coupler_alloc(snd_shcomp,rcv_shcomp);
160 /* assign new DOF labels to nodes */
161 Paso_SharedComponents_free(rcv_shcomp);
162 Paso_SharedComponents_free(snd_shcomp);
163 }
164 TMPMEMFREE(nodeMask);
165 TMPMEMFREE(neighbor);
166 TMPMEMFREE(shared);
167 TMPMEMFREE(offsetInShared);
168 TMPMEMFREE(locDOFMask);
169 if (Finley_noError()) {
170 if (use_reduced_elements) {
171 in->Nodes->reducedDegreesOfFreedomMapping=this_mapping;
172 in->Nodes->reducedDegreesOfFreedomCoupler=this_coupler;
173 } else {
174 in->Nodes->degreesOfFreedomMapping=this_mapping;
175 in->Nodes->degreesOfFreedomCoupler=this_coupler;
176 }
177 } else {
178
179 Finley_NodeMapping_free(this_mapping);
180 Paso_Coupler_free(this_coupler);
181
182 }
183
184 }
185 void Finley_Mesh_createNodeFileMappings(Finley_Mesh* in, dim_t numReducedNodes, index_t* indexReducedNodes, index_t* dof_first_component) {
186
187
188 index_t myFirstDOF, myLastDOF, myFirstNode, myLastNode, *reduced_dof_first_component=NULL, *nodeMask=NULL,
189 *reduced_nodes_first_component=NULL, *nodes_first_component=NULL,k,
190 *maskMyReducedDOF=NULL, *indexMyReducedDOF=NULL, *maskMyReducedNodes=NULL, *indexMyReducedNodes=NULL;
191 dim_t myNumDOF, myNumNodes, myNumReducedNodes, myNumReducedDOF, globalNumReducedNodes, globalNumReducedDOF,i,mpiSize, globalNumNodes, n, lastn,n0, numNeighbors;
192 Paso_MPI_rank myRank;
193
194 mpiSize=in->Nodes->MPIInfo->size;
195 myRank=in->Nodes->MPIInfo->rank;
196 /* mark the nodes used by the reduced mesh */
197
198 reduced_dof_first_component=TMPMEMALLOC(mpiSize+1,index_t);
199 reduced_nodes_first_component=TMPMEMALLOC(mpiSize+1,index_t);
200 nodes_first_component=TMPMEMALLOC(mpiSize+1,index_t);
201
202 if (! ( Finley_checkPtr(reduced_dof_first_component) || Finley_checkPtr(reduced_nodes_first_component) || Finley_checkPtr(nodes_first_component) ) ) {
203
204 globalNumNodes=Finley_NodeFile_maxGlobalNodeIDIndex(in->Nodes);
205 Paso_MPIInfo_setDistribution(in->Nodes->MPIInfo,0,globalNumNodes-1,nodes_first_component);
206
207 myFirstDOF=dof_first_component[myRank];
208 myLastDOF=dof_first_component[myRank+1];
209 myNumDOF=myLastDOF-myFirstDOF;
210 myFirstNode=nodes_first_component[myRank];
211 myLastNode=nodes_first_component[myRank+1];
212 myNumNodes=myLastNode-myFirstNode;
213
214 maskMyReducedDOF=TMPMEMALLOC(myNumDOF,index_t);
215 indexMyReducedDOF=TMPMEMALLOC(myNumDOF,index_t);
216 maskMyReducedNodes=TMPMEMALLOC(myNumNodes,index_t);
217 indexMyReducedNodes=TMPMEMALLOC(myNumNodes,index_t);
218
219 if (! ( Finley_checkPtr(maskMyReducedDOF) || Finley_checkPtr(indexMyReducedDOF) || Finley_checkPtr(maskMyReducedNodes) || Finley_checkPtr(indexMyReducedNodes) ) ) {
220
221 #pragma omp parallel private(i)
222 {
223 #pragma omp for schedule(static)
224 for (i=0;i<myNumNodes;++i) maskMyReducedNodes[i]=-1;
225 #pragma omp for schedule(static)
226 for (i=0;i<myNumDOF;++i) maskMyReducedDOF[i]=-1;
227 #pragma omp for private(k) schedule(static)
228 for (i=0;i<numReducedNodes;++i) {
229 k=in->Nodes->globalNodesIndex[indexReducedNodes[i]];
230 if ( (k>=myFirstNode) && (myLastNode>k) ) maskMyReducedNodes[k-myFirstNode]=i;
231 k=in->Nodes->globalDegreesOfFreedom[indexReducedNodes[i]];
232 if ( (k>=myFirstDOF) && (myLastDOF>k) ) maskMyReducedDOF[k-myFirstDOF]=i;
233 }
234 }
235 myNumReducedNodes=Finley_Util_packMask(myNumNodes,maskMyReducedNodes,indexMyReducedNodes);
236 myNumReducedDOF=Finley_Util_packMask(myNumDOF,maskMyReducedDOF,indexMyReducedDOF);
237
238 #ifdef PASO_MPI
239 MPI_Allgather(&myNumReducedNodes,1,MPI_INT,reduced_nodes_first_component,1,MPI_INT,in->Nodes->MPIInfo->comm);
240 MPI_Allgather(&myNumReducedDOF,1,MPI_INT,reduced_dof_first_component,1,MPI_INT,in->Nodes->MPIInfo->comm);
241 #else
242 reduced_nodes_first_component[0]=myNumReducedNodes;
243 reduced_dof_first_component[0]=myNumReducedDOF;
244 #endif
245 globalNumReducedNodes=0;
246 globalNumReducedDOF=0;
247 for (i=0;i<mpiSize;++i) {
248 k=reduced_nodes_first_component[i];
249 reduced_nodes_first_component[i]=globalNumReducedNodes;
250 globalNumReducedNodes+=k;
251
252 k=reduced_dof_first_component[i];
253 reduced_dof_first_component[i]=globalNumReducedDOF;
254 globalNumReducedDOF+=k;
255 }
256 reduced_nodes_first_component[mpiSize]=globalNumReducedNodes;
257 reduced_dof_first_component[mpiSize]=globalNumReducedDOF;
258 /* ==== distribution of Nodes ===============================*/
259 Paso_MPIInfo_setDistribution(in->Nodes->MPIInfo,0,globalNumNodes-1,nodes_first_component);
260 in->Nodes->nodesDistribution=Paso_Distribution_alloc(in->Nodes->MPIInfo,nodes_first_component,1,0);
261
262 /* ==== distribution of Nodes ===============================*/
263 in->Nodes->degreesOfFreedomDistribution=Paso_Distribution_alloc(in->Nodes->MPIInfo,dof_first_component,1,0);
264
265 /* ==== distribution of reduced Nodes ===============================*/
266 reduced_nodes_first_component[mpiSize]=globalNumReducedNodes;
267 in->Nodes->reducedNodesDistribution=Paso_Distribution_alloc(in->Nodes->MPIInfo,reduced_nodes_first_component,1,0);
268
269 /* ==== distribution of reduced DOF ===============================*/
270 in->Nodes->reducedDegreesOfFreedomDistribution=Paso_Distribution_alloc(in->Nodes->MPIInfo,reduced_dof_first_component,1,0);
271 }
272 TMPMEMFREE(maskMyReducedDOF);
273 TMPMEMFREE(indexMyReducedDOF);
274 TMPMEMFREE(maskMyReducedNodes);
275 TMPMEMFREE(indexMyReducedNodes);
276 }
277 TMPMEMFREE(reduced_dof_first_component);
278 TMPMEMFREE(reduced_nodes_first_component);
279 TMPMEMFREE(nodes_first_component);
280
281 nodeMask=TMPMEMALLOC(in->Nodes->numNodes,index_t);
282 if (! Finley_checkPtr(nodeMask) && Finley_noError()) {
283
284 /* ==== nodes mapping which is a dummy structure ======== */
285 #pragma omp parallel for private(i) schedule(static)
286 for (i=0;i<in->Nodes->numNodes;++i) nodeMask[i]=i;
287 in->Nodes->nodesMapping=Finley_NodeMapping_alloc(in->Nodes->numNodes,nodeMask,UNUSED);
288
289 /* ==== mapping between nodes and reduced nodes ========== */
290 #pragma omp parallel for private(i) schedule(static)
291 for (i=0;i<in->Nodes->numNodes;++i) nodeMask[i]=UNUSED;
292 #pragma omp parallel for private(i) schedule(static)
293 for (i=0;i<numReducedNodes;++i) nodeMask[indexReducedNodes[i]]=i;
294 in->Nodes->reducedNodesMapping=Finley_NodeMapping_alloc(in->Nodes->numNodes,nodeMask,UNUSED);
295
296 }
297 TMPMEMFREE(nodeMask);
298 /* ==== mapping between nodes and DOFs + DOF coupler ========== */
299 if ( Finley_noError()) Mesh_createDOFMappingAndCoupling(in,FALSE);
300 /* ==== mapping between nodes and reduced DOFs + reduced DOF coupler ========== */
301 if ( Finley_noError()) Mesh_createDOFMappingAndCoupling(in,TRUE);
302
303 /* get the Ids for DOFs and reduced nodes */
304 if (Finley_noError()) {
305 #pragma omp parallel private(i)
306 {
307 #pragma omp for
308 for (i=0;i<in->Nodes->reducedNodesMapping->numTargets;++i) in->Nodes->reducedNodesId[i]=in->Nodes->Id[in->Nodes->reducedNodesMapping->map[i]];
309 #pragma omp for
310 for (i=0;i<in->Nodes->degreesOfFreedomMapping->numTargets;++i) in->Nodes->degreesOfFreedomId[i]=in->Nodes->Id[in->Nodes->degreesOfFreedomMapping->map[i]];
311 #pragma omp for
312 for (i=0;i<in->Nodes->reducedDegreesOfFreedomMapping->numTargets;++i) in->Nodes->reducedDegreesOfFreedomId[i]=in->Nodes->Id[in->Nodes->reducedDegreesOfFreedomMapping->map[i]];
313 }
314 } else {
315 Finley_NodeMapping_free(in->Nodes->nodesMapping);
316 Finley_NodeMapping_free(in->Nodes->reducedNodesMapping);
317 Finley_NodeMapping_free(in->Nodes->degreesOfFreedomMapping);
318 Finley_NodeMapping_free(in->Nodes->reducedDegreesOfFreedomMapping);
319 Paso_Distribution_free(in->Nodes->nodesDistribution);
320 Paso_Distribution_free(in->Nodes->reducedNodesDistribution);
321 Paso_Distribution_free(in->Nodes->degreesOfFreedomDistribution);
322 Paso_Distribution_free(in->Nodes->reducedDegreesOfFreedomDistribution);
323 Paso_Coupler_free(in->Nodes->degreesOfFreedomCoupler);
324 Paso_Coupler_free(in->Nodes->reducedDegreesOfFreedomCoupler);
325 }
326 }
327

  ViewVC Help
Powered by ViewVC 1.1.26