/[escript]/trunk-mpi-branch/finley/src/Mesh_createNodeFileMappings.c
ViewVC logotype

Contents of /trunk-mpi-branch/finley/src/Mesh_createNodeFileMappings.c

Parent Directory Parent Directory | Revision Log Revision Log


Revision 1293 - (show annotations)
Fri Sep 7 01:10:05 2007 UTC (12 years, 8 months ago) by gross
File MIME type: text/plain
File size: 16567 byte(s)
Finally I managed to add a DOF relabeling algeorithm. It is very simple at this stage. It runs for MPI 
where it is looking at the part of the stiffness matrix with rows and columns (mainBlock) controlled by a processor. 
The local optimization is then distributed and used in the coupleBlock matrix. 



1 /*
2 ************************************************************
3 * Copyright 2007 by ACcESS MNRF *
4 * *
5 * http://www.access.edu.au *
6 * Primary Business: Queensland, Australia *
7 * Licensed under the Open Software License version 3.0 *
8 * http://www.opensource.org/licenses/osl-3.0.php *
9 * *
10 ************************************************************
11 */
12
13 /**************************************************************/
14
15 /* Finley: NodeFile : creates the mappings using the indexReducedNodes */
16 /* no distribution is happening */
17
18 /**************************************************************/
19
20 /* Author: gross@access.edu.au */
21 /* Version: $Id:$ */
22
23 /**************************************************************/
24
25 #include "Mesh.h"
26 #define UNUSED -1
27 /* #define BOUNDS_CHECK 1 */
28
29 /**************************************************************/
30
31 void Mesh_createDOFMappingAndCoupling(Finley_Mesh* in, bool_t use_reduced_elements)
32 {
33 index_t min_DOF, max_DOF, *shared=NULL, *offsetInShared=NULL, *locDOFMask=NULL, i, k, myFirstDOF, myLastDOF, *nodeMask=NULL, firstDOF, lastDOF, *globalDOFIndex;
34 dim_t mpiSize, len_loc_dof, numNeighbors, n, lastn, numNodes;
35 Paso_MPI_rank myRank,p,p_min,p_max, *neighbor=NULL;
36 Paso_SharedComponents *rcv_shcomp=NULL, *snd_shcomp=NULL;
37 Finley_NodeMapping *this_mapping=NULL;
38 Paso_Coupler* this_coupler=NULL;
39 Paso_Distribution* dof_distribution;
40
41 numNodes=in->Nodes->numNodes;
42 if (use_reduced_elements) {
43 dof_distribution=in->Nodes->reducedDegreesOfFreedomDistribution;
44 globalDOFIndex=in->Nodes->globalReducedDOFIndex;
45 Finley_NodeFile_setReducedDOFRange(&min_DOF, &max_DOF,in->Nodes);
46 } else {
47 dof_distribution=in->Nodes->degreesOfFreedomDistribution;
48 globalDOFIndex=in->Nodes->globalDegreesOfFreedom;
49 Finley_NodeFile_setDOFRange(&min_DOF, &max_DOF,in->Nodes);
50 }
51
52 mpiSize=dof_distribution->mpi_info->size;
53 myRank=dof_distribution->mpi_info->rank;
54
55 min_DOF=Finley_Util_getFlaggedMinInt(1,numNodes,globalDOFIndex,-1);
56 max_DOF=Finley_Util_getFlaggedMaxInt(1,numNodes,globalDOFIndex,-1);
57
58 p_min=mpiSize;
59 p_max=-1;
60
61 for (p=0; p<mpiSize; ++p) {
62 if (dof_distribution->first_component[p]<=min_DOF) p_min=p;
63 if (dof_distribution->first_component[p]<=max_DOF) p_max=p;
64 }
65
66 len_loc_dof=max_DOF-min_DOF+1;
67 myFirstDOF=Paso_Distribution_getFirstComponent(dof_distribution);
68 myLastDOF=Paso_Distribution_getLastComponent(dof_distribution);
69 if (! ((min_DOF<=myFirstDOF) && (myLastDOF-1<=max_DOF)) ) {
70 Finley_setError(SYSTEM_ERROR,"Local elements do not span local degrees of freedom.");
71 return;
72 }
73
74 nodeMask=TMPMEMALLOC(numNodes,index_t);
75 neighbor=TMPMEMALLOC(mpiSize,Paso_MPI_rank);
76 shared=TMPMEMALLOC(numNodes*(p_max-p_min+1),index_t);
77 offsetInShared=TMPMEMALLOC(mpiSize+1,index_t);
78 locDOFMask=TMPMEMALLOC(len_loc_dof, index_t);
79 if (! ( Finley_checkPtr(neighbor) || Finley_checkPtr(shared) || Finley_checkPtr(offsetInShared) || Finley_checkPtr(locDOFMask) || Finley_checkPtr(nodeMask) )) {
80
81
82 #pragma omp parallel for private(i) schedule(static)
83 for (i=0;i<len_loc_dof;++i) locDOFMask[i]=UNUSED;
84 #pragma omp parallel for private(i) schedule(static)
85 for (i=0;i<numNodes;++i) nodeMask[i]=UNUSED;
86
87 for (i=0;i<numNodes;++i) {
88 k=globalDOFIndex[i];
89 #ifdef BOUNDS_CHECK
90 if ((k-min_DOF) >= len_loc_dof) { printf("BOUNDS_CHECK %s:%i i=%i k=%i min_DOF=%i\n", __FILE__, __LINE__, i, k, min_DOF); exit(1); }
91 #endif
92 if (k>-1) locDOFMask[k-min_DOF]=UNUSED-1;
93 }
94
95 for (i=myFirstDOF-min_DOF;i<myLastDOF-min_DOF;++i) {
96 locDOFMask[i]=i-myFirstDOF+min_DOF;
97 #ifdef BOUNDS_CHECK
98 if (i < 0 || i >= len_loc_dof) { printf("BOUNDS_CHECK %s:%i i=%i\n", __FILE__, __LINE__, i); exit(1); }
99 #endif
100 }
101
102 numNeighbors=0;
103 n=0;
104 lastn=n;
105 for (p=p_min;p<=p_max;++p) {
106 firstDOF=MAX(min_DOF,dof_distribution->first_component[p]);
107 lastDOF=MIN(max_DOF+1,dof_distribution->first_component[p+1]);
108 if (p != myRank) {
109 for (i=firstDOF-min_DOF;i<lastDOF-min_DOF;++i) {
110 #ifdef BOUNDS_CHECK
111 if (i < 0 || i >= len_loc_dof) { printf("BOUNDS_CHECK %s:%i p=%i i=%i\n", __FILE__, __LINE__, p, i); exit(1); }
112 #endif
113 if (locDOFMask[i] == UNUSED-1) {
114 locDOFMask[i]=myLastDOF-myFirstDOF+n;
115 ++n;
116 }
117 }
118 if (n>lastn) {
119 neighbor[numNeighbors]=p;
120 #ifdef BOUNDS_CHECK
121 if (numNeighbors < 0 || numNeighbors >= mpiSize+1) { printf("BOUNDS_CHECK %s:%i p=%i numNeighbors=%i n=%i\n", __FILE__, __LINE__, p, numNeighbors, n); exit(1); }
122 #endif
123 offsetInShared[numNeighbors]=lastn;
124 numNeighbors++;
125 lastn=n;
126 }
127 }
128 }
129 #ifdef BOUNDS_CHECK
130 if (numNeighbors < 0 || numNeighbors >= mpiSize+1) { printf("BOUNDS_CHECK %s:%i numNeighbors=%i\n", __FILE__, __LINE__, numNeighbors); exit(1); }
131 #endif
132 offsetInShared[numNeighbors]=lastn;
133
134 /* assign new DOF labels to nodes */
135 #pragma omp parallel for private(i) schedule(static)
136 for (i=0;i<numNodes;++i) {
137 k=globalDOFIndex[i];
138 if (k>-1) nodeMask[i]=locDOFMask[k-min_DOF];
139 }
140
141 /* now we can set the mapping from nodes to local DOFs */
142 this_mapping=Finley_NodeMapping_alloc(numNodes,nodeMask,UNUSED);
143 /* define how to get DOF values for controlled bu other processors */
144 #ifdef BOUNDS_CHECK
145 for (i=0;i<offsetInShared[numNeighbors];++i) {
146 if (i < 0 || i >= numNodes*(p_max-p_min+1)) { printf("BOUNDS_CHECK %s:%i i=%i\n", __FILE__, __LINE__, i); exit(1); }
147 }
148 #endif
149 #pragma omp parallel for private(i) schedule(static)
150 for (i=0;i<offsetInShared[numNeighbors];++i) shared[i]=myLastDOF-myFirstDOF+i;
151
152 rcv_shcomp=Paso_SharedComponents_alloc(numNeighbors,neighbor,shared,offsetInShared,1,0,dof_distribution->mpi_info);
153
154 /* now it is determined which DOFs needs to be send off:*/
155 #pragma omp parallel for private(i) schedule(static)
156 for (i=0;i<len_loc_dof;++i) locDOFMask[i]=UNUSED;
157 n=0;
158 numNeighbors=0;
159 lastn=n;
160 for (p=p_min;p<=p_max;++p) {
161 firstDOF=dof_distribution->first_component[p];
162 lastDOF=dof_distribution->first_component[p+1];
163 if (p != myRank) {
164 /* mark a DOF by p if it will be requested by processor p */
165 Finley_Mesh_markDOFsConnectedToRange(locDOFMask,min_DOF,p,firstDOF,lastDOF,in,use_reduced_elements);
166
167 for (i=myFirstDOF-min_DOF;i<myLastDOF-min_DOF;++i) {
168 if (locDOFMask[i] == p) {
169 #ifdef BOUNDS_CHECK
170 if (n < 0 || n >= numNodes*(p_max-p_min+1)) { printf("BOUNDS_CHECK %s:%i p=%i i=%i n=%i\n", __FILE__, __LINE__, p, i, n); exit(1); }
171 #endif
172 shared[n]=i-myFirstDOF+min_DOF;
173 ++n;
174 }
175 }
176 if (n>lastn) {
177 neighbor[numNeighbors]=p;
178 #ifdef BOUNDS_CHECK
179 if (numNeighbors < 0 || numNeighbors >= mpiSize+1) { printf("BOUNDS_CHECK %s:%i p=%i n=%i numNeighbors=%i\n", __FILE__, __LINE__, p, n, numNeighbors); exit(1); }
180 #endif
181 offsetInShared[numNeighbors]=lastn;
182 numNeighbors++;
183 lastn=n;
184 }
185 }
186 }
187 #ifdef BOUNDS_CHECK
188 if (numNeighbors < 0 || numNeighbors >= mpiSize+1) { printf("BOUNDS_CHECK %s:%i numNeighbors=%i\n", __FILE__, __LINE__, numNeighbors); exit(1); }
189 #endif
190 offsetInShared[numNeighbors]=lastn;
191 snd_shcomp=Paso_SharedComponents_alloc(numNeighbors,neighbor,shared,offsetInShared,1,0,dof_distribution->mpi_info);
192
193 if (Finley_noError()) this_coupler=Paso_Coupler_alloc(snd_shcomp,rcv_shcomp);
194 /* assign new DOF labels to nodes */
195 Paso_SharedComponents_free(rcv_shcomp);
196 Paso_SharedComponents_free(snd_shcomp);
197 }
198 TMPMEMFREE(nodeMask);
199 TMPMEMFREE(neighbor);
200 TMPMEMFREE(shared);
201 TMPMEMFREE(offsetInShared);
202 TMPMEMFREE(locDOFMask);
203 if (Finley_noError()) {
204 if (use_reduced_elements) {
205 in->Nodes->reducedDegreesOfFreedomMapping=this_mapping;
206 in->Nodes->reducedDegreesOfFreedomCoupler=this_coupler;
207 } else {
208 in->Nodes->degreesOfFreedomMapping=this_mapping;
209 in->Nodes->degreesOfFreedomCoupler=this_coupler;
210 }
211 } else {
212 Finley_NodeMapping_free(this_mapping);
213 Paso_Coupler_free(this_coupler);
214
215 }
216 }
217 void Finley_Mesh_createNodeFileMappings(Finley_Mesh* in, dim_t numReducedNodes, index_t* indexReducedNodes, index_t* dof_first_component) {
218
219
220 index_t myFirstDOF, myLastDOF, myFirstNode, myLastNode, *reduced_dof_first_component=NULL, *nodeMask=NULL,
221 *reduced_nodes_first_component=NULL, *nodes_first_component=NULL,k,
222 *maskMyReducedDOF=NULL, *indexMyReducedDOF=NULL, *maskMyReducedNodes=NULL, *indexMyReducedNodes=NULL;
223 dim_t myNumDOF, myNumNodes, myNumReducedNodes, myNumReducedDOF, globalNumReducedNodes, globalNumReducedDOF,i,mpiSize, globalNumNodes, n, lastn,n0, numNeighbors;
224 Paso_MPI_rank myRank;
225
226 mpiSize=in->Nodes->MPIInfo->size;
227 myRank=in->Nodes->MPIInfo->rank;
228 /* mark the nodes used by the reduced mesh */
229
230 reduced_dof_first_component=TMPMEMALLOC(mpiSize+1,index_t);
231 reduced_nodes_first_component=TMPMEMALLOC(mpiSize+1,index_t);
232 nodes_first_component=TMPMEMALLOC(mpiSize+1,index_t);
233
234 if (! ( Finley_checkPtr(reduced_dof_first_component) || Finley_checkPtr(reduced_nodes_first_component) || Finley_checkPtr(nodes_first_component) ) ) {
235
236 globalNumNodes=Finley_NodeFile_maxGlobalNodeIDIndex(in->Nodes)+1;
237 Paso_MPIInfo_setDistribution(in->Nodes->MPIInfo,0,globalNumNodes-1,nodes_first_component);
238
239 myFirstDOF=dof_first_component[myRank];
240 myLastDOF=dof_first_component[myRank+1];
241 myNumDOF=myLastDOF-myFirstDOF;
242 myFirstNode=nodes_first_component[myRank];
243 myLastNode=nodes_first_component[myRank+1];
244 myNumNodes=myLastNode-myFirstNode;
245
246 maskMyReducedDOF=TMPMEMALLOC(myNumDOF,index_t);
247 indexMyReducedDOF=TMPMEMALLOC(myNumDOF,index_t);
248 maskMyReducedNodes=TMPMEMALLOC(myNumNodes,index_t);
249 indexMyReducedNodes=TMPMEMALLOC(myNumNodes,index_t);
250
251 if (! ( Finley_checkPtr(maskMyReducedDOF) || Finley_checkPtr(indexMyReducedDOF) || Finley_checkPtr(maskMyReducedNodes) || Finley_checkPtr(indexMyReducedNodes) ) ) {
252
253 #pragma omp parallel private(i)
254 {
255 #pragma omp for schedule(static)
256 for (i=0;i<myNumNodes;++i) maskMyReducedNodes[i]=-1;
257 #pragma omp for schedule(static)
258 for (i=0;i<myNumDOF;++i) maskMyReducedDOF[i]=-1;
259 #pragma omp for private(k) schedule(static)
260 for (i=0;i<numReducedNodes;++i) {
261 k=in->Nodes->globalNodesIndex[indexReducedNodes[i]];
262 if ( (k>=myFirstNode) && (myLastNode>k) ) maskMyReducedNodes[k-myFirstNode]=i;
263 k=in->Nodes->globalDegreesOfFreedom[indexReducedNodes[i]];
264 if ( (k>=myFirstDOF) && (myLastDOF>k) ) maskMyReducedDOF[k-myFirstDOF]=i;
265 }
266 }
267 myNumReducedNodes=Finley_Util_packMask(myNumNodes,maskMyReducedNodes,indexMyReducedNodes);
268 myNumReducedDOF=Finley_Util_packMask(myNumDOF,maskMyReducedDOF,indexMyReducedDOF);
269
270 #ifdef PASO_MPI
271 MPI_Allgather(&myNumReducedNodes,1,MPI_INT,reduced_nodes_first_component,1,MPI_INT,in->Nodes->MPIInfo->comm);
272 MPI_Allgather(&myNumReducedDOF,1,MPI_INT,reduced_dof_first_component,1,MPI_INT,in->Nodes->MPIInfo->comm);
273 #else
274 reduced_nodes_first_component[0]=myNumReducedNodes;
275 reduced_dof_first_component[0]=myNumReducedDOF;
276 #endif
277 globalNumReducedNodes=0;
278 globalNumReducedDOF=0;
279 for (i=0;i<mpiSize;++i) {
280 k=reduced_nodes_first_component[i];
281 reduced_nodes_first_component[i]=globalNumReducedNodes;
282 globalNumReducedNodes+=k;
283
284 k=reduced_dof_first_component[i];
285 reduced_dof_first_component[i]=globalNumReducedDOF;
286 globalNumReducedDOF+=k;
287 }
288 reduced_nodes_first_component[mpiSize]=globalNumReducedNodes;
289 reduced_dof_first_component[mpiSize]=globalNumReducedDOF;
290 /* ==== distribution of Nodes ===============================*/
291 Paso_MPIInfo_setDistribution(in->Nodes->MPIInfo,0,globalNumNodes-1,nodes_first_component);
292 in->Nodes->nodesDistribution=Paso_Distribution_alloc(in->Nodes->MPIInfo,nodes_first_component,1,0);
293
294 /* ==== distribution of Nodes ===============================*/
295 in->Nodes->degreesOfFreedomDistribution=Paso_Distribution_alloc(in->Nodes->MPIInfo,dof_first_component,1,0);
296
297 /* ==== distribution of reduced Nodes ===============================*/
298 reduced_nodes_first_component[mpiSize]=globalNumReducedNodes;
299 in->Nodes->reducedNodesDistribution=Paso_Distribution_alloc(in->Nodes->MPIInfo,reduced_nodes_first_component,1,0);
300
301 /* ==== distribution of reduced DOF ===============================*/
302 in->Nodes->reducedDegreesOfFreedomDistribution=Paso_Distribution_alloc(in->Nodes->MPIInfo,reduced_dof_first_component,1,0);
303 }
304 TMPMEMFREE(maskMyReducedDOF);
305 TMPMEMFREE(indexMyReducedDOF);
306 TMPMEMFREE(maskMyReducedNodes);
307 TMPMEMFREE(indexMyReducedNodes);
308 }
309 TMPMEMFREE(reduced_dof_first_component);
310 TMPMEMFREE(reduced_nodes_first_component);
311 TMPMEMFREE(nodes_first_component);
312
313 nodeMask=TMPMEMALLOC(in->Nodes->numNodes,index_t);
314 if (! Finley_checkPtr(nodeMask) && Finley_noError()) {
315
316 /* ==== nodes mapping which is a dummy structure ======== */
317 #pragma omp parallel for private(i) schedule(static)
318 for (i=0;i<in->Nodes->numNodes;++i) nodeMask[i]=i;
319 in->Nodes->nodesMapping=Finley_NodeMapping_alloc(in->Nodes->numNodes,nodeMask,UNUSED);
320
321 /* ==== mapping between nodes and reduced nodes ========== */
322 #pragma omp parallel for private(i) schedule(static)
323 for (i=0;i<in->Nodes->numNodes;++i) nodeMask[i]=UNUSED;
324 #pragma omp parallel for private(i) schedule(static)
325 for (i=0;i<numReducedNodes;++i) nodeMask[indexReducedNodes[i]]=i;
326 in->Nodes->reducedNodesMapping=Finley_NodeMapping_alloc(in->Nodes->numNodes,nodeMask,UNUSED);
327
328 }
329 TMPMEMFREE(nodeMask);
330 /* ==== mapping between nodes and DOFs + DOF coupler ========== */
331 if ( Finley_noError()) Mesh_createDOFMappingAndCoupling(in,FALSE);
332 /* ==== mapping between nodes and reduced DOFs + reduced DOF coupler ========== */
333 if ( Finley_noError()) Mesh_createDOFMappingAndCoupling(in,TRUE);
334
335 /* get the Ids for DOFs and reduced nodes */
336 if (Finley_noError()) {
337 #pragma omp parallel private(i)
338 {
339 #pragma omp for
340 for (i=0;i<in->Nodes->reducedNodesMapping->numTargets;++i) in->Nodes->reducedNodesId[i]=in->Nodes->Id[in->Nodes->reducedNodesMapping->map[i]];
341 #pragma omp for
342 for (i=0;i<in->Nodes->degreesOfFreedomMapping->numTargets;++i) in->Nodes->degreesOfFreedomId[i]=in->Nodes->Id[in->Nodes->degreesOfFreedomMapping->map[i]];
343 #pragma omp for
344 for (i=0;i<in->Nodes->reducedDegreesOfFreedomMapping->numTargets;++i) in->Nodes->reducedDegreesOfFreedomId[i]=in->Nodes->Id[in->Nodes->reducedDegreesOfFreedomMapping->map[i]];
345 }
346 } else {
347 Finley_NodeMapping_free(in->Nodes->nodesMapping);
348 Finley_NodeMapping_free(in->Nodes->reducedNodesMapping);
349 Finley_NodeMapping_free(in->Nodes->degreesOfFreedomMapping);
350 Finley_NodeMapping_free(in->Nodes->reducedDegreesOfFreedomMapping);
351 Paso_Distribution_free(in->Nodes->nodesDistribution);
352 Paso_Distribution_free(in->Nodes->reducedNodesDistribution);
353 Paso_Distribution_free(in->Nodes->degreesOfFreedomDistribution);
354 Paso_Distribution_free(in->Nodes->reducedDegreesOfFreedomDistribution);
355 Paso_Coupler_free(in->Nodes->degreesOfFreedomCoupler);
356 Paso_Coupler_free(in->Nodes->reducedDegreesOfFreedomCoupler);
357 in->Nodes->nodesMapping=NULL;
358 in->Nodes->reducedNodesMapping=NULL;
359 in->Nodes->degreesOfFreedomMapping=NULL;
360 in->Nodes->reducedDegreesOfFreedomMapping=NULL;
361 in->Nodes->nodesDistribution=NULL;
362 in->Nodes->reducedNodesDistribution=NULL;
363 in->Nodes->degreesOfFreedomDistribution=NULL;
364 in->Nodes->reducedDegreesOfFreedomDistribution=NULL;
365 in->Nodes->degreesOfFreedomCoupler=NULL;
366 in->Nodes->reducedDegreesOfFreedomCoupler=NULL;
367 }
368 }
369

  ViewVC Help
Powered by ViewVC 1.1.26