/[escript]/trunk-mpi-branch/finley/src/Mesh_createNodeFileMappings.c
ViewVC logotype

Contents of /trunk-mpi-branch/finley/src/Mesh_createNodeFileMappings.c

Parent Directory Parent Directory | Revision Log Revision Log


Revision 1298 - (show annotations)
Wed Sep 12 05:09:18 2007 UTC (12 years, 8 months ago) by ksteube
File MIME type: text/plain
File size: 16538 byte(s)
Pattern_reduceBandwidth.c had an error finding a minimum.
Implemented scons bounds_check=1 to provide array bounds checking in a few areas of the code.
Provided alternative TMPMEMALLOC and TMPMEMFREE in Common.h to provide help with memory problems.

1 /*
2 ************************************************************
3 * Copyright 2007 by ACcESS MNRF *
4 * *
5 * http://www.access.edu.au *
6 * Primary Business: Queensland, Australia *
7 * Licensed under the Open Software License version 3.0 *
8 * http://www.opensource.org/licenses/osl-3.0.php *
9 * *
10 ************************************************************
11 */
12
13 /**************************************************************/
14
15 /* Finley: NodeFile : creates the mappings using the indexReducedNodes */
16 /* no distribution is happening */
17
18 /**************************************************************/
19
20 /* Author: gross@access.edu.au */
21 /* Version: $Id:$ */
22
23 /**************************************************************/
24
25 #include "Mesh.h"
26 #define UNUSED -1
27
28 /**************************************************************/
29
30 void Mesh_createDOFMappingAndCoupling(Finley_Mesh* in, bool_t use_reduced_elements)
31 {
32 index_t min_DOF, max_DOF, *shared=NULL, *offsetInShared=NULL, *locDOFMask=NULL, i, k, myFirstDOF, myLastDOF, *nodeMask=NULL, firstDOF, lastDOF, *globalDOFIndex;
33 dim_t mpiSize, len_loc_dof, numNeighbors, n, lastn, numNodes;
34 Paso_MPI_rank myRank,p,p_min,p_max, *neighbor=NULL;
35 Paso_SharedComponents *rcv_shcomp=NULL, *snd_shcomp=NULL;
36 Finley_NodeMapping *this_mapping=NULL;
37 Paso_Coupler* this_coupler=NULL;
38 Paso_Distribution* dof_distribution;
39
40 numNodes=in->Nodes->numNodes;
41 if (use_reduced_elements) {
42 dof_distribution=in->Nodes->reducedDegreesOfFreedomDistribution;
43 globalDOFIndex=in->Nodes->globalReducedDOFIndex;
44 Finley_NodeFile_setReducedDOFRange(&min_DOF, &max_DOF,in->Nodes);
45 } else {
46 dof_distribution=in->Nodes->degreesOfFreedomDistribution;
47 globalDOFIndex=in->Nodes->globalDegreesOfFreedom;
48 Finley_NodeFile_setDOFRange(&min_DOF, &max_DOF,in->Nodes);
49 }
50
51 mpiSize=dof_distribution->mpi_info->size;
52 myRank=dof_distribution->mpi_info->rank;
53
54 min_DOF=Finley_Util_getFlaggedMinInt(1,numNodes,globalDOFIndex,-1);
55 max_DOF=Finley_Util_getFlaggedMaxInt(1,numNodes,globalDOFIndex,-1);
56
57 p_min=mpiSize;
58 p_max=-1;
59
60 for (p=0; p<mpiSize; ++p) {
61 if (dof_distribution->first_component[p]<=min_DOF) p_min=p;
62 if (dof_distribution->first_component[p]<=max_DOF) p_max=p;
63 }
64
65 len_loc_dof=max_DOF-min_DOF+1;
66 myFirstDOF=Paso_Distribution_getFirstComponent(dof_distribution);
67 myLastDOF=Paso_Distribution_getLastComponent(dof_distribution);
68 if (! ((min_DOF<=myFirstDOF) && (myLastDOF-1<=max_DOF)) ) {
69 Finley_setError(SYSTEM_ERROR,"Local elements do not span local degrees of freedom.");
70 return;
71 }
72
73 nodeMask=TMPMEMALLOC(numNodes,index_t);
74 neighbor=TMPMEMALLOC(mpiSize,Paso_MPI_rank);
75 shared=TMPMEMALLOC(numNodes*(p_max-p_min+1),index_t);
76 offsetInShared=TMPMEMALLOC(mpiSize+1,index_t);
77 locDOFMask=TMPMEMALLOC(len_loc_dof, index_t);
78 if (! ( Finley_checkPtr(neighbor) || Finley_checkPtr(shared) || Finley_checkPtr(offsetInShared) || Finley_checkPtr(locDOFMask) || Finley_checkPtr(nodeMask) )) {
79
80
81 #pragma omp parallel for private(i) schedule(static)
82 for (i=0;i<len_loc_dof;++i) locDOFMask[i]=UNUSED;
83 #pragma omp parallel for private(i) schedule(static)
84 for (i=0;i<numNodes;++i) nodeMask[i]=UNUSED;
85
86 for (i=0;i<numNodes;++i) {
87 k=globalDOFIndex[i];
88 #ifdef BOUNDS_CHECK
89 if ((k-min_DOF) >= len_loc_dof) { printf("BOUNDS_CHECK %s %i i=%i k=%i min_DOF=%i\n", __FILE__, __LINE__, i, k, min_DOF); exit(1); }
90 #endif
91 if (k>-1) locDOFMask[k-min_DOF]=UNUSED-1;
92 }
93
94 for (i=myFirstDOF-min_DOF;i<myLastDOF-min_DOF;++i) {
95 locDOFMask[i]=i-myFirstDOF+min_DOF;
96 #ifdef BOUNDS_CHECK
97 if (i < 0 || i >= len_loc_dof) { printf("BOUNDS_CHECK %s %i i=%i\n", __FILE__, __LINE__, i); exit(1); }
98 #endif
99 }
100
101 numNeighbors=0;
102 n=0;
103 lastn=n;
104 for (p=p_min;p<=p_max;++p) {
105 firstDOF=MAX(min_DOF,dof_distribution->first_component[p]);
106 lastDOF=MIN(max_DOF+1,dof_distribution->first_component[p+1]);
107 if (p != myRank) {
108 for (i=firstDOF-min_DOF;i<lastDOF-min_DOF;++i) {
109 #ifdef BOUNDS_CHECK
110 if (i < 0 || i >= len_loc_dof) { printf("BOUNDS_CHECK %s %i p=%i i=%i\n", __FILE__, __LINE__, p, i); exit(1); }
111 #endif
112 if (locDOFMask[i] == UNUSED-1) {
113 locDOFMask[i]=myLastDOF-myFirstDOF+n;
114 ++n;
115 }
116 }
117 if (n>lastn) {
118 neighbor[numNeighbors]=p;
119 #ifdef BOUNDS_CHECK
120 if (numNeighbors < 0 || numNeighbors >= mpiSize+1) { printf("BOUNDS_CHECK %s %i p=%i numNeighbors=%i n=%i\n", __FILE__, __LINE__, p, numNeighbors, n); exit(1); }
121 #endif
122 offsetInShared[numNeighbors]=lastn;
123 numNeighbors++;
124 lastn=n;
125 }
126 }
127 }
128 #ifdef BOUNDS_CHECK
129 if (numNeighbors < 0 || numNeighbors >= mpiSize+1) { printf("BOUNDS_CHECK %s %i numNeighbors=%i\n", __FILE__, __LINE__, numNeighbors); exit(1); }
130 #endif
131 offsetInShared[numNeighbors]=lastn;
132
133 /* assign new DOF labels to nodes */
134 #pragma omp parallel for private(i) schedule(static)
135 for (i=0;i<numNodes;++i) {
136 k=globalDOFIndex[i];
137 if (k>-1) nodeMask[i]=locDOFMask[k-min_DOF];
138 }
139
140 /* now we can set the mapping from nodes to local DOFs */
141 this_mapping=Finley_NodeMapping_alloc(numNodes,nodeMask,UNUSED);
142 /* define how to get DOF values for controlled bu other processors */
143 #ifdef BOUNDS_CHECK
144 for (i=0;i<offsetInShared[numNeighbors];++i) {
145 if (i < 0 || i >= numNodes*(p_max-p_min+1)) { printf("BOUNDS_CHECK %s %i i=%i\n", __FILE__, __LINE__, i); exit(1); }
146 }
147 #endif
148 #pragma omp parallel for private(i) schedule(static)
149 for (i=0;i<offsetInShared[numNeighbors];++i) shared[i]=myLastDOF-myFirstDOF+i;
150
151 rcv_shcomp=Paso_SharedComponents_alloc(numNeighbors,neighbor,shared,offsetInShared,1,0,dof_distribution->mpi_info);
152
153 /* now it is determined which DOFs needs to be send off:*/
154 #pragma omp parallel for private(i) schedule(static)
155 for (i=0;i<len_loc_dof;++i) locDOFMask[i]=UNUSED;
156 n=0;
157 numNeighbors=0;
158 lastn=n;
159 for (p=p_min;p<=p_max;++p) {
160 firstDOF=dof_distribution->first_component[p];
161 lastDOF=dof_distribution->first_component[p+1];
162 if (p != myRank) {
163 /* mark a DOF by p if it will be requested by processor p */
164 Finley_Mesh_markDOFsConnectedToRange(locDOFMask,min_DOF,p,firstDOF,lastDOF,in,use_reduced_elements);
165
166 for (i=myFirstDOF-min_DOF;i<myLastDOF-min_DOF;++i) {
167 if (locDOFMask[i] == p) {
168 #ifdef BOUNDS_CHECK
169 if (n < 0 || n >= numNodes*(p_max-p_min+1)) { printf("BOUNDS_CHECK %s %i p=%i i=%i n=%i\n", __FILE__, __LINE__, p, i, n); exit(1); }
170 #endif
171 shared[n]=i-myFirstDOF+min_DOF;
172 ++n;
173 }
174 }
175 if (n>lastn) {
176 neighbor[numNeighbors]=p;
177 #ifdef BOUNDS_CHECK
178 if (numNeighbors < 0 || numNeighbors >= mpiSize+1) { printf("BOUNDS_CHECK %s %i p=%i n=%i numNeighbors=%i\n", __FILE__, __LINE__, p, n, numNeighbors); exit(1); }
179 #endif
180 offsetInShared[numNeighbors]=lastn;
181 numNeighbors++;
182 lastn=n;
183 }
184 }
185 }
186 #ifdef BOUNDS_CHECK
187 if (numNeighbors < 0 || numNeighbors >= mpiSize+1) { printf("BOUNDS_CHECK %s %i numNeighbors=%i\n", __FILE__, __LINE__, numNeighbors); exit(1); }
188 #endif
189 offsetInShared[numNeighbors]=lastn;
190 snd_shcomp=Paso_SharedComponents_alloc(numNeighbors,neighbor,shared,offsetInShared,1,0,dof_distribution->mpi_info);
191
192 if (Finley_noError()) this_coupler=Paso_Coupler_alloc(snd_shcomp,rcv_shcomp);
193 /* assign new DOF labels to nodes */
194 Paso_SharedComponents_free(rcv_shcomp);
195 Paso_SharedComponents_free(snd_shcomp);
196 }
197 TMPMEMFREE(nodeMask);
198 TMPMEMFREE(neighbor);
199 TMPMEMFREE(shared);
200 TMPMEMFREE(offsetInShared);
201 TMPMEMFREE(locDOFMask);
202 if (Finley_noError()) {
203 if (use_reduced_elements) {
204 in->Nodes->reducedDegreesOfFreedomMapping=this_mapping;
205 in->Nodes->reducedDegreesOfFreedomCoupler=this_coupler;
206 } else {
207 in->Nodes->degreesOfFreedomMapping=this_mapping;
208 in->Nodes->degreesOfFreedomCoupler=this_coupler;
209 }
210 } else {
211 Finley_NodeMapping_free(this_mapping);
212 Paso_Coupler_free(this_coupler);
213
214 }
215 }
216 void Finley_Mesh_createNodeFileMappings(Finley_Mesh* in, dim_t numReducedNodes, index_t* indexReducedNodes, index_t* dof_first_component) {
217
218
219 index_t myFirstDOF, myLastDOF, myFirstNode, myLastNode, *reduced_dof_first_component=NULL, *nodeMask=NULL,
220 *reduced_nodes_first_component=NULL, *nodes_first_component=NULL,k,
221 *maskMyReducedDOF=NULL, *indexMyReducedDOF=NULL, *maskMyReducedNodes=NULL, *indexMyReducedNodes=NULL;
222 dim_t myNumDOF, myNumNodes, myNumReducedNodes, myNumReducedDOF, globalNumReducedNodes, globalNumReducedDOF,i,mpiSize, globalNumNodes, n, lastn,n0, numNeighbors;
223 Paso_MPI_rank myRank;
224
225 mpiSize=in->Nodes->MPIInfo->size;
226 myRank=in->Nodes->MPIInfo->rank;
227 /* mark the nodes used by the reduced mesh */
228
229 reduced_dof_first_component=TMPMEMALLOC(mpiSize+1,index_t);
230 reduced_nodes_first_component=TMPMEMALLOC(mpiSize+1,index_t);
231 nodes_first_component=TMPMEMALLOC(mpiSize+1,index_t);
232
233 if (! ( Finley_checkPtr(reduced_dof_first_component) || Finley_checkPtr(reduced_nodes_first_component) || Finley_checkPtr(nodes_first_component) ) ) {
234
235 globalNumNodes=Finley_NodeFile_maxGlobalNodeIDIndex(in->Nodes)+1;
236 Paso_MPIInfo_setDistribution(in->Nodes->MPIInfo,0,globalNumNodes-1,nodes_first_component);
237
238 myFirstDOF=dof_first_component[myRank];
239 myLastDOF=dof_first_component[myRank+1];
240 myNumDOF=myLastDOF-myFirstDOF;
241 myFirstNode=nodes_first_component[myRank];
242 myLastNode=nodes_first_component[myRank+1];
243 myNumNodes=myLastNode-myFirstNode;
244
245 maskMyReducedDOF=TMPMEMALLOC(myNumDOF,index_t);
246 indexMyReducedDOF=TMPMEMALLOC(myNumDOF,index_t);
247 maskMyReducedNodes=TMPMEMALLOC(myNumNodes,index_t);
248 indexMyReducedNodes=TMPMEMALLOC(myNumNodes,index_t);
249
250 if (! ( Finley_checkPtr(maskMyReducedDOF) || Finley_checkPtr(indexMyReducedDOF) || Finley_checkPtr(maskMyReducedNodes) || Finley_checkPtr(indexMyReducedNodes) ) ) {
251
252 #pragma omp parallel private(i)
253 {
254 #pragma omp for schedule(static)
255 for (i=0;i<myNumNodes;++i) maskMyReducedNodes[i]=-1;
256 #pragma omp for schedule(static)
257 for (i=0;i<myNumDOF;++i) maskMyReducedDOF[i]=-1;
258 #pragma omp for private(k) schedule(static)
259 for (i=0;i<numReducedNodes;++i) {
260 k=in->Nodes->globalNodesIndex[indexReducedNodes[i]];
261 if ( (k>=myFirstNode) && (myLastNode>k) ) maskMyReducedNodes[k-myFirstNode]=i;
262 k=in->Nodes->globalDegreesOfFreedom[indexReducedNodes[i]];
263 if ( (k>=myFirstDOF) && (myLastDOF>k) ) maskMyReducedDOF[k-myFirstDOF]=i;
264 }
265 }
266 myNumReducedNodes=Finley_Util_packMask(myNumNodes,maskMyReducedNodes,indexMyReducedNodes);
267 myNumReducedDOF=Finley_Util_packMask(myNumDOF,maskMyReducedDOF,indexMyReducedDOF);
268
269 #ifdef PASO_MPI
270 MPI_Allgather(&myNumReducedNodes,1,MPI_INT,reduced_nodes_first_component,1,MPI_INT,in->Nodes->MPIInfo->comm);
271 MPI_Allgather(&myNumReducedDOF,1,MPI_INT,reduced_dof_first_component,1,MPI_INT,in->Nodes->MPIInfo->comm);
272 #else
273 reduced_nodes_first_component[0]=myNumReducedNodes;
274 reduced_dof_first_component[0]=myNumReducedDOF;
275 #endif
276 globalNumReducedNodes=0;
277 globalNumReducedDOF=0;
278 for (i=0;i<mpiSize;++i) {
279 k=reduced_nodes_first_component[i];
280 reduced_nodes_first_component[i]=globalNumReducedNodes;
281 globalNumReducedNodes+=k;
282
283 k=reduced_dof_first_component[i];
284 reduced_dof_first_component[i]=globalNumReducedDOF;
285 globalNumReducedDOF+=k;
286 }
287 reduced_nodes_first_component[mpiSize]=globalNumReducedNodes;
288 reduced_dof_first_component[mpiSize]=globalNumReducedDOF;
289 /* ==== distribution of Nodes ===============================*/
290 Paso_MPIInfo_setDistribution(in->Nodes->MPIInfo,0,globalNumNodes-1,nodes_first_component);
291 in->Nodes->nodesDistribution=Paso_Distribution_alloc(in->Nodes->MPIInfo,nodes_first_component,1,0);
292
293 /* ==== distribution of Nodes ===============================*/
294 in->Nodes->degreesOfFreedomDistribution=Paso_Distribution_alloc(in->Nodes->MPIInfo,dof_first_component,1,0);
295
296 /* ==== distribution of reduced Nodes ===============================*/
297 reduced_nodes_first_component[mpiSize]=globalNumReducedNodes;
298 in->Nodes->reducedNodesDistribution=Paso_Distribution_alloc(in->Nodes->MPIInfo,reduced_nodes_first_component,1,0);
299
300 /* ==== distribution of reduced DOF ===============================*/
301 in->Nodes->reducedDegreesOfFreedomDistribution=Paso_Distribution_alloc(in->Nodes->MPIInfo,reduced_dof_first_component,1,0);
302 }
303 TMPMEMFREE(maskMyReducedDOF);
304 TMPMEMFREE(indexMyReducedDOF);
305 TMPMEMFREE(maskMyReducedNodes);
306 TMPMEMFREE(indexMyReducedNodes);
307 }
308 TMPMEMFREE(reduced_dof_first_component);
309 TMPMEMFREE(reduced_nodes_first_component);
310 TMPMEMFREE(nodes_first_component);
311
312 nodeMask=TMPMEMALLOC(in->Nodes->numNodes,index_t);
313 if (! Finley_checkPtr(nodeMask) && Finley_noError()) {
314
315 /* ==== nodes mapping which is a dummy structure ======== */
316 #pragma omp parallel for private(i) schedule(static)
317 for (i=0;i<in->Nodes->numNodes;++i) nodeMask[i]=i;
318 in->Nodes->nodesMapping=Finley_NodeMapping_alloc(in->Nodes->numNodes,nodeMask,UNUSED);
319
320 /* ==== mapping between nodes and reduced nodes ========== */
321 #pragma omp parallel for private(i) schedule(static)
322 for (i=0;i<in->Nodes->numNodes;++i) nodeMask[i]=UNUSED;
323 #pragma omp parallel for private(i) schedule(static)
324 for (i=0;i<numReducedNodes;++i) nodeMask[indexReducedNodes[i]]=i;
325 in->Nodes->reducedNodesMapping=Finley_NodeMapping_alloc(in->Nodes->numNodes,nodeMask,UNUSED);
326
327 }
328 TMPMEMFREE(nodeMask);
329 /* ==== mapping between nodes and DOFs + DOF coupler ========== */
330 if ( Finley_noError()) Mesh_createDOFMappingAndCoupling(in,FALSE);
331 /* ==== mapping between nodes and reduced DOFs + reduced DOF coupler ========== */
332 if ( Finley_noError()) Mesh_createDOFMappingAndCoupling(in,TRUE);
333
334 /* get the Ids for DOFs and reduced nodes */
335 if (Finley_noError()) {
336 #pragma omp parallel private(i)
337 {
338 #pragma omp for
339 for (i=0;i<in->Nodes->reducedNodesMapping->numTargets;++i) in->Nodes->reducedNodesId[i]=in->Nodes->Id[in->Nodes->reducedNodesMapping->map[i]];
340 #pragma omp for
341 for (i=0;i<in->Nodes->degreesOfFreedomMapping->numTargets;++i) in->Nodes->degreesOfFreedomId[i]=in->Nodes->Id[in->Nodes->degreesOfFreedomMapping->map[i]];
342 #pragma omp for
343 for (i=0;i<in->Nodes->reducedDegreesOfFreedomMapping->numTargets;++i) in->Nodes->reducedDegreesOfFreedomId[i]=in->Nodes->Id[in->Nodes->reducedDegreesOfFreedomMapping->map[i]];
344 }
345 } else {
346 Finley_NodeMapping_free(in->Nodes->nodesMapping);
347 Finley_NodeMapping_free(in->Nodes->reducedNodesMapping);
348 Finley_NodeMapping_free(in->Nodes->degreesOfFreedomMapping);
349 Finley_NodeMapping_free(in->Nodes->reducedDegreesOfFreedomMapping);
350 Paso_Distribution_free(in->Nodes->nodesDistribution);
351 Paso_Distribution_free(in->Nodes->reducedNodesDistribution);
352 Paso_Distribution_free(in->Nodes->degreesOfFreedomDistribution);
353 Paso_Distribution_free(in->Nodes->reducedDegreesOfFreedomDistribution);
354 Paso_Coupler_free(in->Nodes->degreesOfFreedomCoupler);
355 Paso_Coupler_free(in->Nodes->reducedDegreesOfFreedomCoupler);
356 in->Nodes->nodesMapping=NULL;
357 in->Nodes->reducedNodesMapping=NULL;
358 in->Nodes->degreesOfFreedomMapping=NULL;
359 in->Nodes->reducedDegreesOfFreedomMapping=NULL;
360 in->Nodes->nodesDistribution=NULL;
361 in->Nodes->reducedNodesDistribution=NULL;
362 in->Nodes->degreesOfFreedomDistribution=NULL;
363 in->Nodes->reducedDegreesOfFreedomDistribution=NULL;
364 in->Nodes->degreesOfFreedomCoupler=NULL;
365 in->Nodes->reducedDegreesOfFreedomCoupler=NULL;
366 }
367 }
368

  ViewVC Help
Powered by ViewVC 1.1.26