/[escript]/branches/trilinos_from_5897/dudley/src/NodeFile_createMappings.cpp
ViewVC logotype

Contents of /branches/trilinos_from_5897/dudley/src/NodeFile_createMappings.cpp

Parent Directory Parent Directory | Revision Log Revision Log


Revision 6079 - (show annotations)
Mon Mar 21 12:22:38 2016 UTC (2 years, 11 months ago) by caltinay
File size: 9439 byte(s)
Big commit - making dudley much more like finley to make it more
managable. Fixed quite a few issues that had been fixed in finley.
Disposed of all ReducedNode/ReducedDOF entities that dudley never supported.
Compiles and passes tests.

1
2 /*****************************************************************************
3 *
4 * Copyright (c) 2003-2016 by The University of Queensland
5 * http://www.uq.edu.au
6 *
7 * Primary Business: Queensland, Australia
8 * Licensed under the Open Software License version 3.0
9 * http://www.opensource.org/licenses/osl-3.0.php
10 *
11 * Development until 2012 by Earth Systems Science Computational Center (ESSCC)
12 * Development 2012-2013 by School of Earth Sciences
13 * Development from 2014 by Centre for Geoscience Computing (GeoComp)
14 *
15 *****************************************************************************/
16
17 /****************************************************************************/
18
19 /* Dudley: NodeFile : creates the mappings using the indexReducedNodes */
20 /* no distribution is happening */
21
22 /****************************************************************************/
23
24 #include "Mesh.h"
25
26 namespace dudley {
27
28 void NodeFile::createDOFMappingAndCoupling()
29 {
30 const index_t myFirstDOF = dofDistribution->getFirstComponent();
31 const index_t myLastDOF = dofDistribution->getLastComponent();
32 const int mpiSize = MPIInfo->size;
33 const int myRank = MPIInfo->rank;
34
35 index_t min_DOF, max_DOF;
36 std::pair<index_t,index_t> DOF_range(util::getFlaggedMinMaxInt(
37 numNodes, globalDegreesOfFreedom, -1));
38
39 if (DOF_range.second < DOF_range.first) {
40 min_DOF = myFirstDOF;
41 max_DOF = myLastDOF - 1;
42 } else {
43 min_DOF = DOF_range.first;
44 max_DOF = DOF_range.second;
45 }
46
47 int p_min = mpiSize;
48 int p_max = -1;
49 if (max_DOF >= min_DOF) {
50 for (int p = 0; p < mpiSize; ++p) {
51 if (dofDistribution->first_component[p] <= min_DOF)
52 p_min = p;
53 if (dofDistribution->first_component[p] <= max_DOF)
54 p_max = p;
55 }
56 }
57
58 std::stringstream ss;
59 if (myFirstDOF<myLastDOF && !(min_DOF <= myFirstDOF && myLastDOF-1 <= max_DOF)) {
60 ss << "createDOFMappingAndCoupling: Local elements do not span local "
61 "degrees of freedom. min_DOF=" << min_DOF << ", myFirstDOF="
62 << myFirstDOF << ", myLastDOF-1=" << myLastDOF-1
63 << ", max_DOF=" << max_DOF << " on rank=" << MPIInfo->rank;
64 }
65 const std::string msg(ss.str());
66 int error = msg.length();
67 int gerror = error;
68 escript::checkResult(error, gerror, MPIInfo);
69 if (gerror > 0) {
70 char* gmsg;
71 escript::shipString(msg.c_str(), &gmsg, MPIInfo->comm);
72 throw DudleyException(gmsg);
73 }
74
75 const index_t UNUSED = -1;
76 const dim_t len_loc_dof = max_DOF - min_DOF + 1;
77 index_t* shared = new index_t[numNodes * (p_max - p_min + 1)];
78 std::vector<index_t> offsetInShared(mpiSize + 1);
79 index_t* locDOFMask = new index_t[len_loc_dof];
80 index_t* nodeMask = new index_t[numNodes];
81
82 ESYS_ASSERT(myLastDOF-min_DOF <= len_loc_dof, "out of bounds!");
83
84 #pragma omp parallel
85 {
86 #pragma omp for
87 for (index_t i = 0; i < len_loc_dof; ++i)
88 locDOFMask[i] = UNUSED;
89 #pragma omp for
90 for (index_t i = 0; i < numNodes; ++i)
91 nodeMask[i] = UNUSED;
92 #pragma omp for
93 for (index_t i = 0; i < numNodes; ++i) {
94 const index_t k = globalDegreesOfFreedom[i];
95 if (k > -1) {
96 #ifdef BOUNDS_CHECK
97 if ((k - min_DOF) >= len_loc_dof) {
98 printf("BOUNDS_CHECK %s %d i=%d k=%d min_DOF=%d\n", __FILE__, __LINE__, i, k, min_DOF);
99 exit(1);
100 }
101 #endif
102 locDOFMask[k - min_DOF] = UNUSED - 1;
103 }
104 }
105 #pragma omp for
106 for (index_t i = myFirstDOF - min_DOF; i < myLastDOF - min_DOF; ++i) {
107 locDOFMask[i] = i - myFirstDOF + min_DOF;
108 }
109 }
110
111 index_t* wanted_DOFs = new index_t[numNodes];
112 std::vector<index_t> rcv_len(mpiSize);
113 std::vector<index_t> snd_len(mpiSize);
114 std::vector<int> neighbor(mpiSize);
115 int numNeighbors = 0;
116 dim_t n = 0;
117 dim_t lastn = n;
118
119 for (int p = p_min; p <= p_max; ++p) {
120 if (p != myRank) {
121 const index_t firstDOF = std::max(min_DOF, dofDistribution->first_component[p]);
122 const index_t lastDOF = std::min(max_DOF + 1, dofDistribution->first_component[p + 1]);
123 #ifdef BOUNDS_CHECK
124 if (lastDOF-min_DOF > len_loc_dof) {
125 printf("BOUNDS_CHECK %s %d p=%d\n", __FILE__, __LINE__, p);
126 exit(1);
127 }
128 #endif
129 for (index_t i = firstDOF - min_DOF; i < lastDOF - min_DOF; ++i) {
130 if (locDOFMask[i] == UNUSED - 1) {
131 locDOFMask[i] = myLastDOF - myFirstDOF + n;
132 wanted_DOFs[n] = i + min_DOF;
133 ++n;
134 }
135 }
136 if (n > lastn) {
137 rcv_len[p] = n - lastn;
138 #ifdef BOUNDS_CHECK
139 if (numNeighbors >= mpiSize + 1) {
140 printf("BOUNDS_CHECK %s %d p=%d numNeighbors=%d n=%d\n", __FILE__, __LINE__, p, numNeighbors, n);
141 exit(1);
142 }
143 #endif
144 neighbor[numNeighbors] = p;
145 offsetInShared[numNeighbors] = lastn;
146 numNeighbors++;
147 lastn = n;
148 }
149 } // if p!=myRank
150 } // for p
151
152 #ifdef BOUNDS_CHECK
153 if (numNeighbors >= mpiSize + 1) {
154 printf("BOUNDS_CHECK %s %d numNeighbors=%d\n", __FILE__, __LINE__, numNeighbors);
155 exit(1);
156 }
157 #endif
158 offsetInShared[numNeighbors] = lastn;
159
160 // assign new DOF labels to nodes
161 #pragma omp parallel for
162 for (index_t i = 0; i < numNodes; ++i) {
163 const index_t k = globalDegreesOfFreedom[i];
164 if (k > -1)
165 nodeMask[i] = locDOFMask[k - min_DOF];
166 }
167
168 degreesOfFreedomMapping.assign(nodeMask, numNodes, UNUSED);
169
170 // define how to get DOF values for controlled but other processors
171 #ifdef BOUNDS_CHECK
172 if (numNodes && offsetInShared[numNeighbors] >= numNodes * (p_max - p_min + 1)) {
173 printf("BOUNDS_CHECK %s %d\n", __FILE__, __LINE__);
174 exit(1);
175 }
176 #endif
177 #pragma omp parallel for
178 for (index_t i = 0; i < lastn; ++i)
179 shared[i] = myLastDOF - myFirstDOF + i;
180
181 paso::SharedComponents_ptr rcv_shcomp(new paso::SharedComponents(
182 myLastDOF - myFirstDOF, numNeighbors, &neighbor[0], shared,
183 &offsetInShared[0], 1, 0, MPIInfo));
184
185 /////////////////////////////////
186 // now we build the sender //
187 /////////////////////////////////
188 #ifdef ESYS_MPI
189 std::vector<MPI_Request> mpi_requests(mpiSize * 2);
190 std::vector<MPI_Status> mpi_stati(mpiSize * 2);
191 MPI_Alltoall(&rcv_len[0], 1, MPI_DIM_T, &snd_len[0], 1, MPI_DIM_T,
192 MPIInfo->comm);
193 int count = 0;
194 #else
195 snd_len[0] = rcv_len[0];
196 #endif
197
198 for (int p = 0; p < rcv_shcomp->numNeighbors; p++) {
199 #ifdef ESYS_MPI
200 MPI_Isend(&wanted_DOFs[rcv_shcomp->offsetInShared[p]],
201 rcv_shcomp->offsetInShared[p+1]-rcv_shcomp->offsetInShared[p],
202 MPI_DIM_T, rcv_shcomp->neighbor[p],
203 MPIInfo->counter() + myRank, MPIInfo->comm,
204 &mpi_requests[count]);
205 count++;
206 #endif
207 }
208 n = 0;
209 numNeighbors = 0;
210 for (int p = 0; p < mpiSize; p++) {
211 if (snd_len[p] > 0) {
212 #ifdef ESYS_MPI
213 MPI_Irecv(&shared[n], snd_len[p], MPI_DIM_T, p,
214 MPIInfo->counter() + p, MPIInfo->comm,
215 &mpi_requests[count]);
216 count++;
217 #endif
218 neighbor[numNeighbors] = p;
219 offsetInShared[numNeighbors] = n;
220 numNeighbors++;
221 n += snd_len[p];
222 }
223 }
224 offsetInShared[numNeighbors] = n;
225 #ifdef ESYS_MPI
226 MPIInfo->incCounter(MPIInfo->size);
227 MPI_Waitall(count, &mpi_requests[0], &mpi_stati[0]);
228 #endif
229 // map global IDs to local IDs
230 #pragma omp parallel for
231 for (index_t i = 0; i < n; ++i) {
232 shared[i] = locDOFMask[shared[i] - min_DOF];
233 }
234
235 paso::SharedComponents_ptr snd_shcomp(new paso::SharedComponents(
236 myLastDOF - myFirstDOF, numNeighbors, &neighbor[0], shared,
237 &offsetInShared[0], 1, 0, MPIInfo));
238
239 degreesOfFreedomConnector.reset(new paso::Connector(snd_shcomp, rcv_shcomp));
240
241 delete[] wanted_DOFs;
242 delete[] nodeMask;
243 delete[] shared;
244 delete[] locDOFMask;
245 }
246
247 void NodeFile::createNodeMappings(const std::vector<index_t>& dofDist,
248 const std::vector<index_t>& nodeDist)
249 {
250 // ==== distribution of Nodes ====
251 nodesDistribution.reset(new paso::Distribution(
252 MPIInfo, &nodeDist[0], 1, 0));
253
254 // ==== distribution of DOFs ====
255 dofDistribution.reset(new paso::Distribution(MPIInfo, &dofDist[0], 1, 0));
256
257 index_t* nodeMask = new index_t[numNodes];
258 const index_t UNUSED = -1;
259
260 // ==== nodes mapping (dummy) ====
261 #pragma omp parallel for
262 for (index_t i = 0; i < numNodes; ++i)
263 nodeMask[i] = i;
264 nodesMapping.assign(nodeMask, numNodes, UNUSED);
265
266 // ==== mapping between nodes and DOFs + DOF connector ====
267 createDOFMappingAndCoupling();
268
269 // get the IDs for DOFs
270 #pragma omp parallel for
271 for (index_t i = 0; i < degreesOfFreedomMapping.numTargets; ++i)
272 degreesOfFreedomId[i] = Id[degreesOfFreedomMapping.map[i]];
273 }
274
275 } // namespace dudley
276

Properties

Name Value
svn:mergeinfo /branches/4.0fordebian/dudley/src/Mesh_createNodeFileMappings.cpp:5567-5588 /branches/lapack2681/finley/src/Mesh_createNodeFileMappings.cpp:2682-2741 /branches/pasowrap/dudley/src/Mesh_createNodeFileMappings.cpp:3661-3674 /branches/py3_attempt2/dudley/src/Mesh_createNodeFileMappings.cpp:3871-3891 /branches/restext/finley/src/Mesh_createNodeFileMappings.cpp:2610-2624 /branches/ripleygmg_from_3668/dudley/src/Mesh_createNodeFileMappings.cpp:3669-3791 /branches/stage3.0/finley/src/Mesh_createNodeFileMappings.cpp:2569-2590 /branches/symbolic_from_3470/dudley/src/Mesh_createNodeFileMappings.cpp:3471-3974 /branches/symbolic_from_3470/ripley/test/python/dudley/src/Mesh_createNodeFileMappings.cpp:3517-3974 /release/3.0/finley/src/Mesh_createNodeFileMappings.cpp:2591-2601 /release/4.0/dudley/src/Mesh_createNodeFileMappings.cpp:5380-5406 /trunk/dudley/src/Mesh_createNodeFileMappings.cpp:4257-4344,5898-6029 /trunk/ripley/test/python/dudley/src/Mesh_createNodeFileMappings.cpp:3480-3515

  ViewVC Help
Powered by ViewVC 1.1.26