/[escript]/branches/trilinos_from_5897/dudley/src/ElementFile_distributeByRankOfDOF.cpp
ViewVC logotype

Contents of /branches/trilinos_from_5897/dudley/src/ElementFile_distributeByRankOfDOF.cpp

Parent Directory Parent Directory | Revision Log Revision Log


Revision 6079 - (show annotations)
Mon Mar 21 12:22:38 2016 UTC (2 years, 11 months ago) by caltinay
File size: 7649 byte(s)
Big commit - making dudley much more like finley to make it more
managable. Fixed quite a few issues that had been fixed in finley.
Disposed of all ReducedNode/ReducedDOF entities that dudley never supported.
Compiles and passes tests.

1
2 /*****************************************************************************
3 *
4 * Copyright (c) 2003-2016 by The University of Queensland
5 * http://www.uq.edu.au
6 *
7 * Primary Business: Queensland, Australia
8 * Licensed under the Open Software License version 3.0
9 * http://www.opensource.org/licenses/osl-3.0.php
10 *
11 * Development until 2012 by Earth Systems Science Computational Center (ESSCC)
12 * Development 2012-2013 by School of Earth Sciences
13 * Development from 2014 by Centre for Geoscience Computing (GeoComp)
14 *
15 *****************************************************************************/
16
17 #include "ElementFile.h"
18
19 namespace dudley {
20
21 void ElementFile::distributeByRankOfDOF(const int* mpiRankOfDOF,
22 const index_t* nodesId)
23 {
24 const int size = MPIInfo->size;
25 if (size > 1) {
26 #ifdef ESYS_MPI
27 const int myRank = MPIInfo->rank;
28 int numRequests = 0;
29 std::vector<MPI_Request> mpi_requests(8 * size);
30 std::vector<MPI_Status> mpi_stati(8 * size);
31
32 // count the number elements that have to be sent to each processor
33 // (send_count) and define a new element owner as the processor with
34 // the largest number of DOFs and the smallest id
35 std::vector<dim_t> send_count(size);
36 std::vector<dim_t> recv_count(size);
37 int* newOwner = new int[numElements];
38 #pragma omp parallel
39 {
40 std::vector<dim_t> loc_proc_mask(size);
41 std::vector<dim_t> loc_send_count(size);
42 #pragma omp for
43 for (index_t e = 0; e < numElements; e++) {
44 if (Owner[e] == myRank) {
45 newOwner[e] = myRank;
46 loc_proc_mask.assign(size, 0);
47 for (int j = 0; j < numNodes; j++) {
48 const int p = mpiRankOfDOF[Nodes[INDEX2(j, e, numNodes)]];
49 loc_proc_mask[p]++;
50 }
51 dim_t loc_proc_mask_max = 0;
52 for (int p = 0; p < size; ++p) {
53 if (loc_proc_mask[p] > 0)
54 loc_send_count[p]++;
55 if (loc_proc_mask[p] > loc_proc_mask_max) {
56 newOwner[e] = p;
57 loc_proc_mask_max = loc_proc_mask[p];
58 }
59 }
60 } else {
61 newOwner[e] = -1;
62 }
63 }
64 #pragma omp critical
65 {
66 for (int p = 0; p < size; ++p)
67 send_count[p] += loc_send_count[p];
68 }
69 } // end parallel section
70 MPI_Alltoall(&send_count[0], 1, MPI_DIM_T, &recv_count[0], 1,
71 MPI_DIM_T, MPIInfo->comm);
72 // get the new number of elements for this processor
73 dim_t newNumElements = 0;
74 dim_t numElementsInBuffer = 0;
75 for (int p = 0; p < size; ++p) {
76 newNumElements += recv_count[p];
77 numElementsInBuffer += send_count[p];
78 }
79
80 std::vector<index_t> Id_buffer(numElementsInBuffer);
81 std::vector<int> Tag_buffer(numElementsInBuffer);
82 std::vector<int> Owner_buffer(numElementsInBuffer);
83 std::vector<index_t> Nodes_buffer(numElementsInBuffer * numNodes);
84 std::vector<index_t> send_offset(size);
85 std::vector<index_t> recv_offset(size);
86 std::vector<unsigned char> proc_mask(size);
87
88 // calculate the offsets for the processor buffers
89 for (int p = 0; p < size - 1; ++p) {
90 recv_offset[p + 1] = recv_offset[p] + recv_count[p];
91 send_offset[p + 1] = send_offset[p] + send_count[p];
92 }
93
94 send_count.assign(size, 0);
95 // copy element into buffers. proc_mask makes sure that an element is
96 // copied once only for each processor
97 for (index_t e = 0; e < numElements; e++) {
98 if (Owner[e] == myRank) {
99 proc_mask.assign(size, 1);
100 for (int j = 0; j < numNodes; j++) {
101 const int p = mpiRankOfDOF[Nodes[INDEX2(j, e, numNodes)]];
102 if (proc_mask[p]) {
103 const index_t k = send_offset[p] + send_count[p];
104 Id_buffer[k] = Id[e];
105 Tag_buffer[k] = Tag[e];
106 Owner_buffer[k] = newOwner[e];
107 for (int i = 0; i < numNodes; i++)
108 Nodes_buffer[INDEX2(i, k, numNodes)] =
109 nodesId[Nodes[INDEX2(i, e, numNodes)]];
110 send_count[p]++;
111 proc_mask[p] = 0;
112 }
113 }
114 }
115 }
116 // allocate new tables
117 allocTable(newNumElements);
118
119 // start to receive new elements
120 for (int p = 0; p < size; ++p) {
121 if (recv_count[p] > 0) {
122 MPI_Irecv(&Id[recv_offset[p]], recv_count[p], MPI_DIM_T, p,
123 MPIInfo->counter() + myRank, MPIInfo->comm,
124 &mpi_requests[numRequests]);
125 numRequests++;
126 MPI_Irecv(&Tag[recv_offset[p]], recv_count[p], MPI_INT, p,
127 MPIInfo->counter() + size + myRank, MPIInfo->comm,
128 &mpi_requests[numRequests]);
129 numRequests++;
130 MPI_Irecv(&Owner[recv_offset[p]], recv_count[p], MPI_INT, p,
131 MPIInfo->counter() + 2 * size + myRank,
132 MPIInfo->comm, &mpi_requests[numRequests]);
133 numRequests++;
134 MPI_Irecv(&Nodes[recv_offset[p] * numNodes],
135 recv_count[p] * numNodes, MPI_DIM_T, p,
136 MPIInfo->counter() + 3 * size + myRank,
137 MPIInfo->comm, &mpi_requests[numRequests]);
138 numRequests++;
139 }
140 }
141 // now the buffers can be sent away
142 for (int p = 0; p < size; ++p) {
143 if (send_count[p] > 0) {
144 MPI_Issend(&Id_buffer[send_offset[p]], send_count[p],
145 MPI_DIM_T, p, MPIInfo->counter() + p,
146 MPIInfo->comm, &mpi_requests[numRequests]);
147 numRequests++;
148 MPI_Issend(&Tag_buffer[send_offset[p]], send_count[p],
149 MPI_INT, p, MPIInfo->counter() + size + p,
150 MPIInfo->comm, &mpi_requests[numRequests]);
151 numRequests++;
152 MPI_Issend(&Owner_buffer[send_offset[p]], send_count[p],
153 MPI_INT, p, MPIInfo->counter() + 2 * size + p,
154 MPIInfo->comm, &mpi_requests[numRequests]);
155 numRequests++;
156 MPI_Issend(&Nodes_buffer[send_offset[p] * numNodes],
157 send_count[p] * numNodes, MPI_DIM_T, p,
158 MPIInfo->counter() + 3 * size + p,
159 MPIInfo->comm, &mpi_requests[numRequests]);
160 numRequests++;
161 }
162 }
163 MPIInfo->incCounter(4 * size);
164 // wait for the requests to be finalized
165 MPI_Waitall(numRequests, &mpi_requests[0], &mpi_stati[0]);
166 delete[] newOwner;
167 #endif
168 } else { // single rank
169 #pragma omp parallel for
170 for (index_t e = 0; e < numElements; e++) {
171 Owner[e] = 0;
172 for (int i = 0; i < numNodes; i++)
173 Nodes[INDEX2(i, e, numNodes)] =
174 nodesId[Nodes[INDEX2(i, e, numNodes)]];
175 }
176 }
177 }
178
179 } // namespace dudley
180

Properties

Name Value
svn:mergeinfo /branches/4.0fordebian/dudley/src/ElementFile_distributeByRankOfDOF.cpp:5567-5588 /branches/lapack2681/finley/src/ElementFile_distributeByRankOfDOF.cpp:2682-2741 /branches/pasowrap/dudley/src/ElementFile_distributeByRankOfDOF.cpp:3661-3674 /branches/py3_attempt2/dudley/src/ElementFile_distributeByRankOfDOF.cpp:3871-3891 /branches/restext/finley/src/ElementFile_distributeByRankOfDOF.cpp:2610-2624 /branches/ripleygmg_from_3668/dudley/src/ElementFile_distributeByRankOfDOF.cpp:3669-3791 /branches/stage3.0/finley/src/ElementFile_distributeByRankOfDOF.cpp:2569-2590 /branches/symbolic_from_3470/dudley/src/ElementFile_distributeByRankOfDOF.cpp:3471-3974 /branches/symbolic_from_3470/ripley/test/python/dudley/src/ElementFile_distributeByRankOfDOF.cpp:3517-3974 /release/3.0/finley/src/ElementFile_distributeByRankOfDOF.cpp:2591-2601 /release/4.0/dudley/src/ElementFile_distributeByRankOfDOF.cpp:5380-5406 /trunk/dudley/src/ElementFile_distributeByRankOfDOF.cpp:4257-4344,5898-6007 /trunk/ripley/test/python/dudley/src/ElementFile_distributeByRankOfDOF.cpp:3480-3515

  ViewVC Help
Powered by ViewVC 1.1.26