/[escript]/trunk/finley/src/ElementFile_distributeByRankOfDOF.c
ViewVC logotype

Contents of /trunk/finley/src/ElementFile_distributeByRankOfDOF.c

Parent Directory Parent Directory | Revision Log Revision Log


Revision 3259 - (show annotations)
Mon Oct 11 01:48:14 2010 UTC (9 years, 2 months ago) by jfenwick
File MIME type: text/plain
File size: 10183 byte(s)
Merging dudley and scons updates from branches

1
2 /*******************************************************
3 *
4 * Copyright (c) 2003-2010 by University of Queensland
5 * Earth Systems Science Computational Center (ESSCC)
6 * http://www.uq.edu.au/esscc
7 *
8 * Primary Business: Queensland, Australia
9 * Licensed under the Open Software License version 3.0
10 * http://www.opensource.org/licenses/osl-3.0.php
11 *
12 *******************************************************/
13
14
15 /**************************************************************/
16
17 /* Finley: ElementFile: this will redistribute the Elements including overlap by */
18
19 /**************************************************************/
20
21 #include "ElementFile.h"
22 #ifdef _OPENMP
23 #include <omp.h>
24 #endif
25
26 /**************************************************************/
27
28 void Finley_ElementFile_distributeByRankOfDOF(Finley_ElementFile* self, Esys_MPI_rank* mpiRankOfDOF, index_t* Id) {
29 size_t size_size;
30 Esys_MPI_rank myRank, p, *Owner_buffer=NULL, loc_proc_mask_max;
31 dim_t e, j, i, size, *send_count=NULL, *recv_count=NULL, *newOwner=NULL, *loc_proc_mask=NULL, *loc_send_count=NULL,
32 newNumElements, numElementsInBuffer, numNodes, numRequests, NN;
33 index_t *send_offset=NULL, *recv_offset=NULL, *Id_buffer=NULL, *Tag_buffer=NULL, *Nodes_buffer=NULL, k;
34 bool_t *proc_mask=NULL;
35 #ifdef ESYS_MPI
36 MPI_Request* mpi_requests=NULL;
37 MPI_Status* mpi_stati=NULL;
38 #endif
39 if (self==NULL) return;
40 myRank=self->MPIInfo->rank;
41 size=self->MPIInfo->size;
42 size_size=size*sizeof(dim_t);
43 numNodes=self->numNodes;
44 NN=self->numNodes;
45 if (size>1) {
46 #ifdef ESYS_MPI
47 mpi_requests=TMPMEMALLOC(8*size, MPI_Request);
48 mpi_stati=TMPMEMALLOC(8*size, MPI_Status);
49 Finley_checkPtr(mpi_requests);
50 Finley_checkPtr(mpi_stati);
51 #endif
52
53 /* count the number elements that have to be send to each processor (send_count)
54 and define a new element owner as the processor with the largest number of DOFs and the smallest id */
55 send_count=TMPMEMALLOC(size,dim_t);
56 recv_count=TMPMEMALLOC(size,dim_t);
57 newOwner=TMPMEMALLOC(self->numElements,Esys_MPI_rank);
58 if ( !( Finley_checkPtr(send_count) || Finley_checkPtr(recv_count) || Finley_checkPtr(newOwner) ) ) {
59 memset(send_count, 0, size_size);
60 #pragma omp parallel private(p,loc_proc_mask,loc_send_count)
61 {
62 loc_proc_mask=THREAD_MEMALLOC(size,dim_t);
63 loc_send_count=THREAD_MEMALLOC(size,dim_t);
64 memset(loc_send_count, 0, size_size);
65 #pragma omp for private(e,j,loc_proc_mask_max) schedule(static)
66 for (e=0;e<self->numElements;e++) {
67 if (self->Owner[e] == myRank) {
68 newOwner[e]=myRank;
69 memset(loc_proc_mask, 0, size_size);
70 for(j=0;j<numNodes;j++) {
71 p=mpiRankOfDOF[self->Nodes[INDEX2(j,e,NN)]];
72 loc_proc_mask[p]++;
73 }
74 loc_proc_mask_max=0;
75 for (p=0;p<size;++p) {
76 if (loc_proc_mask[p]>0) loc_send_count[p]++;
77 if (loc_proc_mask[p]>loc_proc_mask_max) {
78 newOwner[e]=p;
79 loc_proc_mask_max=loc_proc_mask[p];
80 }
81 }
82 } else {
83 newOwner[e]=-1;
84 }
85 }
86 #pragma omp critical
87 {
88 for (p=0;p<size;++p) send_count[p]+=loc_send_count[p];
89 }
90 THREAD_MEMFREE(loc_proc_mask);
91 THREAD_MEMFREE(loc_send_count);
92 }
93 #ifdef ESYS_MPI
94 MPI_Alltoall(send_count,1,MPI_INT,recv_count,1,MPI_INT,self->MPIInfo->comm);
95 #else
96 for (p=0;p<size;++p) recv_count[p]=send_count[p];
97 #endif
98 /* get the new number of elements for this processor */
99 newNumElements=0;
100 for (p=0;p<size;++p) newNumElements+=recv_count[p];
101
102 /* get the new number of elements for this processor */
103 numElementsInBuffer=0;
104 for (p=0;p<size;++p) numElementsInBuffer+=send_count[p];
105 /* allocate buffers */
106 Id_buffer=TMPMEMALLOC(numElementsInBuffer,index_t);
107 Tag_buffer=TMPMEMALLOC(numElementsInBuffer,index_t);
108 Owner_buffer=TMPMEMALLOC(numElementsInBuffer,Esys_MPI_rank);
109 Nodes_buffer=TMPMEMALLOC(numElementsInBuffer*NN,index_t);
110 send_offset=TMPMEMALLOC(size,index_t);
111 recv_offset=TMPMEMALLOC(size,index_t);
112 proc_mask=TMPMEMALLOC(size,bool_t);
113 if ( !( Finley_checkPtr(Id_buffer) || Finley_checkPtr(Tag_buffer) || Finley_checkPtr(Owner_buffer) ||
114 Finley_checkPtr(Nodes_buffer) || Finley_checkPtr(send_offset) || Finley_checkPtr(recv_offset) ||
115 Finley_checkPtr(proc_mask) )) {
116
117 /* callculate the offsets for the processor buffers */
118 recv_offset[0]=0;
119 for (p=0;p<size-1;++p) recv_offset[p+1]=recv_offset[p]+recv_count[p];
120 send_offset[0]=0;
121 for (p=0;p<size-1;++p) send_offset[p+1]=send_offset[p]+send_count[p];
122
123 memset(send_count, 0, size_size);
124 /* copy element into buffers. proc_mask makes sure that an element is copied once only for each processor */
125 for (e=0;e<self->numElements;e++) {
126 if (self->Owner[e] == myRank) {
127 memset(proc_mask, TRUE, size_size);
128 for(j=0;j<numNodes;j++) {
129 p=mpiRankOfDOF[self->Nodes[INDEX2(j,e,NN)]];
130 if (proc_mask[p]) {
131 k=send_offset[p]+send_count[p];
132 Id_buffer[k]=self->Id[e];
133 Tag_buffer[k]=self->Tag[e];
134 Owner_buffer[k]=newOwner[e];
135 for (i=0;i<numNodes;i++) Nodes_buffer[INDEX2(i,k,NN)]=Id[self->Nodes[INDEX2(i,e,NN)]];
136 send_count[p]++;
137 proc_mask[p]=FALSE;
138 }
139 }
140 }
141 }
142 /* allocate new tables */
143 Finley_ElementFile_allocTable(self,newNumElements);
144
145 /* start to receive new elements */
146 numRequests=0;
147 for (p=0;p<size;++p) {
148 if (recv_count[p]>0) {
149 #ifdef ESYS_MPI
150 MPI_Irecv(&(self->Id[recv_offset[p]]), recv_count[p],
151 MPI_INT, p, self->MPIInfo->msg_tag_counter+myRank,
152 self->MPIInfo->comm, &mpi_requests[numRequests]);
153 numRequests++;
154 MPI_Irecv(&(self->Tag[recv_offset[p]]), recv_count[p],
155 MPI_INT, p, self->MPIInfo->msg_tag_counter+size+myRank,
156 self->MPIInfo->comm, &mpi_requests[numRequests]);
157 numRequests++;
158 MPI_Irecv(&(self->Owner[recv_offset[p]]), recv_count[p],
159 MPI_INT, p, self->MPIInfo->msg_tag_counter+2*size+myRank,
160 self->MPIInfo->comm, &mpi_requests[numRequests]);
161 numRequests++;
162 MPI_Irecv(&(self->Nodes[recv_offset[p]*NN]), recv_count[p]*NN,
163 MPI_INT, p, self->MPIInfo->msg_tag_counter+3*size+myRank,
164 self->MPIInfo->comm, &mpi_requests[numRequests]);
165 numRequests++;
166 #endif
167 }
168 }
169 /* now the buffers can be send away */
170 for (p=0;p<size;++p) {
171 if (send_count[p]>0) {
172 #ifdef ESYS_MPI
173 MPI_Issend(&(Id_buffer[send_offset[p]]), send_count[p],
174 MPI_INT, p, self->MPIInfo->msg_tag_counter+p,
175 self->MPIInfo->comm, &mpi_requests[numRequests]);
176 numRequests++;
177 MPI_Issend(&(Tag_buffer[send_offset[p]]), send_count[p],
178 MPI_INT, p, self->MPIInfo->msg_tag_counter+size+p,
179 self->MPIInfo->comm, &mpi_requests[numRequests]);
180 numRequests++;
181 MPI_Issend(&(Owner_buffer[send_offset[p]]), send_count[p],
182 MPI_INT, p, self->MPIInfo->msg_tag_counter+2*size+p,
183 self->MPIInfo->comm, &mpi_requests[numRequests]);
184 numRequests++;
185 MPI_Issend(&(Nodes_buffer[send_offset[p]*NN]), send_count[p]*NN,
186 MPI_INT, p, self->MPIInfo->msg_tag_counter+3*size+p,
187 self->MPIInfo->comm, &mpi_requests[numRequests]);
188 numRequests++;
189 #endif
190
191 }
192 }
193 self->MPIInfo->msg_tag_counter+=4*size;
194 /* wait for the requests to be finalized */
195 #ifdef ESYS_MPI
196 MPI_Waitall(numRequests,mpi_requests,mpi_stati);
197 #endif
198 }
199 /* clear buffer */
200 TMPMEMFREE(Id_buffer);
201 TMPMEMFREE(Tag_buffer);
202 TMPMEMFREE(Owner_buffer);
203 TMPMEMFREE(Nodes_buffer);
204 TMPMEMFREE(send_offset);
205 TMPMEMFREE(recv_offset);
206 TMPMEMFREE(proc_mask);
207 }
208 #ifdef ESYS_MPI
209 TMPMEMFREE(mpi_requests);
210 TMPMEMFREE(mpi_stati);
211 #endif
212 TMPMEMFREE(send_count);
213 TMPMEMFREE(recv_count);
214 TMPMEMFREE(newOwner);
215 } else {
216 #pragma omp for private(e,i) schedule(static)
217 for (e=0;e<self->numElements;e++) {
218 self->Owner[e]=myRank;
219 for (i=0;i<numNodes;i++) self->Nodes[INDEX2(i,e,NN)]=Id[self->Nodes[INDEX2(i,e,NN)]];
220 }
221 }
222 return;
223 }
224

  ViewVC Help
Powered by ViewVC 1.1.26