/[escript]/trunk/finley/src/ElementFile_distributeByRankOfDOF.c
ViewVC logotype

Contents of /trunk/finley/src/ElementFile_distributeByRankOfDOF.c

Parent Directory Parent Directory | Revision Log Revision Log


Revision 3603 - (show annotations)
Mon Sep 19 03:42:53 2011 UTC (7 years, 9 months ago) by caltinay
File MIME type: text/plain
File size: 10170 byte(s)
Fixed some unused var errors in non-mpi build.

1
2 /*******************************************************
3 *
4 * Copyright (c) 2003-2010 by University of Queensland
5 * Earth Systems Science Computational Center (ESSCC)
6 * http://www.uq.edu.au/esscc
7 *
8 * Primary Business: Queensland, Australia
9 * Licensed under the Open Software License version 3.0
10 * http://www.opensource.org/licenses/osl-3.0.php
11 *
12 *******************************************************/
13
14
15 /**************************************************************/
16
17 /* Finley: ElementFile: this will redistribute the Elements including overlap by */
18
19 /**************************************************************/
20
21 #include "ElementFile.h"
22 #ifdef _OPENMP
23 #include <omp.h>
24 #endif
25
26 /**************************************************************/
27
28 void Finley_ElementFile_distributeByRankOfDOF(Finley_ElementFile* self, Esys_MPI_rank* mpiRankOfDOF, index_t* Id) {
29 size_t size_size;
30 Esys_MPI_rank myRank, p, *Owner_buffer=NULL, loc_proc_mask_max;
31 dim_t e, j, i, size, *send_count=NULL, *recv_count=NULL, *newOwner=NULL, *loc_proc_mask=NULL, *loc_send_count=NULL,
32 newNumElements, numElementsInBuffer, numNodes, NN;
33 index_t *send_offset=NULL, *recv_offset=NULL, *Id_buffer=NULL, *Tag_buffer=NULL, *Nodes_buffer=NULL, k;
34 bool_t *proc_mask=NULL;
35 #ifdef ESYS_MPI
36 dim_t numRequests=0;
37 MPI_Request* mpi_requests=NULL;
38 MPI_Status* mpi_stati=NULL;
39 #endif
40 if (self==NULL) return;
41 myRank=self->MPIInfo->rank;
42 size=self->MPIInfo->size;
43 size_size=size*sizeof(dim_t);
44 numNodes=self->numNodes;
45 NN=self->numNodes;
46 if (size>1) {
47 #ifdef ESYS_MPI
48 mpi_requests=TMPMEMALLOC(8*size, MPI_Request);
49 mpi_stati=TMPMEMALLOC(8*size, MPI_Status);
50 Finley_checkPtr(mpi_requests);
51 Finley_checkPtr(mpi_stati);
52 #endif
53
54 /* count the number elements that have to be send to each processor (send_count)
55 and define a new element owner as the processor with the largest number of DOFs and the smallest id */
56 send_count=TMPMEMALLOC(size,dim_t);
57 recv_count=TMPMEMALLOC(size,dim_t);
58 newOwner=TMPMEMALLOC(self->numElements,Esys_MPI_rank);
59 if ( !( Finley_checkPtr(send_count) || Finley_checkPtr(recv_count) || Finley_checkPtr(newOwner) ) ) {
60 memset(send_count, 0, size_size);
61 #pragma omp parallel private(p,loc_proc_mask,loc_send_count)
62 {
63 loc_proc_mask=THREAD_MEMALLOC(size,dim_t);
64 loc_send_count=THREAD_MEMALLOC(size,dim_t);
65 memset(loc_send_count, 0, size_size);
66 #pragma omp for private(e,j,loc_proc_mask_max) schedule(static)
67 for (e=0;e<self->numElements;e++) {
68 if (self->Owner[e] == myRank) {
69 newOwner[e]=myRank;
70 memset(loc_proc_mask, 0, size_size);
71 for(j=0;j<numNodes;j++) {
72 p=mpiRankOfDOF[self->Nodes[INDEX2(j,e,NN)]];
73 loc_proc_mask[p]++;
74 }
75 loc_proc_mask_max=0;
76 for (p=0;p<size;++p) {
77 if (loc_proc_mask[p]>0) loc_send_count[p]++;
78 if (loc_proc_mask[p]>loc_proc_mask_max) {
79 newOwner[e]=p;
80 loc_proc_mask_max=loc_proc_mask[p];
81 }
82 }
83 } else {
84 newOwner[e]=-1;
85 }
86 }
87 #pragma omp critical
88 {
89 for (p=0;p<size;++p) send_count[p]+=loc_send_count[p];
90 }
91 THREAD_MEMFREE(loc_proc_mask);
92 THREAD_MEMFREE(loc_send_count);
93 }
94 #ifdef ESYS_MPI
95 MPI_Alltoall(send_count,1,MPI_INT,recv_count,1,MPI_INT,self->MPIInfo->comm);
96 #else
97 for (p=0;p<size;++p) recv_count[p]=send_count[p];
98 #endif
99 /* get the new number of elements for this processor */
100 newNumElements=0;
101 for (p=0;p<size;++p) newNumElements+=recv_count[p];
102
103 /* get the new number of elements for this processor */
104 numElementsInBuffer=0;
105 for (p=0;p<size;++p) numElementsInBuffer+=send_count[p];
106 /* allocate buffers */
107 Id_buffer=TMPMEMALLOC(numElementsInBuffer,index_t);
108 Tag_buffer=TMPMEMALLOC(numElementsInBuffer,index_t);
109 Owner_buffer=TMPMEMALLOC(numElementsInBuffer,Esys_MPI_rank);
110 Nodes_buffer=TMPMEMALLOC(numElementsInBuffer*NN,index_t);
111 send_offset=TMPMEMALLOC(size,index_t);
112 recv_offset=TMPMEMALLOC(size,index_t);
113 proc_mask=TMPMEMALLOC(size,bool_t);
114 if ( !( Finley_checkPtr(Id_buffer) || Finley_checkPtr(Tag_buffer) || Finley_checkPtr(Owner_buffer) ||
115 Finley_checkPtr(Nodes_buffer) || Finley_checkPtr(send_offset) || Finley_checkPtr(recv_offset) ||
116 Finley_checkPtr(proc_mask) )) {
117
118 /* callculate the offsets for the processor buffers */
119 recv_offset[0]=0;
120 for (p=0;p<size-1;++p) recv_offset[p+1]=recv_offset[p]+recv_count[p];
121 send_offset[0]=0;
122 for (p=0;p<size-1;++p) send_offset[p+1]=send_offset[p]+send_count[p];
123
124 memset(send_count, 0, size_size);
125 /* copy element into buffers. proc_mask makes sure that an element is copied once only for each processor */
126 for (e=0;e<self->numElements;e++) {
127 if (self->Owner[e] == myRank) {
128 memset(proc_mask, TRUE, size_size);
129 for(j=0;j<numNodes;j++) {
130 p=mpiRankOfDOF[self->Nodes[INDEX2(j,e,NN)]];
131 if (proc_mask[p]) {
132 k=send_offset[p]+send_count[p];
133 Id_buffer[k]=self->Id[e];
134 Tag_buffer[k]=self->Tag[e];
135 Owner_buffer[k]=newOwner[e];
136 for (i=0;i<numNodes;i++) Nodes_buffer[INDEX2(i,k,NN)]=Id[self->Nodes[INDEX2(i,e,NN)]];
137 send_count[p]++;
138 proc_mask[p]=FALSE;
139 }
140 }
141 }
142 }
143 /* allocate new tables */
144 Finley_ElementFile_allocTable(self,newNumElements);
145
146 /* start to receive new elements */
147 for (p=0;p<size;++p) {
148 if (recv_count[p]>0) {
149 #ifdef ESYS_MPI
150 MPI_Irecv(&(self->Id[recv_offset[p]]), recv_count[p],
151 MPI_INT, p, self->MPIInfo->msg_tag_counter+myRank,
152 self->MPIInfo->comm, &mpi_requests[numRequests]);
153 numRequests++;
154 MPI_Irecv(&(self->Tag[recv_offset[p]]), recv_count[p],
155 MPI_INT, p, self->MPIInfo->msg_tag_counter+size+myRank,
156 self->MPIInfo->comm, &mpi_requests[numRequests]);
157 numRequests++;
158 MPI_Irecv(&(self->Owner[recv_offset[p]]), recv_count[p],
159 MPI_INT, p, self->MPIInfo->msg_tag_counter+2*size+myRank,
160 self->MPIInfo->comm, &mpi_requests[numRequests]);
161 numRequests++;
162 MPI_Irecv(&(self->Nodes[recv_offset[p]*NN]), recv_count[p]*NN,
163 MPI_INT, p, self->MPIInfo->msg_tag_counter+3*size+myRank,
164 self->MPIInfo->comm, &mpi_requests[numRequests]);
165 numRequests++;
166 #endif
167 }
168 }
169 /* now the buffers can be send away */
170 for (p=0;p<size;++p) {
171 if (send_count[p]>0) {
172 #ifdef ESYS_MPI
173 MPI_Issend(&(Id_buffer[send_offset[p]]), send_count[p],
174 MPI_INT, p, self->MPIInfo->msg_tag_counter+p,
175 self->MPIInfo->comm, &mpi_requests[numRequests]);
176 numRequests++;
177 MPI_Issend(&(Tag_buffer[send_offset[p]]), send_count[p],
178 MPI_INT, p, self->MPIInfo->msg_tag_counter+size+p,
179 self->MPIInfo->comm, &mpi_requests[numRequests]);
180 numRequests++;
181 MPI_Issend(&(Owner_buffer[send_offset[p]]), send_count[p],
182 MPI_INT, p, self->MPIInfo->msg_tag_counter+2*size+p,
183 self->MPIInfo->comm, &mpi_requests[numRequests]);
184 numRequests++;
185 MPI_Issend(&(Nodes_buffer[send_offset[p]*NN]), send_count[p]*NN,
186 MPI_INT, p, self->MPIInfo->msg_tag_counter+3*size+p,
187 self->MPIInfo->comm, &mpi_requests[numRequests]);
188 numRequests++;
189 #endif
190
191 }
192 }
193 self->MPIInfo->msg_tag_counter+=4*size;
194 /* wait for the requests to be finalized */
195 #ifdef ESYS_MPI
196 MPI_Waitall(numRequests,mpi_requests,mpi_stati);
197 #endif
198 }
199 /* clear buffer */
200 TMPMEMFREE(Id_buffer);
201 TMPMEMFREE(Tag_buffer);
202 TMPMEMFREE(Owner_buffer);
203 TMPMEMFREE(Nodes_buffer);
204 TMPMEMFREE(send_offset);
205 TMPMEMFREE(recv_offset);
206 TMPMEMFREE(proc_mask);
207 }
208 #ifdef ESYS_MPI
209 TMPMEMFREE(mpi_requests);
210 TMPMEMFREE(mpi_stati);
211 #endif
212 TMPMEMFREE(send_count);
213 TMPMEMFREE(recv_count);
214 TMPMEMFREE(newOwner);
215 } else {
216 #pragma omp for private(e,i) schedule(static)
217 for (e=0;e<self->numElements;e++) {
218 self->Owner[e]=myRank;
219 for (i=0;i<numNodes;i++) self->Nodes[INDEX2(i,e,NN)]=Id[self->Nodes[INDEX2(i,e,NN)]];
220 }
221 }
222 return;
223 }
224

  ViewVC Help
Powered by ViewVC 1.1.26