/[escript]/trunk-mpi-branch/finley/src/ElementFile_distributeByRankOfDOF.c
ViewVC logotype

Contents of /trunk-mpi-branch/finley/src/ElementFile_distributeByRankOfDOF.c

Parent Directory Parent Directory | Revision Log Revision Log


Revision 1301 - (show annotations)
Mon Sep 17 04:13:48 2007 UTC (12 years, 8 months ago) by ksteube
File MIME type: text/plain
File size: 10468 byte(s)
Lumping was not working because a loop over colors was missing.
Also resolved a few more compiler warnings that turned up.

1 /*
2 ************************************************************
3 * Copyright 2007 by ACcESS MNRF *
4 * *
5 * http://www.access.edu.au *
6 * Primary Business: Queensland, Australia *
7 * Licensed under the Open Software License version 3.0 *
8 * http://www.opensource.org/licenses/osl-3.0.php *
9 * *
10 ************************************************************
11 */
12
13 /**************************************************************/
14
15 /* Finley: ElementFile: this will redistribute the Elements including overlap by */
16
17 /**************************************************************/
18
19 /* Author: gross@access.edu.au */
20 /* Version: $Id:$ */
21
22 /**************************************************************/
23
24 #include "ElementFile.h"
25 #ifdef _OPENMP
26 #include <omp.h>
27 #endif
28
29 /**************************************************************/
30
31 void Finley_ElementFile_distributeByRankOfDOF(Finley_ElementFile* self, Paso_MPI_rank* mpiRankOfDOF, index_t* Id) {
32 size_t size_size;
33 Paso_MPI_rank myRank, p, *Owner_buffer=NULL, loc_proc_mask_max;
34 dim_t e, j, i, size, *send_count=NULL, *recv_count=NULL, *newOwner=NULL, *loc_proc_mask=NULL, *loc_send_count=NULL,
35 newNumElements, numElementsInBuffer, numNodes, numRequests, NN;
36 index_t *send_offset=NULL, *recv_offset=NULL, *Id_buffer=NULL, *Tag_buffer=NULL, *Nodes_buffer=NULL, k;
37 bool_t *proc_mask=NULL;
38 #ifdef PASO_MPI
39 MPI_Request* mpi_requests=NULL;
40 MPI_Status* mpi_stati=NULL;
41 #endif
42 if (self==NULL) return;
43 myRank=self->MPIInfo->rank;
44 size=self->MPIInfo->size;
45 size_size=size*sizeof(dim_t);
46 numNodes=self->numNodes;
47 NN=self->numNodes;
48 if (size>1) {
49 #ifdef PASO_MPI
50 mpi_requests=TMPMEMALLOC(8*size, MPI_Request);
51 mpi_stati=TMPMEMALLOC(8*size, MPI_Status);
52 Finley_checkPtr(mpi_requests);
53 Finley_checkPtr(mpi_stati);
54 #endif
55
56 /* count the number elements that have to be send to each processor (send_count)
57 and define a new element owner as the processor with the largest number of DOFs and the smallest id */
58 send_count=TMPMEMALLOC(size,dim_t);
59 recv_count=TMPMEMALLOC(size,dim_t);
60 newOwner=TMPMEMALLOC(self->numElements,Paso_MPI_rank);
61 if ( !( Finley_checkPtr(send_count) || Finley_checkPtr(recv_count) || Finley_checkPtr(newOwner) ) ) {
62 memset(send_count, 0, size_size);
63 #pragma omp parallel private(p,loc_proc_mask,loc_send_count)
64 {
65 loc_proc_mask=THREAD_MEMALLOC(size,dim_t);
66 loc_send_count=THREAD_MEMALLOC(size,dim_t);
67 memset(loc_send_count, 0, size_size);
68 #pragma omp for private(e,j,loc_proc_mask_max) schedule(static)
69 for (e=0;e<self->numElements;e++) {
70 if (self->Owner[e] == myRank) {
71 newOwner[e]=myRank;
72 memset(loc_proc_mask, 0, size_size);
73 for(j=0;j<numNodes;j++) {
74 p=mpiRankOfDOF[self->Nodes[INDEX2(j,e,NN)]];
75 loc_proc_mask[p]++;
76 }
77 loc_proc_mask_max=0;
78 for (p=0;p<size;++p) {
79 if (loc_proc_mask[p]>0) loc_send_count[p]++;
80 if (loc_proc_mask[p]>loc_proc_mask_max) {
81 newOwner[e]=p;
82 loc_proc_mask_max=loc_proc_mask[p];
83 }
84 }
85 } else {
86 newOwner[e]=-1;
87 }
88 }
89 #pragma omp critical
90 {
91 for (p=0;p<size;++p) send_count[p]+=loc_send_count[p];
92 }
93 THREAD_MEMFREE(loc_proc_mask);
94 THREAD_MEMFREE(loc_send_count);
95 }
96 #ifdef PASO_MPI
97 MPI_Alltoall(send_count,1,MPI_INT,recv_count,1,MPI_INT,self->MPIInfo->comm);
98 #else
99 for (p=0;p<size;++p) send_count[p]=recv_count[p];
100 #endif
101 /* get the new number of elements for this processor */
102 newNumElements=0;
103 for (p=0;p<size;++p) newNumElements+=recv_count[p];
104
105 /* get the new number of elements for this processor */
106 numElementsInBuffer=0;
107 for (p=0;p<size;++p) numElementsInBuffer+=send_count[p];
108 /* allocate buffers */
109 Id_buffer=TMPMEMALLOC(numElementsInBuffer,index_t);
110 Tag_buffer=TMPMEMALLOC(numElementsInBuffer,index_t);
111 Owner_buffer=TMPMEMALLOC(numElementsInBuffer,Paso_MPI_rank);
112 Nodes_buffer=TMPMEMALLOC(numElementsInBuffer*NN,index_t);
113 send_offset=TMPMEMALLOC(size,index_t);
114 recv_offset=TMPMEMALLOC(size,index_t);
115 proc_mask=TMPMEMALLOC(size,bool_t);
116 if ( !( Finley_checkPtr(Id_buffer) || Finley_checkPtr(Tag_buffer) || Finley_checkPtr(Owner_buffer) ||
117 Finley_checkPtr(Nodes_buffer) || Finley_checkPtr(send_offset) || Finley_checkPtr(recv_offset) ||
118 Finley_checkPtr(proc_mask) )) {
119
120 /* callculate the offsets for the processor buffers */
121 recv_offset[0]=0;
122 for (p=0;p<size-1;++p) recv_offset[p+1]=recv_offset[p]+recv_count[p];
123 send_offset[0]=0;
124 for (p=0;p<size-1;++p) send_offset[p+1]=send_offset[p]+send_count[p];
125
126 memset(send_count, 0, size_size);
127 /* copy element into buffers. proc_mask makes sure that an element is copied once only for each processor */
128 for (e=0;e<self->numElements;e++) {
129 if (self->Owner[e] == myRank) {
130 memset(proc_mask, TRUE, size_size);
131 for(j=0;j<numNodes;j++) {
132 p=mpiRankOfDOF[self->Nodes[INDEX2(j,e,NN)]];
133 if (proc_mask[p]) {
134 k=send_offset[p]+send_count[p];
135 Id_buffer[k]=self->Id[e];
136 Tag_buffer[k]=self->Tag[e];
137 Owner_buffer[k]=newOwner[e];
138 for (i=0;i<numNodes;i++) Nodes_buffer[INDEX2(i,k,NN)]=Id[self->Nodes[INDEX2(i,e,NN)]];
139 send_count[p]++;
140 proc_mask[p]=FALSE;
141 }
142 }
143 }
144 }
145 /* allocate new tables */
146 Finley_ElementFile_allocTable(self,newNumElements);
147
148 /* start to receive new elements */
149 numRequests=0;
150 for (p=0;p<size;++p) {
151 if (recv_count[p]>0) {
152 #ifdef PASO_MPI
153 MPI_Irecv(&(self->Id[recv_offset[p]]), recv_count[p],
154 MPI_INT, p, self->MPIInfo->msg_tag_counter+myRank,
155 self->MPIInfo->comm, &mpi_requests[numRequests]);
156 numRequests++;
157 MPI_Irecv(&(self->Tag[recv_offset[p]]), recv_count[p],
158 MPI_INT, p, self->MPIInfo->msg_tag_counter+size+myRank,
159 self->MPIInfo->comm, &mpi_requests[numRequests]);
160 numRequests++;
161 MPI_Irecv(&(self->Owner[recv_offset[p]]), recv_count[p],
162 MPI_INT, p, self->MPIInfo->msg_tag_counter+2*size+myRank,
163 self->MPIInfo->comm, &mpi_requests[numRequests]);
164 numRequests++;
165 MPI_Irecv(&(self->Nodes[recv_offset[p]*NN]), recv_count[p]*NN,
166 MPI_INT, p, self->MPIInfo->msg_tag_counter+3*size+myRank,
167 self->MPIInfo->comm, &mpi_requests[numRequests]);
168 numRequests++;
169 #endif
170 }
171 }
172 /* now the buffers can be send away */
173 for (p=0;p<size;++p) {
174 if (send_count[p]>0) {
175 #ifdef PASO_MPI
176 MPI_Issend(&(Id_buffer[send_offset[p]]), send_count[p],
177 MPI_INT, p, self->MPIInfo->msg_tag_counter+p,
178 self->MPIInfo->comm, &mpi_requests[numRequests]);
179 numRequests++;
180 MPI_Issend(&(Tag_buffer[send_offset[p]]), send_count[p],
181 MPI_INT, p, self->MPIInfo->msg_tag_counter+size+p,
182 self->MPIInfo->comm, &mpi_requests[numRequests]);
183 numRequests++;
184 MPI_Issend(&(Owner_buffer[send_offset[p]]), send_count[p],
185 MPI_INT, p, self->MPIInfo->msg_tag_counter+2*size+p,
186 self->MPIInfo->comm, &mpi_requests[numRequests]);
187 numRequests++;
188 MPI_Issend(&(Nodes_buffer[send_offset[p]*NN]), send_count[p]*NN,
189 MPI_INT, p, self->MPIInfo->msg_tag_counter+3*size+p,
190 self->MPIInfo->comm, &mpi_requests[numRequests]);
191 numRequests++;
192 #endif
193
194 }
195 }
196 self->MPIInfo->msg_tag_counter+=4*size;
197 /* wait for the requests to be finalized */
198 #ifdef PASO_MPI
199 MPI_Waitall(numRequests,mpi_requests,mpi_stati);
200 #endif
201 }
202 /* clear buffer */
203 TMPMEMFREE(Id_buffer);
204 TMPMEMFREE(Tag_buffer);
205 TMPMEMFREE(Owner_buffer);
206 TMPMEMFREE(Nodes_buffer);
207 TMPMEMFREE(send_offset);
208 TMPMEMFREE(recv_offset);
209 TMPMEMFREE(proc_mask);
210 }
211 #ifdef PASO_MPI
212 TMPMEMFREE(mpi_requests);
213 TMPMEMFREE(mpi_stati);
214 #endif
215 TMPMEMFREE(send_count);
216 TMPMEMFREE(recv_count);
217 TMPMEMFREE(newOwner);
218 } else {
219 #pragma omp for private(e,i) schedule(static)
220 for (e=0;e<self->numElements;e++) {
221 self->Owner[e]=myRank;
222 for (i=0;i<numNodes;i++) self->Nodes[INDEX2(i,e,NN)]=Id[self->Nodes[INDEX2(i,e,NN)]];
223 }
224 }
225 return;
226 }
227

  ViewVC Help
Powered by ViewVC 1.1.26