/[escript]/trunk/finley/src/ElementFile_distributeByRankOfDOF.c
ViewVC logotype

Contents of /trunk/finley/src/ElementFile_distributeByRankOfDOF.c

Parent Directory Parent Directory | Revision Log Revision Log


Revision 1315 - (show annotations)
Tue Sep 25 02:41:13 2007 UTC (12 years ago) by ksteube
File MIME type: text/plain
File size: 10217 byte(s)
Copied more files from MPI branch to trunk

1
2 /* $Id$ */
3
4 /*******************************************************
5 *
6 * Copyright 2003-2007 by ACceSS MNRF
7 * Copyright 2007 by University of Queensland
8 *
9 * http://esscc.uq.edu.au
10 * Primary Business: Queensland, Australia
11 * Licensed under the Open Software License version 3.0
12 * http://www.opensource.org/licenses/osl-3.0.php
13 *
14 *******************************************************/
15
16 /**************************************************************/
17
18 /* Finley: ElementFile: this will redistribute the Elements including overlap by */
19
20 /**************************************************************/
21
22 #include "ElementFile.h"
23 #ifdef _OPENMP
24 #include <omp.h>
25 #endif
26
27 /**************************************************************/
28
29 void Finley_ElementFile_distributeByRankOfDOF(Finley_ElementFile* self, Paso_MPI_rank* mpiRankOfDOF, index_t* Id) {
30 size_t size_size;
31 Paso_MPI_rank myRank, p, *Owner_buffer=NULL, loc_proc_mask_max;
32 dim_t e, j, i, size, *send_count=NULL, *recv_count=NULL, *newOwner=NULL, *loc_proc_mask=NULL, *loc_send_count=NULL,
33 newNumElements, numElementsInBuffer, numNodes, numRequests, NN;
34 index_t *send_offset=NULL, *recv_offset=NULL, *Id_buffer=NULL, *Tag_buffer=NULL, *Nodes_buffer=NULL, k;
35 bool_t *proc_mask=NULL;
36 #ifdef PASO_MPI
37 MPI_Request* mpi_requests=NULL;
38 MPI_Status* mpi_stati=NULL;
39 #endif
40 if (self==NULL) return;
41 myRank=self->MPIInfo->rank;
42 size=self->MPIInfo->size;
43 size_size=size*sizeof(dim_t);
44 numNodes=self->numNodes;
45 NN=self->numNodes;
46 if (size>1) {
47 #ifdef PASO_MPI
48 mpi_requests=TMPMEMALLOC(8*size, MPI_Request);
49 mpi_stati=TMPMEMALLOC(8*size, MPI_Status);
50 Finley_checkPtr(mpi_requests);
51 Finley_checkPtr(mpi_stati);
52 #endif
53
54 /* count the number elements that have to be send to each processor (send_count)
55 and define a new element owner as the processor with the largest number of DOFs and the smallest id */
56 send_count=TMPMEMALLOC(size,dim_t);
57 recv_count=TMPMEMALLOC(size,dim_t);
58 newOwner=TMPMEMALLOC(self->numElements,Paso_MPI_rank);
59 if ( !( Finley_checkPtr(send_count) || Finley_checkPtr(recv_count) || Finley_checkPtr(newOwner) ) ) {
60 memset(send_count, 0, size_size);
61 #pragma omp parallel private(p,loc_proc_mask,loc_send_count)
62 {
63 loc_proc_mask=THREAD_MEMALLOC(size,dim_t);
64 loc_send_count=THREAD_MEMALLOC(size,dim_t);
65 memset(loc_send_count, 0, size_size);
66 #pragma omp for private(e,j,loc_proc_mask_max) schedule(static)
67 for (e=0;e<self->numElements;e++) {
68 if (self->Owner[e] == myRank) {
69 newOwner[e]=myRank;
70 memset(loc_proc_mask, 0, size_size);
71 for(j=0;j<numNodes;j++) {
72 p=mpiRankOfDOF[self->Nodes[INDEX2(j,e,NN)]];
73 loc_proc_mask[p]++;
74 }
75 loc_proc_mask_max=0;
76 for (p=0;p<size;++p) {
77 if (loc_proc_mask[p]>0) loc_send_count[p]++;
78 if (loc_proc_mask[p]>loc_proc_mask_max) {
79 newOwner[e]=p;
80 loc_proc_mask_max=loc_proc_mask[p];
81 }
82 }
83 } else {
84 newOwner[e]=-1;
85 }
86 }
87 #pragma omp critical
88 {
89 for (p=0;p<size;++p) send_count[p]+=loc_send_count[p];
90 }
91 THREAD_MEMFREE(loc_proc_mask);
92 THREAD_MEMFREE(loc_send_count);
93 }
94 #ifdef PASO_MPI
95 MPI_Alltoall(send_count,1,MPI_INT,recv_count,1,MPI_INT,self->MPIInfo->comm);
96 #else
97 for (p=0;p<size;++p) send_count[p]=recv_count[p];
98 #endif
99 /* get the new number of elements for this processor */
100 newNumElements=0;
101 for (p=0;p<size;++p) newNumElements+=recv_count[p];
102
103 /* get the new number of elements for this processor */
104 numElementsInBuffer=0;
105 for (p=0;p<size;++p) numElementsInBuffer+=send_count[p];
106 /* allocate buffers */
107 Id_buffer=TMPMEMALLOC(numElementsInBuffer,index_t);
108 Tag_buffer=TMPMEMALLOC(numElementsInBuffer,index_t);
109 Owner_buffer=TMPMEMALLOC(numElementsInBuffer,Paso_MPI_rank);
110 Nodes_buffer=TMPMEMALLOC(numElementsInBuffer*NN,index_t);
111 send_offset=TMPMEMALLOC(size,index_t);
112 recv_offset=TMPMEMALLOC(size,index_t);
113 proc_mask=TMPMEMALLOC(size,bool_t);
114 if ( !( Finley_checkPtr(Id_buffer) || Finley_checkPtr(Tag_buffer) || Finley_checkPtr(Owner_buffer) ||
115 Finley_checkPtr(Nodes_buffer) || Finley_checkPtr(send_offset) || Finley_checkPtr(recv_offset) ||
116 Finley_checkPtr(proc_mask) )) {
117
118 /* callculate the offsets for the processor buffers */
119 recv_offset[0]=0;
120 for (p=0;p<size-1;++p) recv_offset[p+1]=recv_offset[p]+recv_count[p];
121 send_offset[0]=0;
122 for (p=0;p<size-1;++p) send_offset[p+1]=send_offset[p]+send_count[p];
123
124 memset(send_count, 0, size_size);
125 /* copy element into buffers. proc_mask makes sure that an element is copied once only for each processor */
126 for (e=0;e<self->numElements;e++) {
127 if (self->Owner[e] == myRank) {
128 memset(proc_mask, TRUE, size_size);
129 for(j=0;j<numNodes;j++) {
130 p=mpiRankOfDOF[self->Nodes[INDEX2(j,e,NN)]];
131 if (proc_mask[p]) {
132 k=send_offset[p]+send_count[p];
133 Id_buffer[k]=self->Id[e];
134 Tag_buffer[k]=self->Tag[e];
135 Owner_buffer[k]=newOwner[e];
136 for (i=0;i<numNodes;i++) Nodes_buffer[INDEX2(i,k,NN)]=Id[self->Nodes[INDEX2(i,e,NN)]];
137 send_count[p]++;
138 proc_mask[p]=FALSE;
139 }
140 }
141 }
142 }
143 /* allocate new tables */
144 Finley_ElementFile_allocTable(self,newNumElements);
145
146 /* start to receive new elements */
147 numRequests=0;
148 for (p=0;p<size;++p) {
149 if (recv_count[p]>0) {
150 #ifdef PASO_MPI
151 MPI_Irecv(&(self->Id[recv_offset[p]]), recv_count[p],
152 MPI_INT, p, self->MPIInfo->msg_tag_counter+myRank,
153 self->MPIInfo->comm, &mpi_requests[numRequests]);
154 numRequests++;
155 MPI_Irecv(&(self->Tag[recv_offset[p]]), recv_count[p],
156 MPI_INT, p, self->MPIInfo->msg_tag_counter+size+myRank,
157 self->MPIInfo->comm, &mpi_requests[numRequests]);
158 numRequests++;
159 MPI_Irecv(&(self->Owner[recv_offset[p]]), recv_count[p],
160 MPI_INT, p, self->MPIInfo->msg_tag_counter+2*size+myRank,
161 self->MPIInfo->comm, &mpi_requests[numRequests]);
162 numRequests++;
163 MPI_Irecv(&(self->Nodes[recv_offset[p]*NN]), recv_count[p]*NN,
164 MPI_INT, p, self->MPIInfo->msg_tag_counter+3*size+myRank,
165 self->MPIInfo->comm, &mpi_requests[numRequests]);
166 numRequests++;
167 #endif
168 }
169 }
170 /* now the buffers can be send away */
171 for (p=0;p<size;++p) {
172 if (send_count[p]>0) {
173 #ifdef PASO_MPI
174 MPI_Issend(&(Id_buffer[send_offset[p]]), send_count[p],
175 MPI_INT, p, self->MPIInfo->msg_tag_counter+p,
176 self->MPIInfo->comm, &mpi_requests[numRequests]);
177 numRequests++;
178 MPI_Issend(&(Tag_buffer[send_offset[p]]), send_count[p],
179 MPI_INT, p, self->MPIInfo->msg_tag_counter+size+p,
180 self->MPIInfo->comm, &mpi_requests[numRequests]);
181 numRequests++;
182 MPI_Issend(&(Owner_buffer[send_offset[p]]), send_count[p],
183 MPI_INT, p, self->MPIInfo->msg_tag_counter+2*size+p,
184 self->MPIInfo->comm, &mpi_requests[numRequests]);
185 numRequests++;
186 MPI_Issend(&(Nodes_buffer[send_offset[p]*NN]), send_count[p]*NN,
187 MPI_INT, p, self->MPIInfo->msg_tag_counter+3*size+p,
188 self->MPIInfo->comm, &mpi_requests[numRequests]);
189 numRequests++;
190 #endif
191
192 }
193 }
194 self->MPIInfo->msg_tag_counter+=4*size;
195 /* wait for the requests to be finalized */
196 #ifdef PASO_MPI
197 MPI_Waitall(numRequests,mpi_requests,mpi_stati);
198 #endif
199 }
200 /* clear buffer */
201 TMPMEMFREE(Id_buffer);
202 TMPMEMFREE(Tag_buffer);
203 TMPMEMFREE(Owner_buffer);
204 TMPMEMFREE(Nodes_buffer);
205 TMPMEMFREE(send_offset);
206 TMPMEMFREE(recv_offset);
207 TMPMEMFREE(proc_mask);
208 }
209 #ifdef PASO_MPI
210 TMPMEMFREE(mpi_requests);
211 TMPMEMFREE(mpi_stati);
212 #endif
213 TMPMEMFREE(send_count);
214 TMPMEMFREE(recv_count);
215 TMPMEMFREE(newOwner);
216 } else {
217 #pragma omp for private(e,i) schedule(static)
218 for (e=0;e<self->numElements;e++) {
219 self->Owner[e]=myRank;
220 for (i=0;i<numNodes;i++) self->Nodes[INDEX2(i,e,NN)]=Id[self->Nodes[INDEX2(i,e,NN)]];
221 }
222 }
223 return;
224 }
225

  ViewVC Help
Powered by ViewVC 1.1.26