/[escript]/trunk/finley/src/ElementFile_distributeByRankOfDOF.c
ViewVC logotype

Annotation of /trunk/finley/src/ElementFile_distributeByRankOfDOF.c

Parent Directory Parent Directory | Revision Log Revision Log


Revision 2548 - (hide annotations)
Mon Jul 20 06:20:06 2009 UTC (9 years, 11 months ago) by jfenwick
File MIME type: text/plain
File size: 10183 byte(s)
Updating copyright notices
1 ksteube 1315
2     /*******************************************************
3 ksteube 1811 *
4 jfenwick 2548 * Copyright (c) 2003-2009 by University of Queensland
5 ksteube 1811 * Earth Systems Science Computational Center (ESSCC)
6     * http://www.uq.edu.au/esscc
7     *
8     * Primary Business: Queensland, Australia
9     * Licensed under the Open Software License version 3.0
10     * http://www.opensource.org/licenses/osl-3.0.php
11     *
12     *******************************************************/
13 ksteube 1315
14 ksteube 1811
15 ksteube 1315 /**************************************************************/
16    
17     /* Finley: ElementFile: this will redistribute the Elements including overlap by */
18    
19     /**************************************************************/
20    
21     #include "ElementFile.h"
22     #ifdef _OPENMP
23     #include <omp.h>
24     #endif
25    
26     /**************************************************************/
27    
28     void Finley_ElementFile_distributeByRankOfDOF(Finley_ElementFile* self, Paso_MPI_rank* mpiRankOfDOF, index_t* Id) {
29     size_t size_size;
30     Paso_MPI_rank myRank, p, *Owner_buffer=NULL, loc_proc_mask_max;
31     dim_t e, j, i, size, *send_count=NULL, *recv_count=NULL, *newOwner=NULL, *loc_proc_mask=NULL, *loc_send_count=NULL,
32     newNumElements, numElementsInBuffer, numNodes, numRequests, NN;
33     index_t *send_offset=NULL, *recv_offset=NULL, *Id_buffer=NULL, *Tag_buffer=NULL, *Nodes_buffer=NULL, k;
34     bool_t *proc_mask=NULL;
35     #ifdef PASO_MPI
36     MPI_Request* mpi_requests=NULL;
37     MPI_Status* mpi_stati=NULL;
38     #endif
39     if (self==NULL) return;
40     myRank=self->MPIInfo->rank;
41     size=self->MPIInfo->size;
42     size_size=size*sizeof(dim_t);
43     numNodes=self->numNodes;
44     NN=self->numNodes;
45     if (size>1) {
46     #ifdef PASO_MPI
47     mpi_requests=TMPMEMALLOC(8*size, MPI_Request);
48     mpi_stati=TMPMEMALLOC(8*size, MPI_Status);
49     Finley_checkPtr(mpi_requests);
50     Finley_checkPtr(mpi_stati);
51     #endif
52    
53     /* count the number elements that have to be send to each processor (send_count)
54     and define a new element owner as the processor with the largest number of DOFs and the smallest id */
55     send_count=TMPMEMALLOC(size,dim_t);
56     recv_count=TMPMEMALLOC(size,dim_t);
57     newOwner=TMPMEMALLOC(self->numElements,Paso_MPI_rank);
58     if ( !( Finley_checkPtr(send_count) || Finley_checkPtr(recv_count) || Finley_checkPtr(newOwner) ) ) {
59     memset(send_count, 0, size_size);
60     #pragma omp parallel private(p,loc_proc_mask,loc_send_count)
61     {
62     loc_proc_mask=THREAD_MEMALLOC(size,dim_t);
63     loc_send_count=THREAD_MEMALLOC(size,dim_t);
64 gross 1764 memset(loc_send_count, 0, size_size);
65 ksteube 1315 #pragma omp for private(e,j,loc_proc_mask_max) schedule(static)
66     for (e=0;e<self->numElements;e++) {
67     if (self->Owner[e] == myRank) {
68     newOwner[e]=myRank;
69     memset(loc_proc_mask, 0, size_size);
70     for(j=0;j<numNodes;j++) {
71     p=mpiRankOfDOF[self->Nodes[INDEX2(j,e,NN)]];
72     loc_proc_mask[p]++;
73     }
74     loc_proc_mask_max=0;
75     for (p=0;p<size;++p) {
76     if (loc_proc_mask[p]>0) loc_send_count[p]++;
77     if (loc_proc_mask[p]>loc_proc_mask_max) {
78     newOwner[e]=p;
79     loc_proc_mask_max=loc_proc_mask[p];
80     }
81     }
82     } else {
83     newOwner[e]=-1;
84     }
85     }
86     #pragma omp critical
87     {
88     for (p=0;p<size;++p) send_count[p]+=loc_send_count[p];
89     }
90     THREAD_MEMFREE(loc_proc_mask);
91     THREAD_MEMFREE(loc_send_count);
92     }
93     #ifdef PASO_MPI
94     MPI_Alltoall(send_count,1,MPI_INT,recv_count,1,MPI_INT,self->MPIInfo->comm);
95     #else
96 gross 1764 for (p=0;p<size;++p) recv_count[p]=send_count[p];
97 ksteube 1315 #endif
98     /* get the new number of elements for this processor */
99     newNumElements=0;
100     for (p=0;p<size;++p) newNumElements+=recv_count[p];
101    
102     /* get the new number of elements for this processor */
103     numElementsInBuffer=0;
104     for (p=0;p<size;++p) numElementsInBuffer+=send_count[p];
105     /* allocate buffers */
106     Id_buffer=TMPMEMALLOC(numElementsInBuffer,index_t);
107     Tag_buffer=TMPMEMALLOC(numElementsInBuffer,index_t);
108     Owner_buffer=TMPMEMALLOC(numElementsInBuffer,Paso_MPI_rank);
109     Nodes_buffer=TMPMEMALLOC(numElementsInBuffer*NN,index_t);
110     send_offset=TMPMEMALLOC(size,index_t);
111     recv_offset=TMPMEMALLOC(size,index_t);
112     proc_mask=TMPMEMALLOC(size,bool_t);
113     if ( !( Finley_checkPtr(Id_buffer) || Finley_checkPtr(Tag_buffer) || Finley_checkPtr(Owner_buffer) ||
114     Finley_checkPtr(Nodes_buffer) || Finley_checkPtr(send_offset) || Finley_checkPtr(recv_offset) ||
115     Finley_checkPtr(proc_mask) )) {
116    
117     /* callculate the offsets for the processor buffers */
118     recv_offset[0]=0;
119     for (p=0;p<size-1;++p) recv_offset[p+1]=recv_offset[p]+recv_count[p];
120     send_offset[0]=0;
121     for (p=0;p<size-1;++p) send_offset[p+1]=send_offset[p]+send_count[p];
122    
123     memset(send_count, 0, size_size);
124     /* copy element into buffers. proc_mask makes sure that an element is copied once only for each processor */
125     for (e=0;e<self->numElements;e++) {
126     if (self->Owner[e] == myRank) {
127     memset(proc_mask, TRUE, size_size);
128     for(j=0;j<numNodes;j++) {
129     p=mpiRankOfDOF[self->Nodes[INDEX2(j,e,NN)]];
130     if (proc_mask[p]) {
131     k=send_offset[p]+send_count[p];
132     Id_buffer[k]=self->Id[e];
133     Tag_buffer[k]=self->Tag[e];
134     Owner_buffer[k]=newOwner[e];
135     for (i=0;i<numNodes;i++) Nodes_buffer[INDEX2(i,k,NN)]=Id[self->Nodes[INDEX2(i,e,NN)]];
136     send_count[p]++;
137     proc_mask[p]=FALSE;
138     }
139     }
140     }
141     }
142     /* allocate new tables */
143     Finley_ElementFile_allocTable(self,newNumElements);
144    
145     /* start to receive new elements */
146     numRequests=0;
147     for (p=0;p<size;++p) {
148     if (recv_count[p]>0) {
149     #ifdef PASO_MPI
150     MPI_Irecv(&(self->Id[recv_offset[p]]), recv_count[p],
151     MPI_INT, p, self->MPIInfo->msg_tag_counter+myRank,
152     self->MPIInfo->comm, &mpi_requests[numRequests]);
153     numRequests++;
154     MPI_Irecv(&(self->Tag[recv_offset[p]]), recv_count[p],
155     MPI_INT, p, self->MPIInfo->msg_tag_counter+size+myRank,
156     self->MPIInfo->comm, &mpi_requests[numRequests]);
157     numRequests++;
158     MPI_Irecv(&(self->Owner[recv_offset[p]]), recv_count[p],
159     MPI_INT, p, self->MPIInfo->msg_tag_counter+2*size+myRank,
160     self->MPIInfo->comm, &mpi_requests[numRequests]);
161     numRequests++;
162     MPI_Irecv(&(self->Nodes[recv_offset[p]*NN]), recv_count[p]*NN,
163     MPI_INT, p, self->MPIInfo->msg_tag_counter+3*size+myRank,
164     self->MPIInfo->comm, &mpi_requests[numRequests]);
165     numRequests++;
166     #endif
167     }
168     }
169     /* now the buffers can be send away */
170     for (p=0;p<size;++p) {
171     if (send_count[p]>0) {
172     #ifdef PASO_MPI
173     MPI_Issend(&(Id_buffer[send_offset[p]]), send_count[p],
174     MPI_INT, p, self->MPIInfo->msg_tag_counter+p,
175     self->MPIInfo->comm, &mpi_requests[numRequests]);
176     numRequests++;
177     MPI_Issend(&(Tag_buffer[send_offset[p]]), send_count[p],
178     MPI_INT, p, self->MPIInfo->msg_tag_counter+size+p,
179     self->MPIInfo->comm, &mpi_requests[numRequests]);
180     numRequests++;
181     MPI_Issend(&(Owner_buffer[send_offset[p]]), send_count[p],
182     MPI_INT, p, self->MPIInfo->msg_tag_counter+2*size+p,
183     self->MPIInfo->comm, &mpi_requests[numRequests]);
184     numRequests++;
185     MPI_Issend(&(Nodes_buffer[send_offset[p]*NN]), send_count[p]*NN,
186     MPI_INT, p, self->MPIInfo->msg_tag_counter+3*size+p,
187     self->MPIInfo->comm, &mpi_requests[numRequests]);
188     numRequests++;
189     #endif
190    
191     }
192     }
193     self->MPIInfo->msg_tag_counter+=4*size;
194     /* wait for the requests to be finalized */
195     #ifdef PASO_MPI
196     MPI_Waitall(numRequests,mpi_requests,mpi_stati);
197     #endif
198     }
199     /* clear buffer */
200     TMPMEMFREE(Id_buffer);
201     TMPMEMFREE(Tag_buffer);
202     TMPMEMFREE(Owner_buffer);
203     TMPMEMFREE(Nodes_buffer);
204     TMPMEMFREE(send_offset);
205     TMPMEMFREE(recv_offset);
206     TMPMEMFREE(proc_mask);
207     }
208     #ifdef PASO_MPI
209     TMPMEMFREE(mpi_requests);
210     TMPMEMFREE(mpi_stati);
211     #endif
212     TMPMEMFREE(send_count);
213     TMPMEMFREE(recv_count);
214     TMPMEMFREE(newOwner);
215     } else {
216     #pragma omp for private(e,i) schedule(static)
217     for (e=0;e<self->numElements;e++) {
218     self->Owner[e]=myRank;
219     for (i=0;i<numNodes;i++) self->Nodes[INDEX2(i,e,NN)]=Id[self->Nodes[INDEX2(i,e,NN)]];
220     }
221     }
222     return;
223     }
224    

  ViewVC Help
Powered by ViewVC 1.1.26