/[escript]/trunk/finley/src/ElementFile_distributeByRankOfDOF.c
ViewVC logotype

Annotation of /trunk/finley/src/ElementFile_distributeByRankOfDOF.c

Parent Directory Parent Directory | Revision Log Revision Log


Revision 1387 - (hide annotations)
Fri Jan 11 07:45:26 2008 UTC (11 years, 9 months ago) by trankine
Original Path: temp/finley/src/ElementFile_distributeByRankOfDOF.c
File MIME type: text/plain
File size: 10217 byte(s)
Restore the trunk that existed before the windows changes were committed to the (now moved to branches) old trunk.
1 ksteube 1315
2     /* $Id$ */
3    
4     /*******************************************************
5     *
6     * Copyright 2003-2007 by ACceSS MNRF
7     * Copyright 2007 by University of Queensland
8     *
9     * http://esscc.uq.edu.au
10     * Primary Business: Queensland, Australia
11     * Licensed under the Open Software License version 3.0
12     * http://www.opensource.org/licenses/osl-3.0.php
13     *
14     *******************************************************/
15    
16     /**************************************************************/
17    
18     /* Finley: ElementFile: this will redistribute the Elements including overlap by */
19    
20     /**************************************************************/
21    
22     #include "ElementFile.h"
23     #ifdef _OPENMP
24     #include <omp.h>
25     #endif
26    
27     /**************************************************************/
28    
29     void Finley_ElementFile_distributeByRankOfDOF(Finley_ElementFile* self, Paso_MPI_rank* mpiRankOfDOF, index_t* Id) {
30     size_t size_size;
31     Paso_MPI_rank myRank, p, *Owner_buffer=NULL, loc_proc_mask_max;
32     dim_t e, j, i, size, *send_count=NULL, *recv_count=NULL, *newOwner=NULL, *loc_proc_mask=NULL, *loc_send_count=NULL,
33     newNumElements, numElementsInBuffer, numNodes, numRequests, NN;
34     index_t *send_offset=NULL, *recv_offset=NULL, *Id_buffer=NULL, *Tag_buffer=NULL, *Nodes_buffer=NULL, k;
35     bool_t *proc_mask=NULL;
36     #ifdef PASO_MPI
37     MPI_Request* mpi_requests=NULL;
38     MPI_Status* mpi_stati=NULL;
39     #endif
40     if (self==NULL) return;
41     myRank=self->MPIInfo->rank;
42     size=self->MPIInfo->size;
43     size_size=size*sizeof(dim_t);
44     numNodes=self->numNodes;
45     NN=self->numNodes;
46     if (size>1) {
47     #ifdef PASO_MPI
48     mpi_requests=TMPMEMALLOC(8*size, MPI_Request);
49     mpi_stati=TMPMEMALLOC(8*size, MPI_Status);
50     Finley_checkPtr(mpi_requests);
51     Finley_checkPtr(mpi_stati);
52     #endif
53    
54     /* count the number elements that have to be send to each processor (send_count)
55     and define a new element owner as the processor with the largest number of DOFs and the smallest id */
56     send_count=TMPMEMALLOC(size,dim_t);
57     recv_count=TMPMEMALLOC(size,dim_t);
58     newOwner=TMPMEMALLOC(self->numElements,Paso_MPI_rank);
59     if ( !( Finley_checkPtr(send_count) || Finley_checkPtr(recv_count) || Finley_checkPtr(newOwner) ) ) {
60     memset(send_count, 0, size_size);
61     #pragma omp parallel private(p,loc_proc_mask,loc_send_count)
62     {
63     loc_proc_mask=THREAD_MEMALLOC(size,dim_t);
64     loc_send_count=THREAD_MEMALLOC(size,dim_t);
65     memset(loc_send_count, 0, size_size);
66     #pragma omp for private(e,j,loc_proc_mask_max) schedule(static)
67     for (e=0;e<self->numElements;e++) {
68     if (self->Owner[e] == myRank) {
69     newOwner[e]=myRank;
70     memset(loc_proc_mask, 0, size_size);
71     for(j=0;j<numNodes;j++) {
72     p=mpiRankOfDOF[self->Nodes[INDEX2(j,e,NN)]];
73     loc_proc_mask[p]++;
74     }
75     loc_proc_mask_max=0;
76     for (p=0;p<size;++p) {
77     if (loc_proc_mask[p]>0) loc_send_count[p]++;
78     if (loc_proc_mask[p]>loc_proc_mask_max) {
79     newOwner[e]=p;
80     loc_proc_mask_max=loc_proc_mask[p];
81     }
82     }
83     } else {
84     newOwner[e]=-1;
85     }
86     }
87     #pragma omp critical
88     {
89     for (p=0;p<size;++p) send_count[p]+=loc_send_count[p];
90     }
91     THREAD_MEMFREE(loc_proc_mask);
92     THREAD_MEMFREE(loc_send_count);
93     }
94     #ifdef PASO_MPI
95     MPI_Alltoall(send_count,1,MPI_INT,recv_count,1,MPI_INT,self->MPIInfo->comm);
96     #else
97     for (p=0;p<size;++p) send_count[p]=recv_count[p];
98     #endif
99     /* get the new number of elements for this processor */
100     newNumElements=0;
101     for (p=0;p<size;++p) newNumElements+=recv_count[p];
102    
103     /* get the new number of elements for this processor */
104     numElementsInBuffer=0;
105     for (p=0;p<size;++p) numElementsInBuffer+=send_count[p];
106     /* allocate buffers */
107     Id_buffer=TMPMEMALLOC(numElementsInBuffer,index_t);
108     Tag_buffer=TMPMEMALLOC(numElementsInBuffer,index_t);
109     Owner_buffer=TMPMEMALLOC(numElementsInBuffer,Paso_MPI_rank);
110     Nodes_buffer=TMPMEMALLOC(numElementsInBuffer*NN,index_t);
111     send_offset=TMPMEMALLOC(size,index_t);
112     recv_offset=TMPMEMALLOC(size,index_t);
113     proc_mask=TMPMEMALLOC(size,bool_t);
114     if ( !( Finley_checkPtr(Id_buffer) || Finley_checkPtr(Tag_buffer) || Finley_checkPtr(Owner_buffer) ||
115     Finley_checkPtr(Nodes_buffer) || Finley_checkPtr(send_offset) || Finley_checkPtr(recv_offset) ||
116     Finley_checkPtr(proc_mask) )) {
117    
118     /* callculate the offsets for the processor buffers */
119     recv_offset[0]=0;
120     for (p=0;p<size-1;++p) recv_offset[p+1]=recv_offset[p]+recv_count[p];
121     send_offset[0]=0;
122     for (p=0;p<size-1;++p) send_offset[p+1]=send_offset[p]+send_count[p];
123    
124     memset(send_count, 0, size_size);
125     /* copy element into buffers. proc_mask makes sure that an element is copied once only for each processor */
126     for (e=0;e<self->numElements;e++) {
127     if (self->Owner[e] == myRank) {
128     memset(proc_mask, TRUE, size_size);
129     for(j=0;j<numNodes;j++) {
130     p=mpiRankOfDOF[self->Nodes[INDEX2(j,e,NN)]];
131     if (proc_mask[p]) {
132     k=send_offset[p]+send_count[p];
133     Id_buffer[k]=self->Id[e];
134     Tag_buffer[k]=self->Tag[e];
135     Owner_buffer[k]=newOwner[e];
136     for (i=0;i<numNodes;i++) Nodes_buffer[INDEX2(i,k,NN)]=Id[self->Nodes[INDEX2(i,e,NN)]];
137     send_count[p]++;
138     proc_mask[p]=FALSE;
139     }
140     }
141     }
142     }
143     /* allocate new tables */
144     Finley_ElementFile_allocTable(self,newNumElements);
145    
146     /* start to receive new elements */
147     numRequests=0;
148     for (p=0;p<size;++p) {
149     if (recv_count[p]>0) {
150     #ifdef PASO_MPI
151     MPI_Irecv(&(self->Id[recv_offset[p]]), recv_count[p],
152     MPI_INT, p, self->MPIInfo->msg_tag_counter+myRank,
153     self->MPIInfo->comm, &mpi_requests[numRequests]);
154     numRequests++;
155     MPI_Irecv(&(self->Tag[recv_offset[p]]), recv_count[p],
156     MPI_INT, p, self->MPIInfo->msg_tag_counter+size+myRank,
157     self->MPIInfo->comm, &mpi_requests[numRequests]);
158     numRequests++;
159     MPI_Irecv(&(self->Owner[recv_offset[p]]), recv_count[p],
160     MPI_INT, p, self->MPIInfo->msg_tag_counter+2*size+myRank,
161     self->MPIInfo->comm, &mpi_requests[numRequests]);
162     numRequests++;
163     MPI_Irecv(&(self->Nodes[recv_offset[p]*NN]), recv_count[p]*NN,
164     MPI_INT, p, self->MPIInfo->msg_tag_counter+3*size+myRank,
165     self->MPIInfo->comm, &mpi_requests[numRequests]);
166     numRequests++;
167     #endif
168     }
169     }
170     /* now the buffers can be send away */
171     for (p=0;p<size;++p) {
172     if (send_count[p]>0) {
173     #ifdef PASO_MPI
174     MPI_Issend(&(Id_buffer[send_offset[p]]), send_count[p],
175     MPI_INT, p, self->MPIInfo->msg_tag_counter+p,
176     self->MPIInfo->comm, &mpi_requests[numRequests]);
177     numRequests++;
178     MPI_Issend(&(Tag_buffer[send_offset[p]]), send_count[p],
179     MPI_INT, p, self->MPIInfo->msg_tag_counter+size+p,
180     self->MPIInfo->comm, &mpi_requests[numRequests]);
181     numRequests++;
182     MPI_Issend(&(Owner_buffer[send_offset[p]]), send_count[p],
183     MPI_INT, p, self->MPIInfo->msg_tag_counter+2*size+p,
184     self->MPIInfo->comm, &mpi_requests[numRequests]);
185     numRequests++;
186     MPI_Issend(&(Nodes_buffer[send_offset[p]*NN]), send_count[p]*NN,
187     MPI_INT, p, self->MPIInfo->msg_tag_counter+3*size+p,
188     self->MPIInfo->comm, &mpi_requests[numRequests]);
189     numRequests++;
190     #endif
191    
192     }
193     }
194     self->MPIInfo->msg_tag_counter+=4*size;
195     /* wait for the requests to be finalized */
196     #ifdef PASO_MPI
197     MPI_Waitall(numRequests,mpi_requests,mpi_stati);
198     #endif
199     }
200     /* clear buffer */
201     TMPMEMFREE(Id_buffer);
202     TMPMEMFREE(Tag_buffer);
203     TMPMEMFREE(Owner_buffer);
204     TMPMEMFREE(Nodes_buffer);
205     TMPMEMFREE(send_offset);
206     TMPMEMFREE(recv_offset);
207     TMPMEMFREE(proc_mask);
208     }
209     #ifdef PASO_MPI
210     TMPMEMFREE(mpi_requests);
211     TMPMEMFREE(mpi_stati);
212     #endif
213     TMPMEMFREE(send_count);
214     TMPMEMFREE(recv_count);
215     TMPMEMFREE(newOwner);
216     } else {
217     #pragma omp for private(e,i) schedule(static)
218     for (e=0;e<self->numElements;e++) {
219     self->Owner[e]=myRank;
220     for (i=0;i<numNodes;i++) self->Nodes[INDEX2(i,e,NN)]=Id[self->Nodes[INDEX2(i,e,NN)]];
221     }
222     }
223     return;
224     }
225    

  ViewVC Help
Powered by ViewVC 1.1.26