/[escript]/trunk/finley/src/Assemble_CopyNodalData.c
ViewVC logotype

Annotation of /trunk/finley/src/Assemble_CopyNodalData.c

Parent Directory Parent Directory | Revision Log Revision Log


Revision 751 - (hide annotations)
Mon Jun 26 01:46:34 2006 UTC (13 years, 4 months ago) by bcumming
File MIME type: text/plain
File size: 12605 byte(s)
Changes relating to the MPI version of escript
The standard OpenMP version of escript is unchanged

- updated data types (Finley_Mesh, Finley_NodeFile, etc) to store meshes
  over multiple MPI processes.
- added CommBuffer code in Paso for communication of Data associated
  with distributed meshes
- updates in Finley and Escript to support distributed data and operations
  on distributed data (such as interpolation).
- construction of RHS in MPI, so that simple explicit schemes (such as
  /docs/examples/wave.py without IO and the Locator) can run in MPI.
- updated mesh generation for first order line, rectangle and brick
  meshes and second order line meshes in MPI.        
- small changes to trunk/SConstruct and trunk/scons/ess_options.py to
  build the MPI version, these changes are turned off by default.

1 jgs 150 /*
2 elspeth 616 ************************************************************
3     * Copyright 2006 by ACcESS MNRF *
4     * *
5     * http://www.access.edu.au *
6     * Primary Business: Queensland, Australia *
7     * Licensed under the Open Software License version 3.0 *
8     * http://www.opensource.org/licenses/osl-3.0.php *
9     * *
10     ************************************************************
11 jgs 150 */
12 jgs 82 /**************************************************************/
13    
14     /* assemblage routines: copies data between different types nodal representation */
15    
16     /**************************************************************/
17    
18     /* author: gross@access.edu.au */
19     /* Version: $Id$ */
20    
21     /**************************************************************/
22    
23     #include "Util.h"
24     #include "Assemble.h"
25     #ifdef _OPENMP
26     #include <omp.h>
27     #endif
28 bcumming 751 #ifdef PASO_MPI
29     #include "./paso/CommBuffer.h"
30     #endif
31 jgs 82
32 bcumming 751
33     #ifdef PASO_MPI
34     /*
35     this function does no checking that the input are OK, so it has been put here, to
36     only be used in Finley_Assemble_CopyNodalData() which does the checking for it.
37    
38     bufferExternal must be large enough to accomdate the external nodal data (numComps*numExternal)
39     */
40     static bool_t getExternalDOF( Finley_NodeFile *nodes, escriptDataC* in, double *externalBuffer, dim_t numComps )
41     {
42     double *sendBuffer=NULL;
43     index_t n, i, myMax=0;
44     bool_t result;
45    
46     for( i=0; i<nodes->degreeOfFreedomDistribution->numNeighbours; i++ )
47     if( myMax < nodes->degreeOfFreedomDistribution->edges[i]->numForward )
48     myMax = nodes->degreeOfFreedomDistribution->edges[i]->numForward;
49     if( myMax )
50     sendBuffer = MEMALLOC( myMax*numComps, double );
51    
52     /* pack and send DOF information to neighbour domains */
53     for( n=0; n<nodes->degreeOfFreedomDistribution->numNeighbours; n++ )
54     {
55     /* pack data */
56     for( i=0; i<nodes->degreeOfFreedomDistribution->edges[n]->numForward; i++ )
57     memcpy( sendBuffer + i*numComps, getSampleData(in,nodes->degreeOfFreedomDistribution->edges[n]->indexForward[i]), numComps*sizeof(double) );
58    
59     /* place it in the send buffer */
60     Paso_CommBuffer_pack( nodes->CommBuffer, nodes->degreeOfFreedomDistribution->neighbours[n], NULL, sendBuffer, numComps*sizeof(double), 0 );
61    
62     /* send it */
63     result = Paso_CommBuffer_send( nodes->CommBuffer, nodes->degreeOfFreedomDistribution->neighbours[n], numComps*sizeof(double) );
64     if( result==FALSE )
65     return FALSE;
66     }
67    
68     MEMFREE( sendBuffer );
69    
70     /* receive and unpack external DOF information from neigbours */
71     for( n=0; n<nodes->degreeOfFreedomDistribution->numNeighbours; n++ )
72     {
73     /* receive data */
74     result = Paso_CommBuffer_recv( nodes->CommBuffer, nodes->degreeOfFreedomDistribution->neighbours[n], numComps*sizeof(double) );
75     if( result==FALSE )
76     return FALSE;
77    
78     /* unpack the data */
79     Paso_CommBuffer_unpack( nodes->CommBuffer, nodes->degreeOfFreedomDistribution->neighbours[n], nodes->degreeOfFreedomDistribution->edges[n]->indexBackward, externalBuffer, numComps*sizeof(double), -nodes->degreeOfFreedomDistribution->numLocal );
80     }
81    
82     return TRUE;
83     }
84     #endif
85    
86 jgs 82 /******************************************************************************************************/
87    
88    
89     void Finley_Assemble_CopyNodalData(Finley_NodeFile* nodes,escriptDataC* out,escriptDataC* in) {
90     if (nodes==NULL) return;
91 jgs 123 dim_t n,i;
92     dim_t numComps=getDataPointSize(out);
93     type_t in_data_type=getFunctionSpaceType(in);
94     type_t out_data_type=getFunctionSpaceType(out);
95 jgs 150 Finley_resetError();
96 jgs 82
97     /* check out and in */
98     if (numComps!=getDataPointSize(in)) {
99 jgs 150 Finley_setError(TYPE_ERROR,"__FILE__: number of components of input and output Data do not match.");
100 jgs 82 } else if (!isExpanded(out)) {
101 jgs 150 Finley_setError(TYPE_ERROR,"__FILE__: expanded Data object is expected for output data.");
102 jgs 82 }
103    
104 bcumming 751 /* TODO */
105     /* more sophisticated test needed for overlapping node/DOF counts */
106 jgs 82 if (in_data_type == FINLEY_NODES) {
107     if (! numSamplesEqual(in,1,nodes->numNodes)) {
108 jgs 150 Finley_setError(TYPE_ERROR,"__FILE__: illegal number of samples of input Data object");
109 jgs 82 }
110     } else if (in_data_type == FINLEY_DEGREES_OF_FREEDOM) {
111 bcumming 751 #ifdef PASO_MPI
112     if (! numSamplesEqual(in,1,nodes->degreeOfFreedomDistribution->numLocal)) {
113     #else
114 jgs 82 if (! numSamplesEqual(in,1,nodes->numDegreesOfFreedom)) {
115 bcumming 751 #endif
116 jgs 150 Finley_setError(TYPE_ERROR,"__FILE__: illegal number of samples of input Data object");
117 jgs 82 }
118     } else if (in_data_type == FINLEY_REDUCED_DEGREES_OF_FREEDOM) {
119 bcumming 751 #ifdef PASO_MPI
120     if (! numSamplesEqual(in,1,nodes->reducedDegreeOfFreedomDistribution->numLocal)) {
121     #else
122 jgs 82 if (! numSamplesEqual(in,1,nodes->reducedNumDegreesOfFreedom)) {
123 bcumming 751 #endif
124 jgs 150 Finley_setError(TYPE_ERROR,"__FILE__: illegal number of samples of input Data object");
125 jgs 82 }
126     } else {
127 jgs 150 Finley_setError(TYPE_ERROR,"__FILE__: illegal function space type for target object");
128 jgs 82 }
129    
130     if (out_data_type == FINLEY_NODES) {
131     if (! numSamplesEqual(out,1,nodes->numNodes)) {
132 jgs 150 Finley_setError(TYPE_ERROR,"__FILE__: illegal number of samples of output Data object");
133 jgs 82 }
134     } else if (out_data_type == FINLEY_DEGREES_OF_FREEDOM) {
135 bcumming 751 #ifdef PASO_MPI
136     if (! numSamplesEqual(out,1,nodes->degreeOfFreedomDistribution->numLocal)) {
137     #else
138 jgs 82 if (! numSamplesEqual(out,1,nodes->numDegreesOfFreedom)) {
139 bcumming 751 #endif
140 jgs 150 Finley_setError(TYPE_ERROR,"__FILE__: illegal number of samples of output Data object");
141 jgs 82 }
142     } else if (out_data_type == FINLEY_REDUCED_DEGREES_OF_FREEDOM) {
143 bcumming 751 #ifdef PASO_MPI
144     if (! numSamplesEqual(out,1,nodes->reducedDegreeOfFreedomDistribution->numLocal)) {
145     #else
146 jgs 82 if (! numSamplesEqual(out,1,nodes->reducedNumDegreesOfFreedom)) {
147 bcumming 751 #endif
148 jgs 150 Finley_setError(TYPE_ERROR,"__FILE__: illegal number of samples of output Data object");
149 jgs 82 }
150     } else {
151 jgs 150 Finley_setError(TYPE_ERROR,"__FILE__: illegal function space type for source object");
152 jgs 82 }
153    
154     /* now we can start */
155 bcumming 751
156     /* This is where the "MPI magic" that shares the data accross domain boundaries occurs.
157     when the user asks to copy from DegreesOfFreedom (non-overlapping solution) to
158     Nodes (overlapping continuous), communication is required to get the DOF values
159     corresponding to external nodes from neighbouring domains. Communication is handled by
160     the Paso_CommBuffer that is attached to nodes. Copying the other direction (nodes to DOF)
161     is similar to the serial case, with just a little bit more care required to ensure that
162     only local values are copied. Piece of cake. */
163 jgs 150 if (Finley_noError()) {
164 jgs 82 if (in_data_type == FINLEY_NODES) {
165     if (out_data_type == FINLEY_NODES) {
166     #pragma omp parallel for private(n) schedule(static)
167     for (n=0;n<nodes->numNodes;n++)
168     Finley_copyDouble(numComps,getSampleData(in,n),getSampleData(out,n));
169     } else if (out_data_type == FINLEY_DEGREES_OF_FREEDOM) {
170     #pragma omp parallel for private(n) schedule(static)
171 bcumming 751 for (n=0;n<nodes->numNodes;n++)
172     #ifdef PASO_MPI
173     if( nodes->degreeOfFreedom[n]<nodes->degreeOfFreedomDistribution->numLocal )
174     #endif
175 jgs 82 Finley_copyDouble(numComps,getSampleData(in,n),getSampleData(out,nodes->degreeOfFreedom[n]));
176     } else if (out_data_type == FINLEY_REDUCED_DEGREES_OF_FREEDOM) {
177     #pragma omp parallel for private(n,i) schedule(static)
178     for (i=0;i<nodes->numNodes;i++) {
179     n=nodes->reducedDegreeOfFreedom[i];
180 bcumming 751 #ifdef PASO_MPI
181     if( n>=0 && nodes->degreeOfFreedom[n]<nodes->degreeOfFreedomDistribution->numLocal )
182     #else
183     if (n>=0)
184     #endif
185     Finley_copyDouble(numComps,getSampleData(in,i),getSampleData(out,n));
186 jgs 82 }
187     }
188     } else if (in_data_type == FINLEY_DEGREES_OF_FREEDOM) {
189 bcumming 751 if (out_data_type == FINLEY_NODES)
190     {
191     #ifdef PASO_MPI
192     double *externalBuffer = MEMALLOC( numComps*nodes->degreeOfFreedomDistribution->numExternal, double );
193     getExternalDOF( nodes, in, externalBuffer, numComps );
194     for (n=0;n<nodes->numNodes;n++) {
195     if( nodes->degreeOfFreedom[n]<nodes->degreeOfFreedomDistribution->numLocal )
196     Finley_copyDouble(numComps,getSampleData(in,nodes->degreeOfFreedom[n]),getSampleData(out,n));
197     else
198     Finley_copyDouble(numComps,externalBuffer + numComps*(nodes->degreeOfFreedom[n] - nodes->degreeOfFreedomDistribution->numLocal),getSampleData(out,n));
199     }
200    
201     MEMFREE( externalBuffer );
202     #else
203 jgs 82 #pragma omp parallel for private(n) schedule(static)
204 bcumming 751 for (n=0;n<nodes->numNodes;n++)
205 jgs 82 Finley_copyDouble(numComps,getSampleData(in,nodes->degreeOfFreedom[n]),getSampleData(out,n));
206 bcumming 751 #endif
207 jgs 82 } else if (out_data_type == FINLEY_DEGREES_OF_FREEDOM) {
208     #pragma omp parallel for private(n) schedule(static)
209     for (n=0;n<nodes->numDegreesOfFreedom;n++)
210     Finley_copyDouble(numComps,getSampleData(in,n),getSampleData(out,n));
211     } else if (out_data_type == FINLEY_REDUCED_DEGREES_OF_FREEDOM) {
212     #pragma omp parallel for private(n,i) schedule(static)
213     for (i=0;i<nodes->numNodes;i++) {
214     n=nodes->reducedDegreeOfFreedom[i];
215 bcumming 751 #ifdef PASO_MPI
216     if( n>=0 && nodes->degreeOfFreedom[n]<nodes->degreeOfFreedomDistribution->numLocal )
217     #else
218     if (n>=0)
219     #endif
220     Finley_copyDouble(numComps,getSampleData(in,nodes->degreeOfFreedom[i]),getSampleData(out,n));
221 jgs 82 }
222     }
223 bcumming 751 }
224     #ifndef PASO_MPI
225     else if (in_data_type == FINLEY_REDUCED_DEGREES_OF_FREEDOM) {
226 jgs 82 if (out_data_type == FINLEY_REDUCED_DEGREES_OF_FREEDOM) {
227     #pragma omp parallel for private(n) schedule(static)
228     for (n=0;n<nodes->reducedNumDegreesOfFreedom;n++)
229     Finley_copyDouble(numComps,getSampleData(in,n),getSampleData(out,n));
230 jgs 147 } else if (out_data_type == FINLEY_DEGREES_OF_FREEDOM && numSamplesEqual(out,1,nodes->reducedNumDegreesOfFreedom)) {
231     #pragma omp parallel for private(n) schedule(static)
232     for (i=0;i<nodes->numNodes;i++) {
233     n=nodes->reducedDegreeOfFreedom[i];
234     if (n>=0) Finley_copyDouble(numComps,getSampleData(in,n),getSampleData(out,nodes->degreeOfFreedom[i]));
235     }
236     } else if (out_data_type == FINLEY_NODES && numSamplesEqual(out,1,nodes->reducedNumDegreesOfFreedom)) {
237     #pragma omp parallel for private(n) schedule(static)
238     for (i=0;i<nodes->numNodes;i++) {
239     n=nodes->reducedDegreeOfFreedom[i];
240     if (n>=0) Finley_copyDouble(numComps,getSampleData(in,n),getSampleData(out,i));
241     }
242 jgs 82 } else {
243 jgs 150 Finley_setError(TYPE_ERROR,"__FILE__: cannot copy from data on reduced degrees of freedom");
244 jgs 82 }
245 jgs 147 }
246 bcumming 751 #else
247     else if (in_data_type == FINLEY_REDUCED_DEGREES_OF_FREEDOM) {
248     Finley_setError(TYPE_ERROR,"__FILE__: cannot copy from data on reduced degrees of freedom : MPI implementation does not support this yet");
249     }
250     #endif
251 jgs 82 }
252     return;
253     }
254 bcumming 751
255    
256    
257 jgs 82 /*
258     * $Log$
259 jgs 150 * Revision 1.4 2005/09/15 03:44:21 jgs
260     * Merge of development branch dev-02 back to main trunk on 2005-09-15
261     *
262 jgs 147 * Revision 1.3 2005/08/12 01:45:42 jgs
263     * erge of development branch dev-02 back to main trunk on 2005-08-12
264     *
265 jgs 150 * Revision 1.2.2.3 2005/09/07 06:26:17 gross
266     * the solver from finley are put into the standalone package paso now
267     *
268 jgs 147 * Revision 1.2.2.2 2005/08/09 02:23:12 gross
269     * print statement removed
270     *
271     * Revision 1.2.2.1 2005/08/03 09:55:33 gross
272     * ContactTest is passing now./mk install!
273     *
274 jgs 123 * Revision 1.2 2005/07/08 04:07:45 jgs
275     * Merge of development branch back to main trunk on 2005-07-08
276 jgs 82 *
277 jgs 123 * Revision 1.1.1.1.2.1 2005/06/29 02:34:46 gross
278     * some changes towards 64 integers in finley
279     *
280     * Revision 1.1.1.1 2004/10/26 06:53:56 jgs
281     * initial import of project esys2
282     *
283 jgs 82 * Revision 1.2 2004/07/21 05:00:54 gross
284     * name changes in DataC
285     *
286     * Revision 1.1 2004/07/02 04:21:13 gross
287     * Finley C code has been included
288     *
289     *
290     */

Properties

Name Value
svn:eol-style native
svn:keywords Author Date Id Revision

  ViewVC Help
Powered by ViewVC 1.1.26