/[escript]/trunk/finley/src/Assemble_CopyNodalData.c
ViewVC logotype

Contents of /trunk/finley/src/Assemble_CopyNodalData.c

Parent Directory Parent Directory | Revision Log Revision Log


Revision 751 - (show annotations)
Mon Jun 26 01:46:34 2006 UTC (13 years, 2 months ago) by bcumming
File MIME type: text/plain
File size: 12605 byte(s)
Changes relating to the MPI version of escript
The standard OpenMP version of escript is unchanged

- updated data types (Finley_Mesh, Finley_NodeFile, etc) to store meshes
  over multiple MPI processes.
- added CommBuffer code in Paso for communication of Data associated
  with distributed meshes
- updates in Finley and Escript to support distributed data and operations
  on distributed data (such as interpolation).
- construction of RHS in MPI, so that simple explicit schemes (such as
  /docs/examples/wave.py without IO and the Locator) can run in MPI.
- updated mesh generation for first order line, rectangle and brick
  meshes and second order line meshes in MPI.        
- small changes to trunk/SConstruct and trunk/scons/ess_options.py to
  build the MPI version, these changes are turned off by default.

1 /*
2 ************************************************************
3 * Copyright 2006 by ACcESS MNRF *
4 * *
5 * http://www.access.edu.au *
6 * Primary Business: Queensland, Australia *
7 * Licensed under the Open Software License version 3.0 *
8 * http://www.opensource.org/licenses/osl-3.0.php *
9 * *
10 ************************************************************
11 */
12 /**************************************************************/
13
14 /* assemblage routines: copies data between different types nodal representation */
15
16 /**************************************************************/
17
18 /* author: gross@access.edu.au */
19 /* Version: $Id$ */
20
21 /**************************************************************/
22
23 #include "Util.h"
24 #include "Assemble.h"
25 #ifdef _OPENMP
26 #include <omp.h>
27 #endif
28 #ifdef PASO_MPI
29 #include "./paso/CommBuffer.h"
30 #endif
31
32
33 #ifdef PASO_MPI
34 /*
35 this function does no checking that the input are OK, so it has been put here, to
36 only be used in Finley_Assemble_CopyNodalData() which does the checking for it.
37
38 bufferExternal must be large enough to accomdate the external nodal data (numComps*numExternal)
39 */
40 static bool_t getExternalDOF( Finley_NodeFile *nodes, escriptDataC* in, double *externalBuffer, dim_t numComps )
41 {
42 double *sendBuffer=NULL;
43 index_t n, i, myMax=0;
44 bool_t result;
45
46 for( i=0; i<nodes->degreeOfFreedomDistribution->numNeighbours; i++ )
47 if( myMax < nodes->degreeOfFreedomDistribution->edges[i]->numForward )
48 myMax = nodes->degreeOfFreedomDistribution->edges[i]->numForward;
49 if( myMax )
50 sendBuffer = MEMALLOC( myMax*numComps, double );
51
52 /* pack and send DOF information to neighbour domains */
53 for( n=0; n<nodes->degreeOfFreedomDistribution->numNeighbours; n++ )
54 {
55 /* pack data */
56 for( i=0; i<nodes->degreeOfFreedomDistribution->edges[n]->numForward; i++ )
57 memcpy( sendBuffer + i*numComps, getSampleData(in,nodes->degreeOfFreedomDistribution->edges[n]->indexForward[i]), numComps*sizeof(double) );
58
59 /* place it in the send buffer */
60 Paso_CommBuffer_pack( nodes->CommBuffer, nodes->degreeOfFreedomDistribution->neighbours[n], NULL, sendBuffer, numComps*sizeof(double), 0 );
61
62 /* send it */
63 result = Paso_CommBuffer_send( nodes->CommBuffer, nodes->degreeOfFreedomDistribution->neighbours[n], numComps*sizeof(double) );
64 if( result==FALSE )
65 return FALSE;
66 }
67
68 MEMFREE( sendBuffer );
69
70 /* receive and unpack external DOF information from neigbours */
71 for( n=0; n<nodes->degreeOfFreedomDistribution->numNeighbours; n++ )
72 {
73 /* receive data */
74 result = Paso_CommBuffer_recv( nodes->CommBuffer, nodes->degreeOfFreedomDistribution->neighbours[n], numComps*sizeof(double) );
75 if( result==FALSE )
76 return FALSE;
77
78 /* unpack the data */
79 Paso_CommBuffer_unpack( nodes->CommBuffer, nodes->degreeOfFreedomDistribution->neighbours[n], nodes->degreeOfFreedomDistribution->edges[n]->indexBackward, externalBuffer, numComps*sizeof(double), -nodes->degreeOfFreedomDistribution->numLocal );
80 }
81
82 return TRUE;
83 }
84 #endif
85
86 /******************************************************************************************************/
87
88
89 void Finley_Assemble_CopyNodalData(Finley_NodeFile* nodes,escriptDataC* out,escriptDataC* in) {
90 if (nodes==NULL) return;
91 dim_t n,i;
92 dim_t numComps=getDataPointSize(out);
93 type_t in_data_type=getFunctionSpaceType(in);
94 type_t out_data_type=getFunctionSpaceType(out);
95 Finley_resetError();
96
97 /* check out and in */
98 if (numComps!=getDataPointSize(in)) {
99 Finley_setError(TYPE_ERROR,"__FILE__: number of components of input and output Data do not match.");
100 } else if (!isExpanded(out)) {
101 Finley_setError(TYPE_ERROR,"__FILE__: expanded Data object is expected for output data.");
102 }
103
104 /* TODO */
105 /* more sophisticated test needed for overlapping node/DOF counts */
106 if (in_data_type == FINLEY_NODES) {
107 if (! numSamplesEqual(in,1,nodes->numNodes)) {
108 Finley_setError(TYPE_ERROR,"__FILE__: illegal number of samples of input Data object");
109 }
110 } else if (in_data_type == FINLEY_DEGREES_OF_FREEDOM) {
111 #ifdef PASO_MPI
112 if (! numSamplesEqual(in,1,nodes->degreeOfFreedomDistribution->numLocal)) {
113 #else
114 if (! numSamplesEqual(in,1,nodes->numDegreesOfFreedom)) {
115 #endif
116 Finley_setError(TYPE_ERROR,"__FILE__: illegal number of samples of input Data object");
117 }
118 } else if (in_data_type == FINLEY_REDUCED_DEGREES_OF_FREEDOM) {
119 #ifdef PASO_MPI
120 if (! numSamplesEqual(in,1,nodes->reducedDegreeOfFreedomDistribution->numLocal)) {
121 #else
122 if (! numSamplesEqual(in,1,nodes->reducedNumDegreesOfFreedom)) {
123 #endif
124 Finley_setError(TYPE_ERROR,"__FILE__: illegal number of samples of input Data object");
125 }
126 } else {
127 Finley_setError(TYPE_ERROR,"__FILE__: illegal function space type for target object");
128 }
129
130 if (out_data_type == FINLEY_NODES) {
131 if (! numSamplesEqual(out,1,nodes->numNodes)) {
132 Finley_setError(TYPE_ERROR,"__FILE__: illegal number of samples of output Data object");
133 }
134 } else if (out_data_type == FINLEY_DEGREES_OF_FREEDOM) {
135 #ifdef PASO_MPI
136 if (! numSamplesEqual(out,1,nodes->degreeOfFreedomDistribution->numLocal)) {
137 #else
138 if (! numSamplesEqual(out,1,nodes->numDegreesOfFreedom)) {
139 #endif
140 Finley_setError(TYPE_ERROR,"__FILE__: illegal number of samples of output Data object");
141 }
142 } else if (out_data_type == FINLEY_REDUCED_DEGREES_OF_FREEDOM) {
143 #ifdef PASO_MPI
144 if (! numSamplesEqual(out,1,nodes->reducedDegreeOfFreedomDistribution->numLocal)) {
145 #else
146 if (! numSamplesEqual(out,1,nodes->reducedNumDegreesOfFreedom)) {
147 #endif
148 Finley_setError(TYPE_ERROR,"__FILE__: illegal number of samples of output Data object");
149 }
150 } else {
151 Finley_setError(TYPE_ERROR,"__FILE__: illegal function space type for source object");
152 }
153
154 /* now we can start */
155
156 /* This is where the "MPI magic" that shares the data accross domain boundaries occurs.
157 when the user asks to copy from DegreesOfFreedom (non-overlapping solution) to
158 Nodes (overlapping continuous), communication is required to get the DOF values
159 corresponding to external nodes from neighbouring domains. Communication is handled by
160 the Paso_CommBuffer that is attached to nodes. Copying the other direction (nodes to DOF)
161 is similar to the serial case, with just a little bit more care required to ensure that
162 only local values are copied. Piece of cake. */
163 if (Finley_noError()) {
164 if (in_data_type == FINLEY_NODES) {
165 if (out_data_type == FINLEY_NODES) {
166 #pragma omp parallel for private(n) schedule(static)
167 for (n=0;n<nodes->numNodes;n++)
168 Finley_copyDouble(numComps,getSampleData(in,n),getSampleData(out,n));
169 } else if (out_data_type == FINLEY_DEGREES_OF_FREEDOM) {
170 #pragma omp parallel for private(n) schedule(static)
171 for (n=0;n<nodes->numNodes;n++)
172 #ifdef PASO_MPI
173 if( nodes->degreeOfFreedom[n]<nodes->degreeOfFreedomDistribution->numLocal )
174 #endif
175 Finley_copyDouble(numComps,getSampleData(in,n),getSampleData(out,nodes->degreeOfFreedom[n]));
176 } else if (out_data_type == FINLEY_REDUCED_DEGREES_OF_FREEDOM) {
177 #pragma omp parallel for private(n,i) schedule(static)
178 for (i=0;i<nodes->numNodes;i++) {
179 n=nodes->reducedDegreeOfFreedom[i];
180 #ifdef PASO_MPI
181 if( n>=0 && nodes->degreeOfFreedom[n]<nodes->degreeOfFreedomDistribution->numLocal )
182 #else
183 if (n>=0)
184 #endif
185 Finley_copyDouble(numComps,getSampleData(in,i),getSampleData(out,n));
186 }
187 }
188 } else if (in_data_type == FINLEY_DEGREES_OF_FREEDOM) {
189 if (out_data_type == FINLEY_NODES)
190 {
191 #ifdef PASO_MPI
192 double *externalBuffer = MEMALLOC( numComps*nodes->degreeOfFreedomDistribution->numExternal, double );
193 getExternalDOF( nodes, in, externalBuffer, numComps );
194 for (n=0;n<nodes->numNodes;n++) {
195 if( nodes->degreeOfFreedom[n]<nodes->degreeOfFreedomDistribution->numLocal )
196 Finley_copyDouble(numComps,getSampleData(in,nodes->degreeOfFreedom[n]),getSampleData(out,n));
197 else
198 Finley_copyDouble(numComps,externalBuffer + numComps*(nodes->degreeOfFreedom[n] - nodes->degreeOfFreedomDistribution->numLocal),getSampleData(out,n));
199 }
200
201 MEMFREE( externalBuffer );
202 #else
203 #pragma omp parallel for private(n) schedule(static)
204 for (n=0;n<nodes->numNodes;n++)
205 Finley_copyDouble(numComps,getSampleData(in,nodes->degreeOfFreedom[n]),getSampleData(out,n));
206 #endif
207 } else if (out_data_type == FINLEY_DEGREES_OF_FREEDOM) {
208 #pragma omp parallel for private(n) schedule(static)
209 for (n=0;n<nodes->numDegreesOfFreedom;n++)
210 Finley_copyDouble(numComps,getSampleData(in,n),getSampleData(out,n));
211 } else if (out_data_type == FINLEY_REDUCED_DEGREES_OF_FREEDOM) {
212 #pragma omp parallel for private(n,i) schedule(static)
213 for (i=0;i<nodes->numNodes;i++) {
214 n=nodes->reducedDegreeOfFreedom[i];
215 #ifdef PASO_MPI
216 if( n>=0 && nodes->degreeOfFreedom[n]<nodes->degreeOfFreedomDistribution->numLocal )
217 #else
218 if (n>=0)
219 #endif
220 Finley_copyDouble(numComps,getSampleData(in,nodes->degreeOfFreedom[i]),getSampleData(out,n));
221 }
222 }
223 }
224 #ifndef PASO_MPI
225 else if (in_data_type == FINLEY_REDUCED_DEGREES_OF_FREEDOM) {
226 if (out_data_type == FINLEY_REDUCED_DEGREES_OF_FREEDOM) {
227 #pragma omp parallel for private(n) schedule(static)
228 for (n=0;n<nodes->reducedNumDegreesOfFreedom;n++)
229 Finley_copyDouble(numComps,getSampleData(in,n),getSampleData(out,n));
230 } else if (out_data_type == FINLEY_DEGREES_OF_FREEDOM && numSamplesEqual(out,1,nodes->reducedNumDegreesOfFreedom)) {
231 #pragma omp parallel for private(n) schedule(static)
232 for (i=0;i<nodes->numNodes;i++) {
233 n=nodes->reducedDegreeOfFreedom[i];
234 if (n>=0) Finley_copyDouble(numComps,getSampleData(in,n),getSampleData(out,nodes->degreeOfFreedom[i]));
235 }
236 } else if (out_data_type == FINLEY_NODES && numSamplesEqual(out,1,nodes->reducedNumDegreesOfFreedom)) {
237 #pragma omp parallel for private(n) schedule(static)
238 for (i=0;i<nodes->numNodes;i++) {
239 n=nodes->reducedDegreeOfFreedom[i];
240 if (n>=0) Finley_copyDouble(numComps,getSampleData(in,n),getSampleData(out,i));
241 }
242 } else {
243 Finley_setError(TYPE_ERROR,"__FILE__: cannot copy from data on reduced degrees of freedom");
244 }
245 }
246 #else
247 else if (in_data_type == FINLEY_REDUCED_DEGREES_OF_FREEDOM) {
248 Finley_setError(TYPE_ERROR,"__FILE__: cannot copy from data on reduced degrees of freedom : MPI implementation does not support this yet");
249 }
250 #endif
251 }
252 return;
253 }
254
255
256
257 /*
258 * $Log$
259 * Revision 1.4 2005/09/15 03:44:21 jgs
260 * Merge of development branch dev-02 back to main trunk on 2005-09-15
261 *
262 * Revision 1.3 2005/08/12 01:45:42 jgs
263 * erge of development branch dev-02 back to main trunk on 2005-08-12
264 *
265 * Revision 1.2.2.3 2005/09/07 06:26:17 gross
266 * the solver from finley are put into the standalone package paso now
267 *
268 * Revision 1.2.2.2 2005/08/09 02:23:12 gross
269 * print statement removed
270 *
271 * Revision 1.2.2.1 2005/08/03 09:55:33 gross
272 * ContactTest is passing now./mk install!
273 *
274 * Revision 1.2 2005/07/08 04:07:45 jgs
275 * Merge of development branch back to main trunk on 2005-07-08
276 *
277 * Revision 1.1.1.1.2.1 2005/06/29 02:34:46 gross
278 * some changes towards 64 integers in finley
279 *
280 * Revision 1.1.1.1 2004/10/26 06:53:56 jgs
281 * initial import of project esys2
282 *
283 * Revision 1.2 2004/07/21 05:00:54 gross
284 * name changes in DataC
285 *
286 * Revision 1.1 2004/07/02 04:21:13 gross
287 * Finley C code has been included
288 *
289 *
290 */

Properties

Name Value
svn:eol-style native
svn:keywords Author Date Id Revision

  ViewVC Help
Powered by ViewVC 1.1.26