/[escript]/trunk/finley/src/Assemble_CopyNodalData.c
ViewVC logotype

Diff of /trunk/finley/src/Assemble_CopyNodalData.c

Parent Directory Parent Directory | Revision Log Revision Log | View Patch Patch

trunk/finley/src/finley/Assemble_CopyNodalData.c revision 201 by jgs, Wed Nov 23 04:10:21 2005 UTC trunk/finley/src/Assemble_CopyNodalData.c revision 751 by bcumming, Mon Jun 26 01:46:34 2006 UTC
# Line 1  Line 1 
1  /*  /*
2   ******************************************************************************   ************************************************************
3   *                                                                            *   *          Copyright 2006 by ACcESS MNRF                   *
4   *       COPYRIGHT  ACcESS 2003,2004,2005 -  All Rights Reserved              *   *                                                          *
5   *                                                                            *   *              http://www.access.edu.au                    *
6   * This software is the property of ACcESS. No part of this code              *   *       Primary Business: Queensland, Australia            *
7   * may be copied in any form or by any means without the expressed written    *   *  Licensed under the Open Software License version 3.0    *
8   * consent of ACcESS.  Copying, use or modification of this software          *   *     http://www.opensource.org/licenses/osl-3.0.php       *
9   * by any unauthorised person is illegal unless that person has a software    *   *                                                          *
10   * license agreement with ACcESS.                                             *   ************************************************************
  *                                                                            *  
  ******************************************************************************  
11  */  */
12  /**************************************************************/  /**************************************************************/
13    
# Line 27  Line 25 
25  #ifdef _OPENMP  #ifdef _OPENMP
26  #include <omp.h>  #include <omp.h>
27  #endif  #endif
28    #ifdef PASO_MPI
29    #include "./paso/CommBuffer.h"
30    #endif
31    
32    
33    #ifdef PASO_MPI
34    /*  
35      this function does no checking that the input are OK, so it has been put here, to
36      only be used in Finley_Assemble_CopyNodalData() which does the checking for it.
37    
38      bufferExternal must be large enough to accomdate the external nodal data (numComps*numExternal)
39    */
40    static bool_t getExternalDOF( Finley_NodeFile *nodes, escriptDataC* in, double *externalBuffer, dim_t numComps )
41    {
42      double *sendBuffer=NULL;
43      index_t n, i, myMax=0;
44      bool_t result;
45    
46      for( i=0; i<nodes->degreeOfFreedomDistribution->numNeighbours; i++ )
47        if( myMax < nodes->degreeOfFreedomDistribution->edges[i]->numForward )
48          myMax = nodes->degreeOfFreedomDistribution->edges[i]->numForward;
49      if( myMax )
50        sendBuffer = MEMALLOC( myMax*numComps, double );
51    
52      /* pack and send DOF information to neighbour domains */
53      for( n=0; n<nodes->degreeOfFreedomDistribution->numNeighbours; n++ )
54      {
55        /* pack data */
56        for( i=0; i<nodes->degreeOfFreedomDistribution->edges[n]->numForward; i++ )
57          memcpy( sendBuffer + i*numComps, getSampleData(in,nodes->degreeOfFreedomDistribution->edges[n]->indexForward[i]), numComps*sizeof(double) );
58        
59        /* place it in the send buffer */
60        Paso_CommBuffer_pack( nodes->CommBuffer, nodes->degreeOfFreedomDistribution->neighbours[n], NULL, sendBuffer, numComps*sizeof(double), 0 );
61      
62        /* send it */
63        result = Paso_CommBuffer_send( nodes->CommBuffer, nodes->degreeOfFreedomDistribution->neighbours[n], numComps*sizeof(double) );
64        if( result==FALSE )
65          return FALSE;
66      }
67    
68      MEMFREE( sendBuffer );
69    
70      /* receive and unpack external DOF information from neigbours */
71      for( n=0; n<nodes->degreeOfFreedomDistribution->numNeighbours; n++ )
72      {
73        /* receive data */
74        result = Paso_CommBuffer_recv( nodes->CommBuffer, nodes->degreeOfFreedomDistribution->neighbours[n], numComps*sizeof(double) );
75        if( result==FALSE )
76          return FALSE;
77    
78        /* unpack the data */
79        Paso_CommBuffer_unpack( nodes->CommBuffer, nodes->degreeOfFreedomDistribution->neighbours[n], nodes->degreeOfFreedomDistribution->edges[n]->indexBackward, externalBuffer, numComps*sizeof(double), -nodes->degreeOfFreedomDistribution->numLocal );
80      }
81    
82      return TRUE;
83    }
84    #endif
85    
86  /******************************************************************************************************/  /******************************************************************************************************/
87    
# Line 46  void Finley_Assemble_CopyNodalData(Finle Line 101  void Finley_Assemble_CopyNodalData(Finle
101         Finley_setError(TYPE_ERROR,"__FILE__: expanded Data object is expected for output data.");         Finley_setError(TYPE_ERROR,"__FILE__: expanded Data object is expected for output data.");
102      }      }
103    
104        /* TODO */
105        /* more sophisticated test needed for overlapping node/DOF counts */
106      if (in_data_type == FINLEY_NODES) {      if (in_data_type == FINLEY_NODES) {
107          if (! numSamplesEqual(in,1,nodes->numNodes)) {          if (! numSamplesEqual(in,1,nodes->numNodes)) {
108                 Finley_setError(TYPE_ERROR,"__FILE__: illegal number of samples of input Data object");                 Finley_setError(TYPE_ERROR,"__FILE__: illegal number of samples of input Data object");
109         }         }
110      } else if (in_data_type == FINLEY_DEGREES_OF_FREEDOM) {      } else if (in_data_type == FINLEY_DEGREES_OF_FREEDOM) {
111    #ifdef PASO_MPI
112            if (! numSamplesEqual(in,1,nodes->degreeOfFreedomDistribution->numLocal)) {
113    #else
114          if (! numSamplesEqual(in,1,nodes->numDegreesOfFreedom)) {          if (! numSamplesEqual(in,1,nodes->numDegreesOfFreedom)) {
115    #endif
116                 Finley_setError(TYPE_ERROR,"__FILE__: illegal number of samples of input Data object");                 Finley_setError(TYPE_ERROR,"__FILE__: illegal number of samples of input Data object");
117         }         }
118      } else if (in_data_type == FINLEY_REDUCED_DEGREES_OF_FREEDOM) {      } else if (in_data_type == FINLEY_REDUCED_DEGREES_OF_FREEDOM) {
119    #ifdef PASO_MPI
120            if (! numSamplesEqual(in,1,nodes->reducedDegreeOfFreedomDistribution->numLocal)) {
121    #else
122          if (! numSamplesEqual(in,1,nodes->reducedNumDegreesOfFreedom)) {          if (! numSamplesEqual(in,1,nodes->reducedNumDegreesOfFreedom)) {
123    #endif
124                 Finley_setError(TYPE_ERROR,"__FILE__: illegal number of samples of input Data object");                 Finley_setError(TYPE_ERROR,"__FILE__: illegal number of samples of input Data object");
125         }         }
126      } else {      } else {
# Line 67  void Finley_Assemble_CopyNodalData(Finle Line 132  void Finley_Assemble_CopyNodalData(Finle
132                 Finley_setError(TYPE_ERROR,"__FILE__: illegal number of samples of output Data object");                 Finley_setError(TYPE_ERROR,"__FILE__: illegal number of samples of output Data object");
133         }         }
134      } else if (out_data_type == FINLEY_DEGREES_OF_FREEDOM) {      } else if (out_data_type == FINLEY_DEGREES_OF_FREEDOM) {
135    #ifdef PASO_MPI
136            if (! numSamplesEqual(out,1,nodes->degreeOfFreedomDistribution->numLocal)) {
137    #else
138          if (! numSamplesEqual(out,1,nodes->numDegreesOfFreedom)) {          if (! numSamplesEqual(out,1,nodes->numDegreesOfFreedom)) {
139    #endif
140                 Finley_setError(TYPE_ERROR,"__FILE__: illegal number of samples of output Data object");                 Finley_setError(TYPE_ERROR,"__FILE__: illegal number of samples of output Data object");
141         }         }
142      } else if (out_data_type == FINLEY_REDUCED_DEGREES_OF_FREEDOM) {      } else if (out_data_type == FINLEY_REDUCED_DEGREES_OF_FREEDOM) {
143    #ifdef PASO_MPI
144            if (! numSamplesEqual(out,1,nodes->reducedDegreeOfFreedomDistribution->numLocal)) {
145    #else
146          if (! numSamplesEqual(out,1,nodes->reducedNumDegreesOfFreedom)) {          if (! numSamplesEqual(out,1,nodes->reducedNumDegreesOfFreedom)) {
147    #endif
148                 Finley_setError(TYPE_ERROR,"__FILE__: illegal number of samples of output Data object");                 Finley_setError(TYPE_ERROR,"__FILE__: illegal number of samples of output Data object");
149         }         }
150      } else {      } else {
# Line 79  void Finley_Assemble_CopyNodalData(Finle Line 152  void Finley_Assemble_CopyNodalData(Finle
152      }      }
153    
154      /* now we can start */      /* now we can start */
155    
156       /* This is where the "MPI magic" that shares the data accross domain boundaries occurs.
157          when the user asks to copy from DegreesOfFreedom (non-overlapping solution) to
158          Nodes (overlapping continuous), communication is required to get the DOF values
159          corresponding to external nodes from neighbouring domains. Communication is handled by
160          the Paso_CommBuffer that is attached to nodes. Copying the other direction (nodes to DOF)
161          is similar to the serial case, with just a little bit more care required to ensure that
162          only local values are copied. Piece of cake. */
163      if (Finley_noError()) {      if (Finley_noError()) {
164          if (in_data_type == FINLEY_NODES) {          if (in_data_type == FINLEY_NODES) {
165             if  (out_data_type == FINLEY_NODES) {             if  (out_data_type == FINLEY_NODES) {
# Line 88  void Finley_Assemble_CopyNodalData(Finle Line 168  void Finley_Assemble_CopyNodalData(Finle
168                     Finley_copyDouble(numComps,getSampleData(in,n),getSampleData(out,n));                     Finley_copyDouble(numComps,getSampleData(in,n),getSampleData(out,n));
169             } else if (out_data_type == FINLEY_DEGREES_OF_FREEDOM) {             } else if (out_data_type == FINLEY_DEGREES_OF_FREEDOM) {
170                #pragma omp parallel for private(n) schedule(static)                #pragma omp parallel for private(n) schedule(static)
171                for (n=0;n<nodes->numNodes;n++)                for (n=0;n<nodes->numNodes;n++)
172    #ifdef PASO_MPI
173                    if( nodes->degreeOfFreedom[n]<nodes->degreeOfFreedomDistribution->numLocal )
174    #endif
175                     Finley_copyDouble(numComps,getSampleData(in,n),getSampleData(out,nodes->degreeOfFreedom[n]));                     Finley_copyDouble(numComps,getSampleData(in,n),getSampleData(out,nodes->degreeOfFreedom[n]));
176             } else if (out_data_type == FINLEY_REDUCED_DEGREES_OF_FREEDOM) {             } else if (out_data_type == FINLEY_REDUCED_DEGREES_OF_FREEDOM) {
177                #pragma omp parallel for private(n,i) schedule(static)                #pragma omp parallel for private(n,i) schedule(static)
178                for (i=0;i<nodes->numNodes;i++) {                for (i=0;i<nodes->numNodes;i++) {
179                     n=nodes->reducedDegreeOfFreedom[i];                     n=nodes->reducedDegreeOfFreedom[i];
180                     if (n>=0) Finley_copyDouble(numComps,getSampleData(in,i),getSampleData(out,n));  #ifdef PASO_MPI
181                    if( n>=0 && nodes->degreeOfFreedom[n]<nodes->degreeOfFreedomDistribution->numLocal )
182    #else
183                    if (n>=0)
184    #endif          
185                      Finley_copyDouble(numComps,getSampleData(in,i),getSampleData(out,n));
186                }                }
187             }             }
188          } else if (in_data_type == FINLEY_DEGREES_OF_FREEDOM) {          } else if (in_data_type == FINLEY_DEGREES_OF_FREEDOM) {
189             if  (out_data_type == FINLEY_NODES) {              if  (out_data_type == FINLEY_NODES)
190                {
191    #ifdef PASO_MPI
192                  double *externalBuffer = MEMALLOC( numComps*nodes->degreeOfFreedomDistribution->numExternal, double );
193                  getExternalDOF( nodes, in, externalBuffer, numComps );
194                  for (n=0;n<nodes->numNodes;n++) {
195                    if( nodes->degreeOfFreedom[n]<nodes->degreeOfFreedomDistribution->numLocal )
196                      Finley_copyDouble(numComps,getSampleData(in,nodes->degreeOfFreedom[n]),getSampleData(out,n));
197                    else
198                      Finley_copyDouble(numComps,externalBuffer + numComps*(nodes->degreeOfFreedom[n] - nodes->degreeOfFreedomDistribution->numLocal),getSampleData(out,n));
199                                }
200                  
201                  MEMFREE( externalBuffer );
202    #else
203                #pragma omp parallel for private(n) schedule(static)                #pragma omp parallel for private(n) schedule(static)
204                for (n=0;n<nodes->numNodes;n++)                for (n=0;n<nodes->numNodes;n++)
205                     Finley_copyDouble(numComps,getSampleData(in,nodes->degreeOfFreedom[n]),getSampleData(out,n));                     Finley_copyDouble(numComps,getSampleData(in,nodes->degreeOfFreedom[n]),getSampleData(out,n));
206    #endif
207             } else if (out_data_type == FINLEY_DEGREES_OF_FREEDOM) {             } else if (out_data_type == FINLEY_DEGREES_OF_FREEDOM) {
208                #pragma omp parallel for private(n) schedule(static)                #pragma omp parallel for private(n) schedule(static)
209                for (n=0;n<nodes->numDegreesOfFreedom;n++)                for (n=0;n<nodes->numDegreesOfFreedom;n++)
# Line 110  void Finley_Assemble_CopyNodalData(Finle Line 212  void Finley_Assemble_CopyNodalData(Finle
212                #pragma omp parallel for private(n,i) schedule(static)                #pragma omp parallel for private(n,i) schedule(static)
213                for (i=0;i<nodes->numNodes;i++) {                for (i=0;i<nodes->numNodes;i++) {
214                    n=nodes->reducedDegreeOfFreedom[i];                    n=nodes->reducedDegreeOfFreedom[i];
215                    if (n>=0) Finley_copyDouble(numComps,getSampleData(in,nodes->degreeOfFreedom[i]),getSampleData(out,n));  #ifdef PASO_MPI
216                    if( n>=0 && nodes->degreeOfFreedom[n]<nodes->degreeOfFreedomDistribution->numLocal )
217    #else
218                    if (n>=0)
219    #endif  
220                       Finley_copyDouble(numComps,getSampleData(in,nodes->degreeOfFreedom[i]),getSampleData(out,n));
221                }                }
222             }             }
223          } else if (in_data_type == FINLEY_REDUCED_DEGREES_OF_FREEDOM) {          }
224    #ifndef PASO_MPI
225            else if (in_data_type == FINLEY_REDUCED_DEGREES_OF_FREEDOM) {
226             if (out_data_type == FINLEY_REDUCED_DEGREES_OF_FREEDOM) {             if (out_data_type == FINLEY_REDUCED_DEGREES_OF_FREEDOM) {
227                #pragma omp parallel for private(n) schedule(static)                #pragma omp parallel for private(n) schedule(static)
228                for (n=0;n<nodes->reducedNumDegreesOfFreedom;n++)                for (n=0;n<nodes->reducedNumDegreesOfFreedom;n++)
# Line 134  void Finley_Assemble_CopyNodalData(Finle Line 243  void Finley_Assemble_CopyNodalData(Finle
243               Finley_setError(TYPE_ERROR,"__FILE__: cannot copy from data on reduced degrees of freedom");               Finley_setError(TYPE_ERROR,"__FILE__: cannot copy from data on reduced degrees of freedom");
244             }             }
245          }          }
246    #else
247            else if (in_data_type == FINLEY_REDUCED_DEGREES_OF_FREEDOM) {
248              Finley_setError(TYPE_ERROR,"__FILE__: cannot copy from data on reduced degrees of freedom : MPI implementation does not support this yet");
249            }
250    #endif
251     }     }
252     return;     return;
253  }  }
254    
255    
256    
257  /*  /*
258   * $Log$   * $Log$
259   * Revision 1.4  2005/09/15 03:44:21  jgs   * Revision 1.4  2005/09/15 03:44:21  jgs

Legend:
Removed from v.201  
changed lines
  Added in v.751

  ViewVC Help
Powered by ViewVC 1.1.26