/[escript]/branches/trilinos_from_5897/dudley/src/NodeFile_gather.cpp
ViewVC logotype

Diff of /branches/trilinos_from_5897/dudley/src/NodeFile_gather.cpp

Parent Directory Parent Directory | Revision Log Revision Log | View Patch Patch

revision 6008 by caltinay, Mon Feb 22 06:59:27 2016 UTC revision 6009 by caltinay, Wed Mar 2 04:13:26 2016 UTC
# Line 14  Line 14 
14  *  *
15  *****************************************************************************/  *****************************************************************************/
16    
17  /************************************************************************************  /****************************************************************************
18   *   *
19   *   Dudley: Mesh: NodeFile                                     *   Dudley: Mesh: NodeFile
20   *   gathers the NodeFile out from the NodeFile in using the entries   *   gathers the NodeFile out from the NodeFile in using the entries
21   *   in index[0:out->numNodes-1] which are between min_index and max_index (exclusive)   *   in index[0:out->numNodes-1] which are between min_index and max_index (exclusive)
22   *   the node index[i]   *   the node index[i]
23   *   *
24   ************************************************************************************/   ****************************************************************************/
   
 #define ESNEEDPYTHON  
 #include "esysUtils/first.h"  
25    
26  #include "NodeFile.h"  #include "NodeFile.h"
27    
28  /************************************************************************************/  namespace dudley {
29    
30  void Dudley_NodeFile_gatherEntries(dim_t n, index_t * index, index_t min_index, index_t max_index,  void Dudley_NodeFile_gatherEntries(dim_t n, index_t* index, index_t min_index, index_t max_index,
31                     index_t * Id_out, index_t * Id_in,                                     index_t* Id_out, index_t* Id_in,
32                     index_t * Tag_out, index_t * Tag_in,                                     index_t* Tag_out, index_t* Tag_in,
33                     index_t * globalDegreesOfFreedom_out, index_t * globalDegreesOfFreedom_in,                                     index_t* globalDegreesOfFreedom_out, index_t* globalDegreesOfFreedom_in,
34                     dim_t numDim, double *Coordinates_out, double *Coordinates_in)                                     dim_t numDim, double* Coordinates_out, double* Coordinates_in)
35  {  {
36      dim_t i;      dim_t i;
37      index_t k;      index_t k;
# Line 43  void Dudley_NodeFile_gatherEntries(dim_t Line 40  void Dudley_NodeFile_gatherEntries(dim_t
40  #pragma omp parallel for private(i,k) schedule(static)  #pragma omp parallel for private(i,k) schedule(static)
41      for (i = 0; i < n; i++)      for (i = 0; i < n; i++)
42      {      {
43      k = index[i] - min_index;          k = index[i] - min_index;
44      if ((k >= 0) && (k < range))          if ((k >= 0) && (k < range))
45      {          {
46          Id_out[i] = Id_in[k];              Id_out[i] = Id_in[k];
47          Tag_out[i] = Tag_in[k];              Tag_out[i] = Tag_in[k];
48          globalDegreesOfFreedom_out[i] = globalDegreesOfFreedom_in[k];              globalDegreesOfFreedom_out[i] = globalDegreesOfFreedom_in[k];
49          memcpy(&(Coordinates_out[INDEX2(0, i, numDim)]), &(Coordinates_in[INDEX2(0, k, numDim)]), numDim_size);              memcpy(&(Coordinates_out[INDEX2(0, i, numDim)]), &(Coordinates_in[INDEX2(0, k, numDim)]), numDim_size);
50      }          }
51      }      }
52  }  }
53    
# Line 59  void Dudley_NodeFile_gather(index_t * in Line 56  void Dudley_NodeFile_gather(index_t * in
56      index_t min_id, max_id;      index_t min_id, max_id;
57      Dudley_NodeFile_setGlobalIdRange(&min_id, &max_id, in);      Dudley_NodeFile_setGlobalIdRange(&min_id, &max_id, in);
58      Dudley_NodeFile_gatherEntries(out->numNodes, index, min_id, max_id,      Dudley_NodeFile_gatherEntries(out->numNodes, index, min_id, max_id,
59                    out->Id, in->Id,                                    out->Id, in->Id,
60                    out->Tag, in->Tag,                                    out->Tag, in->Tag,
61                    out->globalDegreesOfFreedom, in->globalDegreesOfFreedom,                                    out->globalDegreesOfFreedom, in->globalDegreesOfFreedom,
62                    out->numDim, out->Coordinates, in->Coordinates);                                    out->numDim, out->Coordinates, in->Coordinates);
63  }  }
64    
65  void Dudley_NodeFile_gather_global(index_t * index, Dudley_NodeFile * in, Dudley_NodeFile * out)  void Dudley_NodeFile_gather_global(index_t * index, Dudley_NodeFile * in, Dudley_NodeFile * out)
66  {  {
67      index_t min_id, max_id, undefined_node;      index_t min_id, max_id, undefined_node;
68      Esys_MPI_rank buffer_rank, *distribution = NULL;      int buffer_rank, *distribution = NULL;
69      index_t *Id_buffer = NULL, *Tag_buffer = NULL, *globalDegreesOfFreedom_buffer = NULL;      index_t *Id_buffer = NULL, *Tag_buffer = NULL, *globalDegreesOfFreedom_buffer = NULL;
70      double *Coordinates_buffer = NULL;      double *Coordinates_buffer = NULL;
71      dim_t p, buffer_len, n;      dim_t p, buffer_len, n;
72      char error_msg[100];      char error_msg[100];
73  #ifdef ESYS_MPI  #ifdef ESYS_MPI
74      Esys_MPI_rank dest, source;      int dest, source;
75      MPI_Status status;      MPI_Status status;
76  #endif  #endif
77    
# Line 82  void Dudley_NodeFile_gather_global(index Line 79  void Dudley_NodeFile_gather_global(index
79      Dudley_NodeFile_setGlobalIdRange(&min_id, &max_id, in);      Dudley_NodeFile_setGlobalIdRange(&min_id, &max_id, in);
80      undefined_node = min_id - 1;      undefined_node = min_id - 1;
81    
82      distribution = new  index_t[in->MPIInfo->size + 1];      distribution = new index_t[in->MPIInfo->size + 1];
83    
84      if (!Dudley_checkPtr(distribution))      /* distribute the range of node ids */
85      {      buffer_len = in->MPIInfo->setDistribution(min_id, max_id, distribution);
86      /* distribute the range of node ids */      /* allocate buffers */
87      buffer_len = in->MPIInfo->setDistribution(min_id, max_id, distribution);      Id_buffer = new  index_t[buffer_len];
88      /* allocate buffers */      Tag_buffer = new  index_t[buffer_len];
89      Id_buffer = new  index_t[buffer_len];      globalDegreesOfFreedom_buffer = new  index_t[buffer_len];
90      Tag_buffer = new  index_t[buffer_len];      Coordinates_buffer = new  double[buffer_len * out->numDim];
91      globalDegreesOfFreedom_buffer = new  index_t[buffer_len];      /* fill Id_buffer by the undefined_node marker to check if nodes are defined */
     Coordinates_buffer = new  double[buffer_len * out->numDim];  
     if (!(Dudley_checkPtr(Id_buffer) || Dudley_checkPtr(Tag_buffer) ||  
           Dudley_checkPtr(globalDegreesOfFreedom_buffer) || Dudley_checkPtr(Coordinates_buffer)))  
     {  
         /* fill Id_buffer by the undefined_node marker to check if nodes are defined */  
92  #pragma omp parallel for private(n) schedule(static)  #pragma omp parallel for private(n) schedule(static)
93          for (n = 0; n < buffer_len; n++)      for (n = 0; n < buffer_len; n++)
94          Id_buffer[n] = undefined_node;          Id_buffer[n] = undefined_node;
95    
96          /* fill the buffer by sending portions around in a circle */      /* fill the buffer by sending portions around in a circle */
97  #ifdef ESYS_MPI  #ifdef ESYS_MPI
98          dest = esysUtils::mod_rank(in->MPIInfo->size, in->MPIInfo->rank + 1);      dest = in->MPIInfo->mod_rank(in->MPIInfo->rank + 1);
99          source = esysUtils::mod_rank(in->MPIInfo->size, in->MPIInfo->rank - 1);      source = in->MPIInfo->mod_rank(in->MPIInfo->rank - 1);
100  #endif  #endif
101          buffer_rank = in->MPIInfo->rank;      buffer_rank = in->MPIInfo->rank;
102          for (p = 0; p < in->MPIInfo->size; ++p)      for (p = 0; p < in->MPIInfo->size; ++p)
103          {      {
104          if (p > 0)          if (p > 0)
105          {       /* the initial send can be skipped */          {               /* the initial send can be skipped */
106  #ifdef ESYS_MPI  #ifdef ESYS_MPI
107              MPI_Sendrecv_replace(Id_buffer, buffer_len, MPI_INT,              MPI_Sendrecv_replace(Id_buffer, buffer_len, MPI_INT,
108                       dest, in->MPIInfo->msg_tag_counter, source, in->MPIInfo->msg_tag_counter,                                   dest, in->MPIInfo->counter(), source, in->MPIInfo->counter(),
109                       in->MPIInfo->comm, &status);                                   in->MPIInfo->comm, &status);
110              MPI_Sendrecv_replace(Tag_buffer, buffer_len, MPI_INT,              MPI_Sendrecv_replace(Tag_buffer, buffer_len, MPI_INT,
111                       dest, in->MPIInfo->msg_tag_counter + 1, source,                                   dest, in->MPIInfo->counter() + 1, source,
112                       in->MPIInfo->msg_tag_counter + 1, in->MPIInfo->comm, &status);                                   in->MPIInfo->counter() + 1, in->MPIInfo->comm, &status);
113              MPI_Sendrecv_replace(globalDegreesOfFreedom_buffer, buffer_len, MPI_INT, dest,              MPI_Sendrecv_replace(globalDegreesOfFreedom_buffer, buffer_len, MPI_INT, dest,
114                       in->MPIInfo->msg_tag_counter + 2, source, in->MPIInfo->msg_tag_counter + 2,                                   in->MPIInfo->counter() + 2, source, in->MPIInfo->counter() + 2,
115                       in->MPIInfo->comm, &status);                                   in->MPIInfo->comm, &status);
116              MPI_Sendrecv_replace(Coordinates_buffer, buffer_len * out->numDim, MPI_DOUBLE, dest,              MPI_Sendrecv_replace(Coordinates_buffer, buffer_len * out->numDim, MPI_DOUBLE, dest,
117                       in->MPIInfo->msg_tag_counter + 3, source, in->MPIInfo->msg_tag_counter + 3,                                   in->MPIInfo->counter() + 3, source, in->MPIInfo->counter() + 3,
118                       in->MPIInfo->comm, &status);                                   in->MPIInfo->comm, &status);
119                in->MPIInfo->incCounter(4);
120  #endif  #endif
121              ESYS_MPI_INC_COUNTER(*(in->MPIInfo), 4)          }
122          }          buffer_rank = in->MPIInfo->mod_rank(buffer_rank - 1);
123          buffer_rank = esysUtils::mod_rank(in->MPIInfo->size, buffer_rank - 1);          Dudley_NodeFile_scatterEntries(in->numNodes, in->Id,
124          Dudley_NodeFile_scatterEntries(in->numNodes, in->Id,                                         distribution[buffer_rank], distribution[buffer_rank + 1],
125                             distribution[buffer_rank], distribution[buffer_rank + 1],                                         Id_buffer, in->Id,
126                             Id_buffer, in->Id,                                         Tag_buffer, in->Tag,
127                             Tag_buffer, in->Tag,                                         globalDegreesOfFreedom_buffer, in->globalDegreesOfFreedom,
128                             globalDegreesOfFreedom_buffer, in->globalDegreesOfFreedom,                                         out->numDim, Coordinates_buffer, in->Coordinates);
129                             out->numDim, Coordinates_buffer, in->Coordinates);      }
130          }      /* now entries are collected from the buffer again by sending the entries around in a circle */
         /* now entries are collected from the buffer again by sending the entries around in a circle */  
131  #ifdef ESYS_MPI  #ifdef ESYS_MPI
132          dest = esysUtils::mod_rank(in->MPIInfo->size, in->MPIInfo->rank + 1);      dest = in->MPIInfo->mod_rank(in->MPIInfo->rank + 1);
133          source = esysUtils::mod_rank(in->MPIInfo->size, in->MPIInfo->rank - 1);      source = in->MPIInfo->mod_rank(in->MPIInfo->rank - 1);
134  #endif  #endif
135          buffer_rank = in->MPIInfo->rank;      buffer_rank = in->MPIInfo->rank;
136          for (p = 0; p < in->MPIInfo->size; ++p)      for (p = 0; p < in->MPIInfo->size; ++p)
137          {      {
138          Dudley_NodeFile_gatherEntries(out->numNodes, index,          Dudley_NodeFile_gatherEntries(out->numNodes, index,
139                            distribution[buffer_rank], distribution[buffer_rank + 1],                                        distribution[buffer_rank], distribution[buffer_rank + 1],
140                            out->Id, Id_buffer,                                        out->Id, Id_buffer,
141                            out->Tag, Tag_buffer,                                        out->Tag, Tag_buffer,
142                            out->globalDegreesOfFreedom, globalDegreesOfFreedom_buffer,                                        out->globalDegreesOfFreedom, globalDegreesOfFreedom_buffer,
143                            out->numDim, out->Coordinates, Coordinates_buffer);                                        out->numDim, out->Coordinates, Coordinates_buffer);
144          if (p < in->MPIInfo->size - 1)          if (p < in->MPIInfo->size - 1)
145          {       /* the last send can be skipped */          {               /* the last send can be skipped */
146  #ifdef ESYS_MPI  #ifdef ESYS_MPI
147              MPI_Sendrecv_replace(Id_buffer, buffer_len, MPI_INT,              MPI_Sendrecv_replace(Id_buffer, buffer_len, MPI_INT,
148                       dest, in->MPIInfo->msg_tag_counter, source, in->MPIInfo->msg_tag_counter,                                   dest, in->MPIInfo->counter(), source, in->MPIInfo->counter(),
149                       in->MPIInfo->comm, &status);                                   in->MPIInfo->comm, &status);
150              MPI_Sendrecv_replace(Tag_buffer, buffer_len, MPI_INT,              MPI_Sendrecv_replace(Tag_buffer, buffer_len, MPI_INT,
151                       dest, in->MPIInfo->msg_tag_counter + 1, source,                                   dest, in->MPIInfo->counter() + 1, source,
152                       in->MPIInfo->msg_tag_counter + 1, in->MPIInfo->comm, &status);                                   in->MPIInfo->counter() + 1, in->MPIInfo->comm, &status);
153              MPI_Sendrecv_replace(globalDegreesOfFreedom_buffer, buffer_len, MPI_INT, dest,              MPI_Sendrecv_replace(globalDegreesOfFreedom_buffer, buffer_len, MPI_INT, dest,
154                       in->MPIInfo->msg_tag_counter + 2, source, in->MPIInfo->msg_tag_counter + 2,                                   in->MPIInfo->counter() + 2, source, in->MPIInfo->counter() + 2,
155                       in->MPIInfo->comm, &status);                                   in->MPIInfo->comm, &status);
156              MPI_Sendrecv_replace(Coordinates_buffer, buffer_len * out->numDim, MPI_DOUBLE, dest,              MPI_Sendrecv_replace(Coordinates_buffer, buffer_len * out->numDim, MPI_DOUBLE, dest,
157                       in->MPIInfo->msg_tag_counter + 3, source, in->MPIInfo->msg_tag_counter + 3,                                   in->MPIInfo->counter() + 3, source, in->MPIInfo->counter() + 3,
158                       in->MPIInfo->comm, &status);                                   in->MPIInfo->comm, &status);
159                in->MPIInfo->incCounter(4);
160  #endif  #endif
161              ESYS_MPI_INC_COUNTER(*(in->MPIInfo), 4)          }
162          }          buffer_rank = in->MPIInfo->mod_rank(buffer_rank - 1);
163          buffer_rank = esysUtils::mod_rank(in->MPIInfo->size, buffer_rank - 1);      }
164          }      /* check if all nodes are set: */
         /* check if all nodes are set: */  
165  #pragma omp parallel for private(n) schedule(static)  #pragma omp parallel for private(n) schedule(static)
166          for (n = 0; n < out->numNodes; ++n)      for (n = 0; n < out->numNodes; ++n)
167          {      {
168          if (out->Id[n] == undefined_node)          if (out->Id[n] == undefined_node)
169          {          {
170              sprintf(error_msg,              sprintf(error_msg,
171                  "Dudley_NodeFile_gather_global: Node id %d at position %d is referenced but is not defined.",                      "Dudley_NodeFile_gather_global: Node id %d at position %d is referenced but is not defined.",
172                  out->Id[n], n);                      out->Id[n], n);
173              Dudley_setError(VALUE_ERROR, error_msg);              throw DudleyException(error_msg);
174          }          }
         }  
   
     }  
     delete[] Id_buffer;  
     delete[] Tag_buffer;  
     delete[] globalDegreesOfFreedom_buffer;  
     delete[] Coordinates_buffer;  
175      }      }
176        delete[] Id_buffer;
177        delete[] Tag_buffer;
178        delete[] globalDegreesOfFreedom_buffer;
179        delete[] Coordinates_buffer;
180      delete[] distribution;      delete[] distribution;
     /* make sure that the error is global */  
     esysUtils::Esys_MPIInfo_noError(in->MPIInfo);  
181  }  }
182    
183    } // namespace dudley
184    

Legend:
Removed from v.6008  
changed lines
  Added in v.6009

  ViewVC Help
Powered by ViewVC 1.1.26