/[escript]/trunk/finley/src/Mesh_createNodeFileMappings.c
ViewVC logotype

Diff of /trunk/finley/src/Mesh_createNodeFileMappings.c

Parent Directory Parent Directory | Revision Log Revision Log | View Patch Patch

revision 1628 by phornby, Fri Jul 11 13:12:46 2008 UTC revision 1735 by gross, Fri Aug 29 00:18:26 2008 UTC
# Line 1  Line 1 
1    
2  /* $Id$ */  /* $Id:$ */
3    
4  /*******************************************************  /*******************************************************
5   *   *
# Line 23  Line 23 
23  #include "Mesh.h"  #include "Mesh.h"
24  #define UNUSED -1  #define UNUSED -1
25    
26    #define BOUNDS_CHECK 1
27    
28  /**************************************************************/  /**************************************************************/
29    
30  void Mesh_createDOFMappingAndCoupling(Finley_Mesh* in, bool_t use_reduced_elements)  void Mesh_createDOFMappingAndCoupling(Finley_Mesh* in, bool_t use_reduced_elements)
31  {  {
32    index_t min_DOF, max_DOF, *shared=NULL, *offsetInShared=NULL, *locDOFMask=NULL, i, k, myFirstDOF, myLastDOF, *nodeMask=NULL, firstDOF, lastDOF, *globalDOFIndex;    index_t min_DOF, max_DOF, *shared=NULL, *offsetInShared=NULL, *locDOFMask=NULL, i, k, myFirstDOF, myLastDOF, *nodeMask=NULL, firstDOF, lastDOF, *globalDOFIndex, *wanted_DOFs=NULL;
33    dim_t mpiSize, len_loc_dof, numNeighbors, n, lastn, numNodes;    dim_t mpiSize, len_loc_dof, numNeighbors, n, lastn, numNodes,*rcv_len=NULL, *snd_len=NULL, count;
34    Paso_MPI_rank myRank,p,p_min,p_max, *neighbor=NULL;    Paso_MPI_rank myRank,p,p_min,p_max, *neighbor=NULL;
35    Paso_SharedComponents *rcv_shcomp=NULL, *snd_shcomp=NULL;    Paso_SharedComponents *rcv_shcomp=NULL, *snd_shcomp=NULL;
36    Finley_NodeMapping *this_mapping=NULL;    Finley_NodeMapping *this_mapping=NULL;
37    Paso_Connector* this_connector=NULL;    Paso_Connector* this_connector=NULL;
38    Paso_Distribution* dof_distribution;    Paso_Distribution* dof_distribution;
39        Paso_MPIInfo *mpi_info = in->MPIInfo;
40      #ifdef PASO_MPI
41          MPI_Request* mpi_requests=NULL;
42          MPI_Status* mpi_stati=NULL;
43      #else
44          int *mpi_requests=NULL, *mpi_stati=NULL;
45      #endif
46    
47    numNodes=in->Nodes->numNodes;    numNodes=in->Nodes->numNodes;
48    if (use_reduced_elements) {    if (use_reduced_elements) {
49      dof_distribution=in->Nodes->reducedDegreesOfFreedomDistribution;      dof_distribution=in->Nodes->reducedDegreesOfFreedomDistribution;
50      globalDOFIndex=in->Nodes->globalReducedDOFIndex;      globalDOFIndex=in->Nodes->globalReducedDOFIndex;
     Finley_NodeFile_setReducedDOFRange(&min_DOF, &max_DOF,in->Nodes);  
51    } else {    } else {
52      dof_distribution=in->Nodes->degreesOfFreedomDistribution;      dof_distribution=in->Nodes->degreesOfFreedomDistribution;
53      globalDOFIndex=in->Nodes->globalDegreesOfFreedom;      globalDOFIndex=in->Nodes->globalDegreesOfFreedom;
     Finley_NodeFile_setDOFRange(&min_DOF, &max_DOF,in->Nodes);  
54    }    }
55    
56    mpiSize=dof_distribution->mpi_info->size;    mpiSize=mpi_info->size;
57    myRank=dof_distribution->mpi_info->rank;    myRank=mpi_info->rank;
58    
59    min_DOF=Finley_Util_getFlaggedMinInt(1,numNodes,globalDOFIndex,-1);    min_DOF=Finley_Util_getFlaggedMinInt(1,numNodes,globalDOFIndex,-1);
60    max_DOF=Finley_Util_getFlaggedMaxInt(1,numNodes,globalDOFIndex,-1);    max_DOF=Finley_Util_getFlaggedMaxInt(1,numNodes,globalDOFIndex,-1);
# Line 67  void Mesh_createDOFMappingAndCoupling(Fi Line 74  void Mesh_createDOFMappingAndCoupling(Fi
74        Finley_setError(SYSTEM_ERROR,"Local elements do not span local degrees of freedom.");        Finley_setError(SYSTEM_ERROR,"Local elements do not span local degrees of freedom.");
75        return;        return;
76    }    }
77      rcv_len=TMPMEMALLOC(mpiSize,dim_t);
78      snd_len=TMPMEMALLOC(mpiSize,dim_t);
79      #ifdef PASO_MPI
80        mpi_requests=MEMALLOC(mpiSize*2,MPI_Request);
81        mpi_stati=MEMALLOC(mpiSize*2,MPI_Status);
82      #else
83        mpi_requests=MEMALLOC(mpiSize*2,int);
84        mpi_stati=MEMALLOC(mpiSize*2,int);
85      #endif
86      wanted_DOFs=TMPMEMALLOC(numNodes,index_t);
87    nodeMask=TMPMEMALLOC(numNodes,index_t);    nodeMask=TMPMEMALLOC(numNodes,index_t);
88    neighbor=TMPMEMALLOC(mpiSize,Paso_MPI_rank);    neighbor=TMPMEMALLOC(mpiSize,Paso_MPI_rank);
89    shared=TMPMEMALLOC(numNodes*(p_max-p_min+1),index_t);    shared=TMPMEMALLOC(numNodes*(p_max-p_min+1),index_t);
90    offsetInShared=TMPMEMALLOC(mpiSize+1,index_t);    offsetInShared=TMPMEMALLOC(mpiSize+1,index_t);
91    locDOFMask=TMPMEMALLOC(len_loc_dof, index_t);    locDOFMask=TMPMEMALLOC(len_loc_dof, index_t);
92    if (! ( Finley_checkPtr(neighbor) || Finley_checkPtr(shared) || Finley_checkPtr(offsetInShared) || Finley_checkPtr(locDOFMask) || Finley_checkPtr(nodeMask) ))  {    if (! ( Finley_checkPtr(neighbor) || Finley_checkPtr(shared) || Finley_checkPtr(offsetInShared) || Finley_checkPtr(locDOFMask) ||
93         Finley_checkPtr(nodeMask) || Finley_checkPtr(rcv_len) || Finley_checkPtr(snd_len) || Finley_checkPtr(mpi_requests) || Finley_checkPtr(mpi_stati) ||
94         Finley_checkPtr(mpi_stati) )) {
95      #pragma omp parallel for private(i) schedule(static)  
96      for (i=0;i<len_loc_dof;++i) locDOFMask[i]=UNUSED;      memset(rcv_len,0,sizeof(dim_t)*mpiSize);
97      #pragma omp parallel for private(i) schedule(static)      #pragma omp parallel
98      for (i=0;i<numNodes;++i) nodeMask[i]=UNUSED;      {
99            #pragma omp for private(i) schedule(static)
100      for (i=0;i<numNodes;++i) {          for (i=0;i<len_loc_dof;++i) locDOFMask[i]=UNUSED;
101         k=globalDOFIndex[i];          #pragma omp for private(i) schedule(static)
102  #ifdef BOUNDS_CHECK          for (i=0;i<numNodes;++i) nodeMask[i]=UNUSED;
103         if ((k-min_DOF) >= len_loc_dof) { printf("BOUNDS_CHECK %s %d i=%d k=%d min_DOF=%d\n", __FILE__, __LINE__, i, k, min_DOF); exit(1); }          #pragma omp for private(i,k) schedule(static)
104  #endif          for (i=0;i<numNodes;++i) {
105         if (k>-1) locDOFMask[k-min_DOF]=UNUSED-1;             k=globalDOFIndex[i];
106      }             if (k>-1) {
107                  locDOFMask[k-min_DOF]=UNUSED-1;
108                  #ifdef BOUNDS_CHECK
109                  if ((k-min_DOF) >= len_loc_dof) { printf("BOUNDS_CHECK %s %d i=%d k=%d min_DOF=%d\n", __FILE__, __LINE__, i, k, min_DOF); exit(1); }
110                  #endif
111               }
112            }
113    
114      for (i=myFirstDOF-min_DOF;i<myLastDOF-min_DOF;++i) {          #pragma omp for private(i) schedule(static)
115        locDOFMask[i]=i-myFirstDOF+min_DOF;          for (i=myFirstDOF-min_DOF;i<myLastDOF-min_DOF;++i) {
116  #ifdef BOUNDS_CHECK            locDOFMask[i]=i-myFirstDOF+min_DOF;
117        if (i < 0 || i >= len_loc_dof) { printf("BOUNDS_CHECK %s %d i=%d\n", __FILE__, __LINE__, i); exit(1); }            #ifdef BOUNDS_CHECK
118  #endif            if (i < 0 || i >= len_loc_dof) { printf("BOUNDS_CHECK %s %d i=%d\n", __FILE__, __LINE__, i); exit(1); }
119              #endif
120            }
121      }      }
122    
123      numNeighbors=0;      numNeighbors=0;
# Line 104  void Mesh_createDOFMappingAndCoupling(Fi Line 128  void Mesh_createDOFMappingAndCoupling(Fi
128         lastDOF=MIN(max_DOF+1,dof_distribution->first_component[p+1]);         lastDOF=MIN(max_DOF+1,dof_distribution->first_component[p+1]);
129         if (p != myRank) {         if (p != myRank) {
130             for (i=firstDOF-min_DOF;i<lastDOF-min_DOF;++i) {             for (i=firstDOF-min_DOF;i<lastDOF-min_DOF;++i) {
131  #ifdef BOUNDS_CHECK                  #ifdef BOUNDS_CHECK
132                     if (i < 0 || i >= len_loc_dof) { printf("BOUNDS_CHECK %s %d p=%d i=%d\n", __FILE__, __LINE__, p, i); exit(1); }                  if (i < 0 || i >= len_loc_dof) { printf("BOUNDS_CHECK %s %d p=%d i=%d\n", __FILE__, __LINE__, p, i); exit(1); }
133  #endif                  #endif
134                  if (locDOFMask[i] == UNUSED-1) {                  if (locDOFMask[i] == UNUSED-1) {
135                     locDOFMask[i]=myLastDOF-myFirstDOF+n;                     locDOFMask[i]=myLastDOF-myFirstDOF+n;
136                       wanted_DOFs[n]=i+min_DOF;
137                     ++n;                     ++n;
138                  }                  }
139             }             }
140             if (n>lastn) {             if (n>lastn) {
141                  rcv_len[p]=n-lastn;
142                neighbor[numNeighbors]=p;                neighbor[numNeighbors]=p;
143  #ifdef BOUNDS_CHECK                #ifdef BOUNDS_CHECK
144                if (numNeighbors < 0 || numNeighbors >= mpiSize+1) { printf("BOUNDS_CHECK %s %d p=%d numNeighbors=%d n=%d\n", __FILE__, __LINE__, p, numNeighbors, n); exit(1); }                if (numNeighbors < 0 || numNeighbors >= mpiSize+1) { printf("BOUNDS_CHECK %s %d p=%d numNeighbors=%d n=%d\n", __FILE__, __LINE__, p, numNeighbors, n); exit(1); }
145  #endif                #endif
146                offsetInShared[numNeighbors]=lastn;                offsetInShared[numNeighbors]=lastn;
147                numNeighbors++;                numNeighbors++;
148                lastn=n;                lastn=n;
# Line 138  void Mesh_createDOFMappingAndCoupling(Fi Line 164  void Mesh_createDOFMappingAndCoupling(Fi
164      /* now we can set the mapping from nodes to local DOFs */      /* now we can set the mapping from nodes to local DOFs */
165      this_mapping=Finley_NodeMapping_alloc(numNodes,nodeMask,UNUSED);      this_mapping=Finley_NodeMapping_alloc(numNodes,nodeMask,UNUSED);
166      /* define how to get DOF values for controlled bu other processors */      /* define how to get DOF values for controlled bu other processors */
167  #ifdef BOUNDS_CHECK      #ifdef BOUNDS_CHECK
168      for (i=0;i<offsetInShared[numNeighbors];++i) {      for (i=0;i<offsetInShared[numNeighbors];++i) {
169        if (i < 0 || i >= numNodes*(p_max-p_min+1)) { printf("BOUNDS_CHECK %s %d i=%d\n", __FILE__, __LINE__, i); exit(1); }        if (i < 0 || i >= numNodes*(p_max-p_min+1)) { printf("BOUNDS_CHECK %s %d i=%d\n", __FILE__, __LINE__, i); exit(1); }
170      }      }
171  #endif      #endif
172      #pragma omp parallel for private(i) schedule(static)      #pragma omp parallel for private(i) schedule(static)
173      for (i=0;i<offsetInShared[numNeighbors];++i) shared[i]=myLastDOF-myFirstDOF+i;      for (i=0;i<offsetInShared[numNeighbors];++i) shared[i]=myLastDOF-myFirstDOF+i;
174    
175      rcv_shcomp=Paso_SharedComponents_alloc(myLastDOF-myFirstDOF,numNeighbors,neighbor,shared,offsetInShared,1,0,dof_distribution->mpi_info);      rcv_shcomp=Paso_SharedComponents_alloc(myLastDOF-myFirstDOF,numNeighbors,neighbor,shared,offsetInShared,1,0,mpi_info);
176    
177      /* now it is determined which DOFs needs to be send off:*/      /*
178      #pragma omp parallel for private(i) schedule(static)       *    now we build the sender
179      for (i=0;i<len_loc_dof;++i) locDOFMask[i]=UNUSED;       */
180        #ifdef PASO_MPI
181             MPI_Alltoall(rcv_len,1,MPI_INT,snd_len,1,MPI_INT,mpi_info->comm);
182        #else
183            for (p=0;p<mpiSize;++p) snd_len[p]=rcv_count[p];
184        #endif
185        count=0;
186        for (p=0;p<rcv_shcomp->numNeighbors;p++) {
187           #ifdef PASO_MPI
188           MPI_Isend(&(wanted_DOFs[rcv_shcomp->offsetInShared[p]]), rcv_shcomp->offsetInShared[p+1]-rcv_shcomp->offsetInShared[p],
189                      MPI_INT,rcv_shcomp->neighbor[p],mpi_info->msg_tag_counter+myRank,mpi_info->comm,&mpi_requests[count]);
190            #endif
191            count++;
192        }
193      n=0;      n=0;
194      numNeighbors=0;      numNeighbors=0;
195      lastn=n;      for (p=0;p<mpiSize;p++) {
196      for (p=p_min;p<=p_max;++p) {           if (snd_len[p] > 0) {
197         firstDOF=dof_distribution->first_component[p];              #ifdef PASO_MPI
198         lastDOF=dof_distribution->first_component[p+1];              MPI_Irecv(&(shared[n]),snd_len[p],
199         if (p != myRank) {                        MPI_INT, p, mpi_info->msg_tag_counter+p, mpi_info->comm, &mpi_requests[count]);
200             /* mark a DOF by p if it will be requested by processor p */              #endif
201             Finley_Mesh_markDOFsConnectedToRange(locDOFMask,min_DOF,p,firstDOF,lastDOF,in,use_reduced_elements);              count++;
202                neighbor[numNeighbors]=p;
203             for (i=myFirstDOF-min_DOF;i<myLastDOF-min_DOF;++i) {              offsetInShared[numNeighbors]=n;
204                  if (locDOFMask[i] == p) {              numNeighbors++;
205  #ifdef BOUNDS_CHECK              n+=snd_len[p];
206             if (n < 0 || n >= numNodes*(p_max-p_min+1)) { printf("BOUNDS_CHECK %s %d p=%d i=%d n=%d\n", __FILE__, __LINE__, p, i, n); exit(1); }           }
 #endif  
                    shared[n]=i-myFirstDOF+min_DOF;  
                    ++n;  
                 }  
            }  
            if (n>lastn) {  
               neighbor[numNeighbors]=p;  
 #ifdef BOUNDS_CHECK  
               if (numNeighbors < 0 || numNeighbors >= mpiSize+1) { printf("BOUNDS_CHECK %s %d p=%d n=%d numNeighbors=%d\n", __FILE__, __LINE__, p, n, numNeighbors); exit(1); }  
 #endif  
               offsetInShared[numNeighbors]=lastn;  
               numNeighbors++;  
               lastn=n;  
            }  
         }  
207      }      }
208  #ifdef BOUNDS_CHECK      mpi_info->msg_tag_counter+=mpi_info->size;
209      if (numNeighbors < 0 || numNeighbors >= mpiSize+1) { printf("BOUNDS_CHECK %s %d numNeighbors=%d\n", __FILE__, __LINE__, numNeighbors); exit(1); }      offsetInShared[numNeighbors]=n;
210  #endif      #ifdef PASO_MPI
211      offsetInShared[numNeighbors]=lastn;      MPI_Waitall(count,mpi_requests,mpi_stati);
212        #endif
213        /* map global ids to local id's */
214        #pragma omp parallel for private(i) schedule(static)
215        for (i=0;i<offsetInShared[numNeighbors];++i) {
216            shared[i]=locDOFMask[shared[i]-min_DOF];
217        }
218    
219      snd_shcomp=Paso_SharedComponents_alloc(myLastDOF-myFirstDOF,numNeighbors,neighbor,shared,offsetInShared,1,0,dof_distribution->mpi_info);      snd_shcomp=Paso_SharedComponents_alloc(myLastDOF-myFirstDOF,numNeighbors,neighbor,shared,offsetInShared,1,0,dof_distribution->mpi_info);
220    
221      if (Finley_noError()) this_connector=Paso_Connector_alloc(snd_shcomp,rcv_shcomp);      if (Finley_noError()) this_connector=Paso_Connector_alloc(snd_shcomp,rcv_shcomp);
# Line 192  void Mesh_createDOFMappingAndCoupling(Fi Line 223  void Mesh_createDOFMappingAndCoupling(Fi
223      Paso_SharedComponents_free(rcv_shcomp);      Paso_SharedComponents_free(rcv_shcomp);
224      Paso_SharedComponents_free(snd_shcomp);      Paso_SharedComponents_free(snd_shcomp);
225    }    }
226      TMPMEMFREE(rcv_len);
227      TMPMEMFREE(snd_len);
228      TMPMEMFREE(mpi_requests);
229      TMPMEMFREE(mpi_stati);
230      TMPMEMFREE(wanted_DOFs);
231    TMPMEMFREE(nodeMask);    TMPMEMFREE(nodeMask);
232    TMPMEMFREE(neighbor);    TMPMEMFREE(neighbor);
233    TMPMEMFREE(shared);    TMPMEMFREE(shared);
# Line 279  void Finley_Mesh_createNodeFileMappings( Line 315  void Finley_Mesh_createNodeFileMappings(
315                 k=in->Nodes->globalNodesIndex[indexReducedNodes[i]];                 k=in->Nodes->globalNodesIndex[indexReducedNodes[i]];
316                 if ( (k>=myFirstNode) && (myLastNode>k) ) maskMyReducedNodes[k-myFirstNode]=i;                 if ( (k>=myFirstNode) && (myLastNode>k) ) maskMyReducedNodes[k-myFirstNode]=i;
317                 k=in->Nodes->globalDegreesOfFreedom[indexReducedNodes[i]];                 k=in->Nodes->globalDegreesOfFreedom[indexReducedNodes[i]];
318                 if ( (k>=myFirstDOF) && (myLastDOF>k) ) maskMyReducedDOF[k-myFirstDOF]=i;                 if ( (k>=myFirstDOF) && (myLastDOF>k) ) {
319                      maskMyReducedDOF[k-myFirstDOF]=i;
320                   }
321              }              }
322          }          }
323          myNumReducedNodes=Finley_Util_packMask(myNumNodes,maskMyReducedNodes,indexMyReducedNodes);          myNumReducedNodes=Finley_Util_packMask(myNumNodes,maskMyReducedNodes,indexMyReducedNodes);

Legend:
Removed from v.1628  
changed lines
  Added in v.1735

  ViewVC Help
Powered by ViewVC 1.1.26