/[escript]/branches/trilinos_from_5897/dudley/src/NodeFile_gather.cpp
ViewVC logotype

Contents of /branches/trilinos_from_5897/dudley/src/NodeFile_gather.cpp

Parent Directory Parent Directory | Revision Log Revision Log


Revision 5963 - (show annotations)
Mon Feb 22 06:59:27 2016 UTC (3 years, 3 months ago) by caltinay
File size: 7831 byte(s)
sync and fix.

1
2 /*****************************************************************************
3 *
4 * Copyright (c) 2003-2016 by The University of Queensland
5 * http://www.uq.edu.au
6 *
7 * Primary Business: Queensland, Australia
8 * Licensed under the Open Software License version 3.0
9 * http://www.opensource.org/licenses/osl-3.0.php
10 *
11 * Development until 2012 by Earth Systems Science Computational Center (ESSCC)
12 * Development 2012-2013 by School of Earth Sciences
13 * Development from 2014 by Centre for Geoscience Computing (GeoComp)
14 *
15 *****************************************************************************/
16
17 /************************************************************************************
18 *
19 * Dudley: Mesh: NodeFile
20 * gathers the NodeFile out from the NodeFile in using the entries
21 * in index[0:out->numNodes-1] which are between min_index and max_index (exclusive)
22 * the node index[i]
23 *
24 ************************************************************************************/
25
26 #define ESNEEDPYTHON
27 #include "esysUtils/first.h"
28
29 #include "NodeFile.h"
30
31 /************************************************************************************/
32
33 void Dudley_NodeFile_gatherEntries(dim_t n, index_t * index, index_t min_index, index_t max_index,
34 index_t * Id_out, index_t * Id_in,
35 index_t * Tag_out, index_t * Tag_in,
36 index_t * globalDegreesOfFreedom_out, index_t * globalDegreesOfFreedom_in,
37 dim_t numDim, double *Coordinates_out, double *Coordinates_in)
38 {
39 dim_t i;
40 index_t k;
41 const index_t range = max_index - min_index;
42 const size_t numDim_size = (size_t) numDim * sizeof(double);
43 #pragma omp parallel for private(i,k) schedule(static)
44 for (i = 0; i < n; i++)
45 {
46 k = index[i] - min_index;
47 if ((k >= 0) && (k < range))
48 {
49 Id_out[i] = Id_in[k];
50 Tag_out[i] = Tag_in[k];
51 globalDegreesOfFreedom_out[i] = globalDegreesOfFreedom_in[k];
52 memcpy(&(Coordinates_out[INDEX2(0, i, numDim)]), &(Coordinates_in[INDEX2(0, k, numDim)]), numDim_size);
53 }
54 }
55 }
56
57 void Dudley_NodeFile_gather(index_t * index, Dudley_NodeFile * in, Dudley_NodeFile * out)
58 {
59 index_t min_id, max_id;
60 Dudley_NodeFile_setGlobalIdRange(&min_id, &max_id, in);
61 Dudley_NodeFile_gatherEntries(out->numNodes, index, min_id, max_id,
62 out->Id, in->Id,
63 out->Tag, in->Tag,
64 out->globalDegreesOfFreedom, in->globalDegreesOfFreedom,
65 out->numDim, out->Coordinates, in->Coordinates);
66 }
67
68 void Dudley_NodeFile_gather_global(index_t * index, Dudley_NodeFile * in, Dudley_NodeFile * out)
69 {
70 index_t min_id, max_id, undefined_node;
71 Esys_MPI_rank buffer_rank, *distribution = NULL;
72 index_t *Id_buffer = NULL, *Tag_buffer = NULL, *globalDegreesOfFreedom_buffer = NULL;
73 double *Coordinates_buffer = NULL;
74 dim_t p, buffer_len, n;
75 char error_msg[100];
76 #ifdef ESYS_MPI
77 Esys_MPI_rank dest, source;
78 MPI_Status status;
79 #endif
80
81 /* get the global range of node ids */
82 Dudley_NodeFile_setGlobalIdRange(&min_id, &max_id, in);
83 undefined_node = min_id - 1;
84
85 distribution = new index_t[in->MPIInfo->size + 1];
86
87 if (!Dudley_checkPtr(distribution))
88 {
89 /* distribute the range of node ids */
90 buffer_len = in->MPIInfo->setDistribution(min_id, max_id, distribution);
91 /* allocate buffers */
92 Id_buffer = new index_t[buffer_len];
93 Tag_buffer = new index_t[buffer_len];
94 globalDegreesOfFreedom_buffer = new index_t[buffer_len];
95 Coordinates_buffer = new double[buffer_len * out->numDim];
96 if (!(Dudley_checkPtr(Id_buffer) || Dudley_checkPtr(Tag_buffer) ||
97 Dudley_checkPtr(globalDegreesOfFreedom_buffer) || Dudley_checkPtr(Coordinates_buffer)))
98 {
99 /* fill Id_buffer by the undefined_node marker to check if nodes are defined */
100 #pragma omp parallel for private(n) schedule(static)
101 for (n = 0; n < buffer_len; n++)
102 Id_buffer[n] = undefined_node;
103
104 /* fill the buffer by sending portions around in a circle */
105 #ifdef ESYS_MPI
106 dest = esysUtils::mod_rank(in->MPIInfo->size, in->MPIInfo->rank + 1);
107 source = esysUtils::mod_rank(in->MPIInfo->size, in->MPIInfo->rank - 1);
108 #endif
109 buffer_rank = in->MPIInfo->rank;
110 for (p = 0; p < in->MPIInfo->size; ++p)
111 {
112 if (p > 0)
113 { /* the initial send can be skipped */
114 #ifdef ESYS_MPI
115 MPI_Sendrecv_replace(Id_buffer, buffer_len, MPI_INT,
116 dest, in->MPIInfo->msg_tag_counter, source, in->MPIInfo->msg_tag_counter,
117 in->MPIInfo->comm, &status);
118 MPI_Sendrecv_replace(Tag_buffer, buffer_len, MPI_INT,
119 dest, in->MPIInfo->msg_tag_counter + 1, source,
120 in->MPIInfo->msg_tag_counter + 1, in->MPIInfo->comm, &status);
121 MPI_Sendrecv_replace(globalDegreesOfFreedom_buffer, buffer_len, MPI_INT, dest,
122 in->MPIInfo->msg_tag_counter + 2, source, in->MPIInfo->msg_tag_counter + 2,
123 in->MPIInfo->comm, &status);
124 MPI_Sendrecv_replace(Coordinates_buffer, buffer_len * out->numDim, MPI_DOUBLE, dest,
125 in->MPIInfo->msg_tag_counter + 3, source, in->MPIInfo->msg_tag_counter + 3,
126 in->MPIInfo->comm, &status);
127 #endif
128 ESYS_MPI_INC_COUNTER(*(in->MPIInfo), 4)
129 }
130 buffer_rank = esysUtils::mod_rank(in->MPIInfo->size, buffer_rank - 1);
131 Dudley_NodeFile_scatterEntries(in->numNodes, in->Id,
132 distribution[buffer_rank], distribution[buffer_rank + 1],
133 Id_buffer, in->Id,
134 Tag_buffer, in->Tag,
135 globalDegreesOfFreedom_buffer, in->globalDegreesOfFreedom,
136 out->numDim, Coordinates_buffer, in->Coordinates);
137 }
138 /* now entries are collected from the buffer again by sending the entries around in a circle */
139 #ifdef ESYS_MPI
140 dest = esysUtils::mod_rank(in->MPIInfo->size, in->MPIInfo->rank + 1);
141 source = esysUtils::mod_rank(in->MPIInfo->size, in->MPIInfo->rank - 1);
142 #endif
143 buffer_rank = in->MPIInfo->rank;
144 for (p = 0; p < in->MPIInfo->size; ++p)
145 {
146 Dudley_NodeFile_gatherEntries(out->numNodes, index,
147 distribution[buffer_rank], distribution[buffer_rank + 1],
148 out->Id, Id_buffer,
149 out->Tag, Tag_buffer,
150 out->globalDegreesOfFreedom, globalDegreesOfFreedom_buffer,
151 out->numDim, out->Coordinates, Coordinates_buffer);
152 if (p < in->MPIInfo->size - 1)
153 { /* the last send can be skipped */
154 #ifdef ESYS_MPI
155 MPI_Sendrecv_replace(Id_buffer, buffer_len, MPI_INT,
156 dest, in->MPIInfo->msg_tag_counter, source, in->MPIInfo->msg_tag_counter,
157 in->MPIInfo->comm, &status);
158 MPI_Sendrecv_replace(Tag_buffer, buffer_len, MPI_INT,
159 dest, in->MPIInfo->msg_tag_counter + 1, source,
160 in->MPIInfo->msg_tag_counter + 1, in->MPIInfo->comm, &status);
161 MPI_Sendrecv_replace(globalDegreesOfFreedom_buffer, buffer_len, MPI_INT, dest,
162 in->MPIInfo->msg_tag_counter + 2, source, in->MPIInfo->msg_tag_counter + 2,
163 in->MPIInfo->comm, &status);
164 MPI_Sendrecv_replace(Coordinates_buffer, buffer_len * out->numDim, MPI_DOUBLE, dest,
165 in->MPIInfo->msg_tag_counter + 3, source, in->MPIInfo->msg_tag_counter + 3,
166 in->MPIInfo->comm, &status);
167 #endif
168 ESYS_MPI_INC_COUNTER(*(in->MPIInfo), 4)
169 }
170 buffer_rank = esysUtils::mod_rank(in->MPIInfo->size, buffer_rank - 1);
171 }
172 /* check if all nodes are set: */
173 #pragma omp parallel for private(n) schedule(static)
174 for (n = 0; n < out->numNodes; ++n)
175 {
176 if (out->Id[n] == undefined_node)
177 {
178 sprintf(error_msg,
179 "Dudley_NodeFile_gather_global: Node id %d at position %d is referenced but is not defined.",
180 out->Id[n], n);
181 Dudley_setError(VALUE_ERROR, error_msg);
182 }
183 }
184
185 }
186 delete[] Id_buffer;
187 delete[] Tag_buffer;
188 delete[] globalDegreesOfFreedom_buffer;
189 delete[] Coordinates_buffer;
190 }
191 delete[] distribution;
192 /* make sure that the error is global */
193 esysUtils::Esys_MPIInfo_noError(in->MPIInfo);
194 }

Properties

Name Value
svn:eol-style native
svn:keywords Author Date Id Revision
svn:mergeinfo /branches/4.0fordebian/dudley/src/NodeFile_gather.cpp:5567-5588 /branches/complex/dudley/src/NodeFile_gather.cpp:5866-5937 /branches/lapack2681/finley/src/NodeFile_gather.cpp:2682-2741 /branches/pasowrap/dudley/src/NodeFile_gather.cpp:3661-3674 /branches/py3_attempt2/dudley/src/NodeFile_gather.cpp:3871-3891 /branches/restext/finley/src/NodeFile_gather.cpp:2610-2624 /branches/ripleygmg_from_3668/dudley/src/NodeFile_gather.cpp:3669-3791 /branches/stage3.0/finley/src/NodeFile_gather.cpp:2569-2590 /branches/symbolic_from_3470/dudley/src/NodeFile_gather.cpp:3471-3974 /branches/symbolic_from_3470/ripley/test/python/dudley/src/NodeFile_gather.cpp:3517-3974 /release/3.0/finley/src/NodeFile_gather.cpp:2591-2601 /release/4.0/dudley/src/NodeFile_gather.cpp:5380-5406 /trunk/dudley/src/NodeFile_gather.cpp:4257-4344,5898-5962 /trunk/ripley/test/python/dudley/src/NodeFile_gather.cpp:3480-3515

  ViewVC Help
Powered by ViewVC 1.1.26