/[escript]/branches/trilinos_from_5897/dudley/src/NodeFile_gather.cpp
ViewVC logotype

Contents of /branches/trilinos_from_5897/dudley/src/NodeFile_gather.cpp

Parent Directory Parent Directory | Revision Log Revision Log


Revision 6079 - (show annotations)
Mon Mar 21 12:22:38 2016 UTC (2 years, 10 months ago) by caltinay
File size: 8091 byte(s)
Big commit - making dudley much more like finley to make it more
managable. Fixed quite a few issues that had been fixed in finley.
Disposed of all ReducedNode/ReducedDOF entities that dudley never supported.
Compiles and passes tests.

1
2 /*****************************************************************************
3 *
4 * Copyright (c) 2003-2016 by The University of Queensland
5 * http://www.uq.edu.au
6 *
7 * Primary Business: Queensland, Australia
8 * Licensed under the Open Software License version 3.0
9 * http://www.opensource.org/licenses/osl-3.0.php
10 *
11 * Development until 2012 by Earth Systems Science Computational Center (ESSCC)
12 * Development 2012-2013 by School of Earth Sciences
13 * Development from 2014 by Centre for Geoscience Computing (GeoComp)
14 *
15 *****************************************************************************/
16
17 #include "NodeFile.h"
18
19 using escript::DataTypes::real_t;
20
21 namespace dudley {
22
23 // helper function
24 static void gatherEntries(dim_t n, const index_t* index,
25 index_t min_index, index_t max_index,
26 index_t* Id_out, const index_t* Id_in,
27 int* Tag_out, const int* Tag_in,
28 index_t* globalDegreesOfFreedom_out,
29 const index_t* globalDegreesOfFreedom_in,
30 int numDim, real_t* Coordinates_out,
31 const real_t* Coordinates_in)
32 {
33 const dim_t range = max_index - min_index;
34 const size_t numDim_size = numDim * sizeof(real_t);
35 #pragma omp parallel for
36 for (index_t i = 0; i < n; i++) {
37 const index_t k = index[i] - min_index;
38 if (k >= 0 && k < range) {
39 Id_out[i] = Id_in[k];
40 Tag_out[i] = Tag_in[k];
41 globalDegreesOfFreedom_out[i] = globalDegreesOfFreedom_in[k];
42 memcpy(&Coordinates_out[INDEX2(0, i, numDim)],
43 &Coordinates_in[INDEX2(0, k, numDim)], numDim_size);
44 }
45 }
46 }
47
48 // helper function
49 static void scatterEntries(dim_t n, const index_t* index,
50 index_t min_index, index_t max_index,
51 index_t* Id_out, const index_t* Id_in,
52 int* Tag_out, const int* Tag_in,
53 index_t* globalDegreesOfFreedom_out,
54 const index_t* globalDegreesOfFreedom_in,
55 int numDim, real_t* Coordinates_out,
56 const real_t* Coordinates_in)
57 {
58 const dim_t range = max_index - min_index;
59 const size_t numDim_size = numDim * sizeof(real_t);
60
61 #pragma omp parallel for
62 for (index_t i = 0; i < n; i++) {
63 const index_t k = index[i] - min_index;
64 if (k >= 0 && k < range) {
65 Id_out[k] = Id_in[i];
66 Tag_out[k] = Tag_in[i];
67 globalDegreesOfFreedom_out[k] = globalDegreesOfFreedom_in[i];
68 memcpy(&Coordinates_out[INDEX2(0, k, numDim)],
69 &Coordinates_in[INDEX2(0, i, numDim)], numDim_size);
70 }
71 }
72 }
73
74 void NodeFile::gather(const index_t* index, const NodeFile* in)
75 {
76 const std::pair<index_t,index_t> idRange(in->getGlobalIdRange());
77 gatherEntries(numNodes, index, idRange.first, idRange.second, Id, in->Id,
78 Tag, in->Tag, globalDegreesOfFreedom, in->globalDegreesOfFreedom,
79 numDim, Coordinates, in->Coordinates);
80 }
81
82 void NodeFile::gather_global(const index_t* index, const NodeFile* in)
83 {
84 // get the global range of node IDs
85 const std::pair<index_t,index_t> idRange(in->getGlobalIdRange());
86 const index_t UNDEFINED = idRange.first - 1;
87 std::vector<index_t> distribution(in->MPIInfo->size + 1);
88
89 // distribute the range of node IDs
90 dim_t buffer_len = MPIInfo->setDistribution(idRange.first, idRange.second,
91 &distribution[0]);
92
93 // allocate buffers
94 index_t* Id_buffer = new index_t[buffer_len];
95 int* Tag_buffer = new int[buffer_len];
96 index_t* globalDegreesOfFreedom_buffer = new index_t[buffer_len];
97 real_t* Coordinates_buffer = new real_t[buffer_len * numDim];
98
99 // fill Id_buffer by the UNDEFINED marker to check if nodes are
100 // defined
101 #pragma omp parallel for
102 for (index_t n = 0; n < buffer_len; n++)
103 Id_buffer[n] = UNDEFINED;
104
105 // fill the buffer by sending portions around in a circle
106 #ifdef ESYS_MPI
107 MPI_Status status;
108 int dest = MPIInfo->mod_rank(MPIInfo->rank + 1);
109 int source = MPIInfo->mod_rank(MPIInfo->rank - 1);
110 #endif
111 int buffer_rank = MPIInfo->rank;
112 for (int p = 0; p < MPIInfo->size; ++p) {
113 #ifdef ESYS_MPI
114 if (p > 0) { // the initial send can be skipped
115 MPI_Sendrecv_replace(Id_buffer, buffer_len, MPI_DIM_T, dest,
116 MPIInfo->counter(), source, MPIInfo->counter(),
117 MPIInfo->comm, &status);
118 MPI_Sendrecv_replace(Tag_buffer, buffer_len, MPI_INT, dest,
119 MPIInfo->counter() + 1, source,
120 MPIInfo->counter() + 1, MPIInfo->comm, &status);
121 MPI_Sendrecv_replace(globalDegreesOfFreedom_buffer, buffer_len,
122 MPI_DIM_T, dest, MPIInfo->counter() + 2, source,
123 MPIInfo->counter() + 2, MPIInfo->comm, &status);
124 MPI_Sendrecv_replace(Coordinates_buffer, buffer_len * numDim,
125 MPI_DOUBLE, dest, MPIInfo->counter() + 3, source,
126 MPIInfo->counter() + 3, MPIInfo->comm, &status);
127 MPIInfo->incCounter(4);
128 }
129 #endif
130 buffer_rank = MPIInfo->mod_rank(buffer_rank - 1);
131 scatterEntries(in->numNodes, in->Id, distribution[buffer_rank],
132 distribution[buffer_rank + 1], Id_buffer, in->Id,
133 Tag_buffer, in->Tag, globalDegreesOfFreedom_buffer,
134 in->globalDegreesOfFreedom, numDim, Coordinates_buffer,
135 in->Coordinates);
136 }
137 // now entries are collected from the buffer again by sending the entries
138 // around in a circle
139 #ifdef ESYS_MPI
140 dest = MPIInfo->mod_rank(MPIInfo->rank + 1);
141 source = MPIInfo->mod_rank(MPIInfo->rank - 1);
142 #endif
143 buffer_rank = MPIInfo->rank;
144 for (int p = 0; p < MPIInfo->size; ++p) {
145 gatherEntries(numNodes, index, distribution[buffer_rank],
146 distribution[buffer_rank + 1], Id, Id_buffer,
147 Tag, Tag_buffer, globalDegreesOfFreedom,
148 globalDegreesOfFreedom_buffer, numDim,
149 Coordinates, Coordinates_buffer);
150 #ifdef ESYS_MPI
151 if (p < MPIInfo->size - 1) { // the last send can be skipped
152 MPI_Sendrecv_replace(Id_buffer, buffer_len, MPI_DIM_T, dest,
153 MPIInfo->counter(), source,
154 MPIInfo->counter(), MPIInfo->comm, &status);
155 MPI_Sendrecv_replace(Tag_buffer, buffer_len, MPI_INT, dest,
156 MPIInfo->counter() + 1, source,
157 MPIInfo->counter() + 1, MPIInfo->comm, &status);
158 MPI_Sendrecv_replace(globalDegreesOfFreedom_buffer, buffer_len,
159 MPI_DIM_T, dest, MPIInfo->counter() + 2, source,
160 MPIInfo->counter() + 2, MPIInfo->comm, &status);
161 MPI_Sendrecv_replace(Coordinates_buffer, buffer_len * numDim,
162 MPI_DOUBLE, dest, MPIInfo->counter() + 3, source,
163 MPIInfo->counter() + 3, MPIInfo->comm, &status);
164 MPIInfo->incCounter(4);
165 }
166 #endif
167 buffer_rank = MPIInfo->mod_rank(buffer_rank - 1);
168 }
169 delete[] Id_buffer;
170 delete[] Tag_buffer;
171 delete[] globalDegreesOfFreedom_buffer;
172 delete[] Coordinates_buffer;
173 #if DOASSERT
174 // check if all nodes are set
175 index_t err = -1;
176 #pragma omp parallel for
177 for (index_t n = 0; n < numNodes; ++n) {
178 if (Id[n] == UNDEFINED) {
179 #pragma omp critical
180 err = n;
181 }
182 }
183 if (err >= 0) {
184 std::stringstream ss;
185 ss << "NodeFile::gather_global: Node id " << Id[err]
186 << " at position " << err << " is referenced but not defined.";
187 throw escript::AssertException(ss.str());
188 }
189 #endif // DOASSERT
190 }
191
192 } // namespace dudley
193

Properties

Name Value
svn:eol-style native
svn:keywords Author Date Id Revision
svn:mergeinfo /branches/4.0fordebian/dudley/src/NodeFile_gather.cpp:5567-5588 /branches/complex/dudley/src/NodeFile_gather.cpp:5866-5937 /branches/lapack2681/finley/src/NodeFile_gather.cpp:2682-2741 /branches/pasowrap/dudley/src/NodeFile_gather.cpp:3661-3674 /branches/py3_attempt2/dudley/src/NodeFile_gather.cpp:3871-3891 /branches/restext/finley/src/NodeFile_gather.cpp:2610-2624 /branches/ripleygmg_from_3668/dudley/src/NodeFile_gather.cpp:3669-3791 /branches/stage3.0/finley/src/NodeFile_gather.cpp:2569-2590 /branches/symbolic_from_3470/dudley/src/NodeFile_gather.cpp:3471-3974 /branches/symbolic_from_3470/ripley/test/python/dudley/src/NodeFile_gather.cpp:3517-3974 /release/3.0/finley/src/NodeFile_gather.cpp:2591-2601 /release/4.0/dudley/src/NodeFile_gather.cpp:5380-5406 /trunk/dudley/src/NodeFile_gather.cpp:4257-4344,5898-6007 /trunk/ripley/test/python/dudley/src/NodeFile_gather.cpp:3480-3515

  ViewVC Help
Powered by ViewVC 1.1.26