/[escript]/trunk/escriptcore/src/MPIDataReducer.cpp
ViewVC logotype

Contents of /trunk/escriptcore/src/MPIDataReducer.cpp

Parent Directory Parent Directory | Revision Log Revision Log


Revision 5645 - (show annotations)
Fri Jun 12 02:56:30 2015 UTC (4 years, 1 month ago) by jfenwick
File size: 10100 byte(s)
more
1 /*****************************************************************************
2 *
3 * Copyright (c) 2014-2015 by The University of Queensland
4 * http://www.uq.edu.au
5 *
6 * Primary Business: Queensland, Australia
7 * Licensed under the Open Software License version 3.0
8 * http://www.opensource.org/licenses/osl-3.0.php
9 *
10 * Development until 2012 by Earth Systems Science Computational Center (ESSCC)
11 * Development 2012-2013 by School of Earth Sciences
12 * Development from 2014 by Centre for Geoscience Computing (GeoComp)
13 *
14 *****************************************************************************/
15
16 #define ESNEEDPYTHON
17 #include "esysUtils/first.h"
18
19
20 #include <sstream>
21 #include <limits>
22 #include <boost/python/extract.hpp>
23 #include <boost/scoped_array.hpp>
24
25 #include "MPIDataReducer.h"
26 #include "SplitWorldException.h"
27
28 using namespace boost::python;
29 using namespace escript;
30
31
32 namespace escript
33 {
34 Reducer_ptr makeDataReducer(std::string type)
35 {
36 MPI_Op op;
37 if (type=="SUM")
38 {
39 op=MPI_SUM;
40 }
41 else if (type=="SET")
42 {
43 op=MPI_OP_NULL;
44 }
45 else
46 {
47 throw SplitWorldException("Unsupported operation for makeDataReducer.");
48 }
49 MPIDataReducer* m=new MPIDataReducer(op);
50 return Reducer_ptr(m);
51 }
52
53 }
54
55 namespace
56 {
57
58 void combineData(Data& d1, const Data& d2, MPI_Op op)
59 {
60 if (op==MPI_SUM)
61 {
62 d1+=d2;
63 }
64 else if (op==MPI_OP_NULL)
65 {
66 throw SplitWorldException("Multiple 'simultaneous' attempts to export a 'SET' variable.");
67 }
68 }
69
70 }
71
72 MPIDataReducer::MPIDataReducer(MPI_Op op)
73 : reduceop(op), had_an_export_this_round(false)
74 {
75 valueadded=false;
76 if ((op==MPI_SUM) || (op==MPI_OP_NULL))
77 {
78 // deliberately left blank
79 }
80 else
81 {
82 throw SplitWorldException("Unsupported MPI_Op");
83 }
84 }
85
86 void MPIDataReducer::newRunJobs()
87 {
88 had_an_export_this_round=false;
89 }
90
91 void MPIDataReducer::setDomain(escript::Domain_ptr d)
92 {
93 dom=d;
94 }
95
96 std::string MPIDataReducer::description()
97 {
98 std::string op="SUM";
99 if (reduceop==MPI_OP_NULL)
100 {
101 op="SET";
102 }
103 return "Reducer("+op+") for Data objects";
104 }
105
106 bool MPIDataReducer::valueCompatible(boost::python::object v)
107 {
108 extract<Data&> ex(v);
109 if (!ex.check())
110 {
111 return false;
112 }
113 if (dom.get()!=0)
114 {
115 const Data& d=ex();
116 if (d.getDomain().get()!=dom.get())
117 {
118 return false; // the domains don't match
119 }
120 }
121 return true;
122 }
123
124
125 bool MPIDataReducer::reduceLocalValue(boost::python::object v, std::string& errstring)
126 {
127 extract<Data&> ex(v);
128 if (!ex.check())
129 {
130 errstring="reduceLocalValue: expected Data object. Got something else.";
131 return false;
132 }
133 Data& d=ex();
134 if ((d.getDomain()!=dom) && (dom.get()!=0))
135 {
136 errstring="reduceLocalValue: Got a Data object, but it was not using the SubWorld's domain.";
137 return false;
138 }
139 d.expand(); // because I don't want to mess about with types of Data
140 if (!valueadded) // first value so answer becomes this one
141 {
142 value=d;
143 dom=d.getDomain();
144 had_an_export_this_round=true;
145 valueadded=true;
146 }
147 else
148 {
149
150 if (reduceop==MPI_OP_NULL)
151 {
152 if (had_an_export_this_round)
153 {
154 errstring="reduceLocalValue: Multiple 'simultaneous' attempts to export a 'SET' variable.";
155 return false;
156 }
157 value=d;
158 dom=d.getDomain();
159 had_an_export_this_round=true;
160 }
161 else
162 {
163 had_an_export_this_round=true;
164 if (d.getFunctionSpace()!=value.getFunctionSpace())
165 {
166 errstring="reduceLocalValue: FunctionSpaces for Data objects being combined must match.";
167 return false;
168 }
169 combineData(value, d, reduceop);
170 }
171 }
172 return true;
173 }
174
175 void MPIDataReducer::reset()
176 {
177 valueadded=false;
178 value=Data();
179 }
180
181 bool MPIDataReducer::checkRemoteCompatibility(esysUtils::JMPI& mpi_info, std::string& errstring)
182 {
183 #ifdef ESYS_MPI
184 // since they can't add it unless it is using the proper domain, we need to check
185
186 std::vector<unsigned> compat(6);
187 getCompatibilityInfo(compat);
188
189 // still need to incorporate domain version into this
190 // or are domains not mutable in any way that matters?
191 int* rbuff=new int[mpi_info->size*compat.size()];
192 boost::scoped_array<int> dummy(rbuff); // to ensure cleanup
193 for (int i=0;i<mpi_info->size;++i)
194 {
195 rbuff[i]=0; // since this won't match any valid value we can use it as a failure check
196 }
197 if (MPI_Allgather(&compat[0], compat.size(), MPI_UNSIGNED, rbuff,
198 compat.size(), MPI_UNSIGNED, mpi_info->comm)!=MPI_SUCCESS)
199 {
200 errstring="MPI failure in checkRemoteCompatibility.";
201 return false;
202 }
203 for (int i=0;i<mpi_info->size-1;++i)
204 {
205 for (int j=0;j<compat.size();++i)
206 {
207 if (rbuff[i*compat.size()+j]!=rbuff[(i+1)*compat.size()+j])
208 {
209 std::ostringstream oss;
210 oss << "Incompatible value found for SubWorld " << i+1 << '.';
211 errstring=oss.str();
212 return false;
213 }
214 }
215 }
216 return true;
217 #else
218 return true;
219 #endif
220 }
221
222 // By the time this function is called, we know that all the values
223 // are compatible
224 bool MPIDataReducer::reduceRemoteValues(esysUtils::JMPI& mpi_info, bool active)
225 {
226 if (!active)
227 {
228 return false; // shutting down this option until I implement it
229 }
230 #ifdef ESYS_MPI
231 DataTypes::ValueType& vr=value.getExpandedVectorReference();
232 Data result(0, value.getDataPointShape(), value.getFunctionSpace(), true);
233 DataTypes::ValueType& rr=value.getExpandedVectorReference();
234 if (reduceop==MPI_OP_NULL)
235 {
236 return false; // this will stop bad things happening but won't give an informative error message
237 }
238 if (MPI_Allreduce(&(vr[0]), &(rr[0]), vr.size(), MPI_DOUBLE, reduceop, mpi_info->comm)!=MPI_SUCCESS)
239 {
240 return false;
241 }
242 return true;
243 #else
244 return true;
245 #endif
246 }
247
248 // populate a vector of ints with enough information to ensure two values are compatible
249 // or to construct a container for incomming data
250 // Format for this:
251 // [0] Type of Data: {0 : error, 1: DataEmpty, 10: constant, 11:tagged, 12:expanded}
252 // [1] Functionspace type code
253 // [2] Only used for tagged --- gives the number of tags (which exist in the data object)
254 // [3..6] Components of the shape
255 void MPIDataReducer::getCompatibilityInfo(std::vector<unsigned>& params)
256 {
257 params.resize(7);
258 for (int i=0;i<7;++i)
259 {
260 params[0]=0;
261 }
262 if (value.isConstant())
263 {
264 params[0]=10;
265 }
266 else if (value.isTagged())
267 {
268 params[0]=11;
269 }
270 else if (value.isExpanded())
271 {
272 params[0]=12;
273 }
274 else // This could be DataEmpty or some other weirdness but we won't allow that
275 {
276 params[0]=0; // invalid type to send
277 }
278 params[1]=value.getFunctionSpace().getTypeCode();
279 params[2]=static_cast<unsigned>(value.getNumberOfTaggedValues());
280 const DataTypes::ShapeType& s=value.getDataPointShape();
281 for (int i=0;i<s.size();++i)
282 {
283 params[3+i]=s[i];
284 }
285 }
286
287
288 // Get a value for this variable from another process
289 // This is not a reduction and will replace any existing value
290 bool MPIDataReducer::recvFrom(Esys_MPI_rank localid, Esys_MPI_rank source, esysUtils::JMPI& mpiinfo)
291 {
292 #ifdef ESYS_MPI
293 // first we need to find out what we are expecting
294 unsigned params[7];
295 MPI_Status stat;
296 if (MPI_Recv(params, 7, MPI_UNSIGNED, source, PARAMTAG, mpiinfo->comm, &stat)!=MPI_SUCCESS)
297 {
298 return false;
299 }
300 if (params[0]<10) // the sender somehow tried to send something invalid
301 {
302 return false;
303 }
304 // now we put the shape object together
305 escript::DataTypes::ShapeType s;
306 for (int i=0;i<4;++i)
307 {
308 if (params[3+i]>0)
309 {
310 s.push_back(params[3+i]);
311 }
312 else
313 {
314 break;
315 }
316 }
317 // Now we need the FunctionSpace
318 FunctionSpace fs=FunctionSpace(dom, static_cast<int>(params[1]));
319 value=Data(0, s, fs, params[0]==12);
320 if (params[0]==11) // The Data is tagged so we need to work out what tags we need
321 {
322 // TODO: Need to ship the tags and names over but for now just make sure there
323 // are the same number of tags
324 value.tag();
325
326 DataVector dv(DataTypes::noValues(s), 0, 1);
327 for (unsigned i=0;i<params[2];++i)
328 {
329 value.setTaggedValueFromCPP(static_cast<int>(i)+1, s, dv, 0);
330 }
331 return false; // because I don't trust this yet
332 }
333 #endif
334 return true;
335 }
336
337 // Send a value to this variable to another process
338 // This is not a reduction and will replace any existing value
339 bool MPIDataReducer::sendTo(Esys_MPI_rank localid, Esys_MPI_rank target, esysUtils::JMPI& mpiinfo)
340 {
341 #ifdef ESYS_MPI
342 // first step is to let the other world know what sort of thing it needs to make
343 if (value.isLazy())
344 {
345 value.resolve();
346 }
347 std::vector<unsigned> params;
348 getCompatibilityInfo(params);
349 if (MPI_Send(&params[0], 6, MPI_UNSIGNED, target, PARAMTAG, mpiinfo->comm)!=MPI_SUCCESS)
350 {
351 return false;
352 }
353 // now we have informed the other end of what happened
354 // are we done or is there actually data to send
355 if (params[0]<10)
356 {
357 return false;
358 }
359 // at this point, we know there is data to send
360 const DataAbstract::ValueType::value_type* vect=value.getDataRO();
361 // now the receiver knows how much data it should be receive
362 // need to make sure that we aren't trying to send data with no local samples
363 if (vect!=0)
364 {
365 // MPI v3 has this first param as a const void* (as it should be)
366 // Version on my machine expects void*
367 if (MPI_Send(const_cast<DataAbstract::ValueType::value_type*>(vect), value.getLength(), MPI_DOUBLE, target, PARAMTAG, mpiinfo->comm)!=MPI_SUCCESS)
368 {
369 return false;
370 }
371 }
372 #endif
373 return true;
374 }
375
376 boost::python::object MPIDataReducer::getPyObj()
377 {
378 throw SplitWorldException("getPyObj Not implemented yet.");
379 }
380
381
382 // send from proc 0 in the communicator to all others
383 bool MPIDataReducer::groupSend(MPI_Comm& com)
384 {
385 throw SplitWorldException("groupSend Not implemented yet.");
386 }
387
388 bool MPIDataReducer::groupReduce(MPI_Comm& com, char mystate)
389 {
390 throw SplitWorldException("groupReduce Not implemented yet.");
391 }
392

  ViewVC Help
Powered by ViewVC 1.1.26