/[escript]/trunk/escriptcore/src/MPIDataReducer.cpp
ViewVC logotype

Annotation of /trunk/escriptcore/src/MPIDataReducer.cpp

Parent Directory Parent Directory | Revision Log Revision Log


Revision 5643 - (hide annotations)
Thu Jun 11 22:50:48 2015 UTC (4 years, 1 month ago) by jfenwick
File size: 10082 byte(s)
adding set only reducer
1 jfenwick 5490 /*****************************************************************************
2     *
3 jfenwick 5593 * Copyright (c) 2014-2015 by The University of Queensland
4 jfenwick 5490 * http://www.uq.edu.au
5     *
6     * Primary Business: Queensland, Australia
7     * Licensed under the Open Software License version 3.0
8     * http://www.opensource.org/licenses/osl-3.0.php
9     *
10     * Development until 2012 by Earth Systems Science Computational Center (ESSCC)
11     * Development 2012-2013 by School of Earth Sciences
12     * Development from 2014 by Centre for Geoscience Computing (GeoComp)
13     *
14     *****************************************************************************/
15    
16     #define ESNEEDPYTHON
17     #include "esysUtils/first.h"
18    
19    
20     #include <sstream>
21     #include <limits>
22     #include <boost/python/extract.hpp>
23     #include <boost/scoped_array.hpp>
24    
25     #include "MPIDataReducer.h"
26     #include "SplitWorldException.h"
27    
28     using namespace boost::python;
29     using namespace escript;
30    
31    
32     namespace escript
33     {
34     Reducer_ptr makeDataReducer(std::string type)
35     {
36     MPI_Op op;
37     if (type=="SUM")
38     {
39     op=MPI_SUM;
40     }
41 jfenwick 5643 else if (type=="SET")
42     {
43     op=MPI_OP_NULL;
44     }
45 jfenwick 5490 else
46     {
47     throw SplitWorldException("Unsupported operation for makeDataReducer.");
48     }
49     MPIDataReducer* m=new MPIDataReducer(op);
50     return Reducer_ptr(m);
51     }
52    
53     }
54    
55     namespace
56     {
57    
58     void combineData(Data& d1, const Data& d2, MPI_Op op)
59     {
60     if (op==MPI_SUM)
61     {
62     d1+=d2;
63 jfenwick 5643 }
64     else if (op==MPI_OP_NULL)
65     {
66     throw SplitWorldException("Multiple 'simultaneous' attempts to export a 'SET' variable.");
67 jfenwick 5490 }
68     }
69    
70     }
71    
72     MPIDataReducer::MPIDataReducer(MPI_Op op)
73 jfenwick 5643 : reduceop(op), had_an_export_this_round(false)
74 jfenwick 5490 {
75     valueadded=false;
76 jfenwick 5643 if ((op==MPI_SUM) || (op==MPI_OP_NULL))
77 jfenwick 5490 {
78     // deliberately left blank
79     }
80     else
81     {
82     throw SplitWorldException("Unsupported MPI_Op");
83     }
84     }
85    
86 jfenwick 5643 void MPIDataReducer::newRunJobs()
87     {
88     had_an_export_this_round=false;
89     }
90 jfenwick 5490
91     void MPIDataReducer::setDomain(escript::Domain_ptr d)
92     {
93     dom=d;
94     }
95    
96     std::string MPIDataReducer::description()
97     {
98     std::string op="SUM";
99 jfenwick 5643 if (reduceop==MPI_OP_NULL)
100     {
101     op="SET";
102     }
103 jfenwick 5490 return "Reducer("+op+") for Data objects";
104     }
105    
106     bool MPIDataReducer::valueCompatible(boost::python::object v)
107     {
108     extract<Data&> ex(v);
109     if (!ex.check())
110     {
111     return false;
112     }
113     if (dom.get()!=0)
114     {
115     const Data& d=ex();
116     if (d.getDomain().get()!=dom.get())
117     {
118     return false; // the domains don't match
119     }
120     }
121     return true;
122     }
123    
124    
125     bool MPIDataReducer::reduceLocalValue(boost::python::object v, std::string& errstring)
126     {
127     extract<Data&> ex(v);
128     if (!ex.check())
129     {
130     errstring="reduceLocalValue: expected Data object. Got something else.";
131     return false;
132     }
133     Data& d=ex();
134     if ((d.getDomain()!=dom) && (dom.get()!=0))
135     {
136     errstring="reduceLocalValue: Got a Data object, but it was not using the SubWorld's domain.";
137     return false;
138     }
139     d.expand(); // because I don't want to mess about with types of Data
140     if (!valueadded) // first value so answer becomes this one
141     {
142     value=d;
143     dom=d.getDomain();
144 jfenwick 5643 had_an_export_this_round=true;
145 jfenwick 5490 }
146     else
147     {
148 jfenwick 5643
149     if (reduceop==MPI_OP_NULL)
150 jfenwick 5490 {
151 jfenwick 5643 if (had_an_export_this_round)
152     {
153     errstring="reduceLocalValue: Multiple 'simultaneous' attempts to export a 'SET' variable.";
154     return false;
155     }
156     value=d;
157     dom=d.getDomain();
158     had_an_export_this_round=true;
159 jfenwick 5490 }
160 jfenwick 5643 else
161     {
162     had_an_export_this_round=true;
163     if (d.getFunctionSpace()!=value.getFunctionSpace())
164     {
165     errstring="reduceLocalValue: FunctionSpaces for Data objects being combined must match.";
166     return false;
167     }
168     combineData(value, d, reduceop);
169     }
170 jfenwick 5490 }
171     return true;
172     }
173    
174     void MPIDataReducer::reset()
175     {
176     valueadded=false;
177     value=Data();
178     }
179    
180     bool MPIDataReducer::checkRemoteCompatibility(esysUtils::JMPI& mpi_info, std::string& errstring)
181     {
182     #ifdef ESYS_MPI
183     // since they can't add it unless it is using the proper domain, we need to check
184    
185     std::vector<unsigned> compat(6);
186     getCompatibilityInfo(compat);
187    
188     // still need to incorporate domain version into this
189     // or are domains not mutable in any way that matters?
190     int* rbuff=new int[mpi_info->size*compat.size()];
191     boost::scoped_array<int> dummy(rbuff); // to ensure cleanup
192     for (int i=0;i<mpi_info->size;++i)
193     {
194     rbuff[i]=0; // since this won't match any valid value we can use it as a failure check
195     }
196     if (MPI_Allgather(&compat[0], compat.size(), MPI_UNSIGNED, rbuff,
197     compat.size(), MPI_UNSIGNED, mpi_info->comm)!=MPI_SUCCESS)
198     {
199     errstring="MPI failure in checkRemoteCompatibility.";
200     return false;
201     }
202     for (int i=0;i<mpi_info->size-1;++i)
203     {
204     for (int j=0;j<compat.size();++i)
205     {
206     if (rbuff[i*compat.size()+j]!=rbuff[(i+1)*compat.size()+j])
207     {
208     std::ostringstream oss;
209     oss << "Incompatible value found for SubWorld " << i+1 << '.';
210     errstring=oss.str();
211     return false;
212     }
213     }
214     }
215     return true;
216     #else
217     return true;
218     #endif
219     }
220    
221     // By the time this function is called, we know that all the values
222     // are compatible
223     bool MPIDataReducer::reduceRemoteValues(esysUtils::JMPI& mpi_info, bool active)
224     {
225     if (!active)
226     {
227     return false; // shutting down this option until I implement it
228     }
229     #ifdef ESYS_MPI
230     DataTypes::ValueType& vr=value.getExpandedVectorReference();
231     Data result(0, value.getDataPointShape(), value.getFunctionSpace(), true);
232     DataTypes::ValueType& rr=value.getExpandedVectorReference();
233 jfenwick 5643 if (reduceop==MPI_OP_NULL)
234     {
235     return false; // this will stop bad things happening but won't give an informative error message
236     }
237 jfenwick 5490 if (MPI_Allreduce(&(vr[0]), &(rr[0]), vr.size(), MPI_DOUBLE, reduceop, mpi_info->comm)!=MPI_SUCCESS)
238     {
239     return false;
240     }
241     return true;
242     #else
243     return true;
244     #endif
245     }
246    
247     // populate a vector of ints with enough information to ensure two values are compatible
248     // or to construct a container for incomming data
249     // Format for this:
250     // [0] Type of Data: {0 : error, 1: DataEmpty, 10: constant, 11:tagged, 12:expanded}
251     // [1] Functionspace type code
252     // [2] Only used for tagged --- gives the number of tags (which exist in the data object)
253     // [3..6] Components of the shape
254     void MPIDataReducer::getCompatibilityInfo(std::vector<unsigned>& params)
255     {
256     params.resize(7);
257     for (int i=0;i<7;++i)
258     {
259     params[0]=0;
260     }
261     if (value.isConstant())
262     {
263     params[0]=10;
264     }
265     else if (value.isTagged())
266     {
267     params[0]=11;
268     }
269     else if (value.isExpanded())
270     {
271     params[0]=12;
272     }
273     else // This could be DataEmpty or some other weirdness but we won't allow that
274     {
275     params[0]=0; // invalid type to send
276     }
277     params[1]=value.getFunctionSpace().getTypeCode();
278     params[2]=static_cast<unsigned>(value.getNumberOfTaggedValues());
279     const DataTypes::ShapeType& s=value.getDataPointShape();
280     for (int i=0;i<s.size();++i)
281     {
282     params[3+i]=s[i];
283     }
284     }
285    
286    
287     // Get a value for this variable from another process
288     // This is not a reduction and will replace any existing value
289     bool MPIDataReducer::recvFrom(Esys_MPI_rank localid, Esys_MPI_rank source, esysUtils::JMPI& mpiinfo)
290     {
291     #ifdef ESYS_MPI
292     // first we need to find out what we are expecting
293     unsigned params[7];
294     MPI_Status stat;
295     if (MPI_Recv(params, 7, MPI_UNSIGNED, source, PARAMTAG, mpiinfo->comm, &stat)!=MPI_SUCCESS)
296     {
297     return false;
298     }
299     if (params[0]<10) // the sender somehow tried to send something invalid
300     {
301     return false;
302     }
303     // now we put the shape object together
304     escript::DataTypes::ShapeType s;
305     for (int i=0;i<4;++i)
306     {
307     if (params[3+i]>0)
308     {
309     s.push_back(params[3+i]);
310     }
311     else
312     {
313     break;
314     }
315     }
316     // Now we need the FunctionSpace
317     FunctionSpace fs=FunctionSpace(dom, static_cast<int>(params[1]));
318     value=Data(0, s, fs, params[0]==12);
319     if (params[0]==11) // The Data is tagged so we need to work out what tags we need
320     {
321     // TODO: Need to ship the tags and names over but for now just make sure there
322     // are the same number of tags
323     value.tag();
324    
325     DataVector dv(DataTypes::noValues(s), 0, 1);
326     for (unsigned i=0;i<params[2];++i)
327     {
328     value.setTaggedValueFromCPP(static_cast<int>(i)+1, s, dv, 0);
329     }
330     return false; // because I don't trust this yet
331     }
332     #endif
333     return true;
334     }
335    
336     // Send a value to this variable to another process
337     // This is not a reduction and will replace any existing value
338     bool MPIDataReducer::sendTo(Esys_MPI_rank localid, Esys_MPI_rank target, esysUtils::JMPI& mpiinfo)
339     {
340     #ifdef ESYS_MPI
341     // first step is to let the other world know what sort of thing it needs to make
342     if (value.isLazy())
343     {
344     value.resolve();
345     }
346     std::vector<unsigned> params;
347     getCompatibilityInfo(params);
348     if (MPI_Send(&params[0], 6, MPI_UNSIGNED, target, PARAMTAG, mpiinfo->comm)!=MPI_SUCCESS)
349     {
350     return false;
351     }
352     // now we have informed the other end of what happened
353     // are we done or is there actually data to send
354     if (params[0]<10)
355     {
356     return false;
357     }
358     // at this point, we know there is data to send
359     const DataAbstract::ValueType::value_type* vect=value.getDataRO();
360     // now the receiver knows how much data it should be receive
361     // need to make sure that we aren't trying to send data with no local samples
362     if (vect!=0)
363     {
364     // MPI v3 has this first param as a const void* (as it should be)
365     // Version on my machine expects void*
366     if (MPI_Send(const_cast<DataAbstract::ValueType::value_type*>(vect), value.getLength(), MPI_DOUBLE, target, PARAMTAG, mpiinfo->comm)!=MPI_SUCCESS)
367     {
368     return false;
369     }
370     }
371     #endif
372     return true;
373     }
374    
375     boost::python::object MPIDataReducer::getPyObj()
376     {
377     throw SplitWorldException("getPyObj Not implemented yet.");
378     }
379    
380    
381     // send from proc 0 in the communicator to all others
382     bool MPIDataReducer::groupSend(MPI_Comm& com)
383     {
384     throw SplitWorldException("groupSend Not implemented yet.");
385     }
386    
387     bool MPIDataReducer::groupReduce(MPI_Comm& com, char mystate)
388     {
389     throw SplitWorldException("groupReduce Not implemented yet.");
390     }
391    

  ViewVC Help
Powered by ViewVC 1.1.26