/[escript]/branches/split/escriptcore/src/SplitWorld.cpp
ViewVC logotype

Annotation of /branches/split/escriptcore/src/SplitWorld.cpp

Parent Directory Parent Directory | Revision Log Revision Log


Revision 4732 - (hide annotations)
Mon Mar 10 04:20:03 2014 UTC (5 years, 5 months ago) by jfenwick
File size: 5788 byte(s)
File rename
1 jfenwick 4730 /*****************************************************************************
2     *
3     * Copyright (c) 2014 by University of Queensland
4     * http://www.uq.edu.au
5     *
6     * Primary Business: Queensland, Australia
7     * Licensed under the Open Software License version 3.0
8     * http://www.opensource.org/licenses/osl-3.0.php
9     *
10     * Development until 2012 by Earth Systems Science Computational Center (ESSCC)
11     * Development 2012-2013 by School of Earth Sciences
12     * Development from 2014 by Centre for Geoscience Computing (GeoComp)
13     *
14     *****************************************************************************/
15    
16     #include "esysUtils/Esys_MPI.h"
17     #include "WorldSplitter.h"
18     #include "AbstractDomain.h"
19     #include "DomainException.h"
20 jfenwick 4731 #include "SplitWorldException.h"
21 jfenwick 4730
22 jfenwick 4731 #include <iostream>
23    
24 jfenwick 4730 using namespace boost::python;
25     using namespace escript;
26    
27     WorldSplitter::WorldSplitter(unsigned int numgroups, MPI_Comm global)
28     :globalcom(global), subcom(MPI_COMM_NULL), localworld((SubWorld*)0), groupcount(numgroups)
29     {
30     int gsize;
31     int grank;
32 jfenwick 4731 if ((MPI_Comm_size(global, &gsize)!=MPI_SUCCESS) || (MPI_Comm_rank(global, &grank)!=MPI_SUCCESS))
33 jfenwick 4730 {
34     throw DomainException("MPI appears to be inoperative.");
35     }
36     if (gsize%numgroups!=0)
37     {
38     throw DomainException("WorldSplitter error: requested number of groups is not a factor of global communicator size.");
39     }
40 jfenwick 4731 int wsize=gsize/numgroups; // each world has this many processes
41     int res=MPI_Comm_split(MPI_COMM_WORLD, grank/wsize, grank%wsize, &subcom);
42 jfenwick 4730 if (res!=MPI_SUCCESS)
43     {
44     throw DomainException("WorldSplitter error: Unable to form communicator.");
45 jfenwick 4731 }
46 jfenwick 4730 localworld=SubWorld_ptr(new SubWorld(subcom));
47 jfenwick 4731 localid=grank/wsize;
48 jfenwick 4730 }
49    
50     // We may need to look into this more closely.
51     // What if the domain lives longer than the world splitter?
52     WorldSplitter::~WorldSplitter()
53     {
54     if (subcom!=MPI_COMM_NULL)
55     {
56     MPI_Comm_free(&subcom);
57     }
58     }
59    
60    
61     // The boost wrapper will ensure that there is at least one entry in the tuple
62     object WorldSplitter::buildDomains(tuple t, dict kwargs)
63     {
64     int tsize=len(t);
65     // get the callable that we will invoke in a sec
66     object tocall=t[0];
67     // make a new tuple without the first element
68     tuple ntup=tuple(t.slice(1,tsize));
69     // now add the subworld to the kwargs
70     kwargs["escriptworld"]=localworld;
71    
72     // std::cerr << "About to call function with:\n";
73     // //extract<std::string> ex(ntup.attr("__str__")());
74     // std::cerr << extract<std::string>(ntup.attr("__str__")())() << std::endl;
75     // // for (int i=0;i<tsize-1;++i)
76     // // {
77     // // std::cout << extract<const char*>(ntup[i])() << " ";
78     // // }
79     // std::cerr << std::endl;
80    
81     // pass the whole package to the python call
82     object dobj=tocall(*ntup, **kwargs);
83     extract<Domain_ptr> ex1(dobj);
84     Domain_ptr dptr=ex1();
85    
86     // now do a sanity check to see if the domain has respected the communicator info we passed it.
87     if (dptr->getMPIComm()!=localworld->getComm())
88     {
89     throw DomainException("The newly constructed domain is not using the correct communicator.");
90     }
91     localworld->setDomain(dptr);
92     return object(); // return None
93     }
94    
95 jfenwick 4731 /** a list of tuples/sequences: (Job class, number of instances)*/
96     void WorldSplitter::runJobs(boost::python::list l)
97     {
98     // first count up how many jobs we have in total
99     unsigned int numjobs=0;
100     std::vector<object> classvec;
101     std::vector<unsigned int> countvec;
102     std::vector<unsigned int> partialcounts;
103     for (int i=0;i<len(l);++i)
104     {
105     extract<tuple> ex(l[i]);
106     if (!ex.check())
107     {
108     throw SplitWorldException("runJobs takes a list of tuples (jobclass, number).");
109     }
110     tuple t=ex();
111     if (len(t)!=2)
112     {
113     throw SplitWorldException("runJobs takes a list of tuples (jobclass, number).");
114     }
115     extract<unsigned int> ex2(t[1]);
116     unsigned int c=0;
117     if (!ex2.check() || ((c=ex2())==0))
118     {
119     throw SplitWorldException("Number of jobs must be a strictly positive integer.");
120    
121     }
122     classvec.push_back(t[0]);
123     countvec.push_back(c);
124     numjobs+=c;
125     partialcounts.push_back(numjobs);
126     }
127     unsigned int classnum=0;
128     unsigned int lowend=1;
129     unsigned int highend=lowend+numjobs/groupcount+(numjobs%groupcount);
130     std::cout << localid << std::endl;
131     for (int i=1;i<=localid;++i)
132     {
133     lowend=highend;
134     highend=lowend+numjobs/groupcount;
135     if (i<numjobs%groupcount)
136     {
137     highend++;
138     }
139     }
140     std::cout << "There are " << numjobs << " jobs with range [" << lowend << ", " << highend << ")\n";
141     // We could do something more clever about trying to fit Jobs to subworlds
142     // to ensure that instances sharing the same Job class would share the same
143     // world as much as possible but for now we'll do this:
144     for (unsigned int j=1;j<=numjobs;++j) // job #0 is a sentinel
145     {
146     if (j>partialcounts[classnum])
147     {
148     classnum++; // we dont' need to loop this because each count >0
149     }
150     // now if this is one of the job numbers in our local range,
151     // create an instance of the appropriate class
152     if (j>=lowend and j<highend)
153     {
154     std::cout << "Added job\n";
155     object o=classvec[classnum](localworld->getDomain(), object(j));
156     localworld->addJob(o);
157     }
158     }
159    
160     // now we actually need to run the jobs
161     // everybody will be executing their localworld's jobs
162     localworld->runJobs();
163     }
164    
165 jfenwick 4730 namespace escript
166     {
167    
168     boost::python::object raw_buildDomains(boost::python::tuple t, boost::python::dict kwargs)
169     {
170     int l=len(t);
171     if (l<2)
172     {
173     throw DomainException("Insufficient parameters to buildDomains.");
174     }
175     extract<WorldSplitter&> exw(t[0]);
176     if (!exw.check())
177     {
178     throw DomainException("First parameter to buildDomains must be a WorldSplitter.");
179     }
180     WorldSplitter& ws=exw();
181     tuple ntup=tuple(t.slice(1,l)); // strip off the object param
182     return ws.buildDomains(ntup, kwargs);
183     }
184    
185     }

  ViewVC Help
Powered by ViewVC 1.1.26