/[escript]/branches/split/escriptcore/src/WorldSplitter.cpp
ViewVC logotype

Contents of /branches/split/escriptcore/src/WorldSplitter.cpp

Parent Directory Parent Directory | Revision Log Revision Log


Revision 4731 - (show annotations)
Mon Mar 10 04:16:07 2014 UTC (3 years, 7 months ago) by jfenwick
File size: 5788 byte(s)
Creating empty Jobs
1 /*****************************************************************************
2 *
3 * Copyright (c) 2014 by University of Queensland
4 * http://www.uq.edu.au
5 *
6 * Primary Business: Queensland, Australia
7 * Licensed under the Open Software License version 3.0
8 * http://www.opensource.org/licenses/osl-3.0.php
9 *
10 * Development until 2012 by Earth Systems Science Computational Center (ESSCC)
11 * Development 2012-2013 by School of Earth Sciences
12 * Development from 2014 by Centre for Geoscience Computing (GeoComp)
13 *
14 *****************************************************************************/
15
16 #include "esysUtils/Esys_MPI.h"
17 #include "WorldSplitter.h"
18 #include "AbstractDomain.h"
19 #include "DomainException.h"
20 #include "SplitWorldException.h"
21
22 #include <iostream>
23
24 using namespace boost::python;
25 using namespace escript;
26
27 WorldSplitter::WorldSplitter(unsigned int numgroups, MPI_Comm global)
28 :globalcom(global), subcom(MPI_COMM_NULL), localworld((SubWorld*)0), groupcount(numgroups)
29 {
30 int gsize;
31 int grank;
32 if ((MPI_Comm_size(global, &gsize)!=MPI_SUCCESS) || (MPI_Comm_rank(global, &grank)!=MPI_SUCCESS))
33 {
34 throw DomainException("MPI appears to be inoperative.");
35 }
36 if (gsize%numgroups!=0)
37 {
38 throw DomainException("WorldSplitter error: requested number of groups is not a factor of global communicator size.");
39 }
40 int wsize=gsize/numgroups; // each world has this many processes
41 int res=MPI_Comm_split(MPI_COMM_WORLD, grank/wsize, grank%wsize, &subcom);
42 if (res!=MPI_SUCCESS)
43 {
44 throw DomainException("WorldSplitter error: Unable to form communicator.");
45 }
46 localworld=SubWorld_ptr(new SubWorld(subcom));
47 localid=grank/wsize;
48 }
49
50 // We may need to look into this more closely.
51 // What if the domain lives longer than the world splitter?
52 WorldSplitter::~WorldSplitter()
53 {
54 if (subcom!=MPI_COMM_NULL)
55 {
56 MPI_Comm_free(&subcom);
57 }
58 }
59
60
61 // The boost wrapper will ensure that there is at least one entry in the tuple
62 object WorldSplitter::buildDomains(tuple t, dict kwargs)
63 {
64 int tsize=len(t);
65 // get the callable that we will invoke in a sec
66 object tocall=t[0];
67 // make a new tuple without the first element
68 tuple ntup=tuple(t.slice(1,tsize));
69 // now add the subworld to the kwargs
70 kwargs["escriptworld"]=localworld;
71
72 // std::cerr << "About to call function with:\n";
73 // //extract<std::string> ex(ntup.attr("__str__")());
74 // std::cerr << extract<std::string>(ntup.attr("__str__")())() << std::endl;
75 // // for (int i=0;i<tsize-1;++i)
76 // // {
77 // // std::cout << extract<const char*>(ntup[i])() << " ";
78 // // }
79 // std::cerr << std::endl;
80
81 // pass the whole package to the python call
82 object dobj=tocall(*ntup, **kwargs);
83 extract<Domain_ptr> ex1(dobj);
84 Domain_ptr dptr=ex1();
85
86 // now do a sanity check to see if the domain has respected the communicator info we passed it.
87 if (dptr->getMPIComm()!=localworld->getComm())
88 {
89 throw DomainException("The newly constructed domain is not using the correct communicator.");
90 }
91 localworld->setDomain(dptr);
92 return object(); // return None
93 }
94
95 /** a list of tuples/sequences: (Job class, number of instances)*/
96 void WorldSplitter::runJobs(boost::python::list l)
97 {
98 // first count up how many jobs we have in total
99 unsigned int numjobs=0;
100 std::vector<object> classvec;
101 std::vector<unsigned int> countvec;
102 std::vector<unsigned int> partialcounts;
103 for (int i=0;i<len(l);++i)
104 {
105 extract<tuple> ex(l[i]);
106 if (!ex.check())
107 {
108 throw SplitWorldException("runJobs takes a list of tuples (jobclass, number).");
109 }
110 tuple t=ex();
111 if (len(t)!=2)
112 {
113 throw SplitWorldException("runJobs takes a list of tuples (jobclass, number).");
114 }
115 extract<unsigned int> ex2(t[1]);
116 unsigned int c=0;
117 if (!ex2.check() || ((c=ex2())==0))
118 {
119 throw SplitWorldException("Number of jobs must be a strictly positive integer.");
120
121 }
122 classvec.push_back(t[0]);
123 countvec.push_back(c);
124 numjobs+=c;
125 partialcounts.push_back(numjobs);
126 }
127 unsigned int classnum=0;
128 unsigned int lowend=1;
129 unsigned int highend=lowend+numjobs/groupcount+(numjobs%groupcount);
130 std::cout << localid << std::endl;
131 for (int i=1;i<=localid;++i)
132 {
133 lowend=highend;
134 highend=lowend+numjobs/groupcount;
135 if (i<numjobs%groupcount)
136 {
137 highend++;
138 }
139 }
140 std::cout << "There are " << numjobs << " jobs with range [" << lowend << ", " << highend << ")\n";
141 // We could do something more clever about trying to fit Jobs to subworlds
142 // to ensure that instances sharing the same Job class would share the same
143 // world as much as possible but for now we'll do this:
144 for (unsigned int j=1;j<=numjobs;++j) // job #0 is a sentinel
145 {
146 if (j>partialcounts[classnum])
147 {
148 classnum++; // we dont' need to loop this because each count >0
149 }
150 // now if this is one of the job numbers in our local range,
151 // create an instance of the appropriate class
152 if (j>=lowend and j<highend)
153 {
154 std::cout << "Added job\n";
155 object o=classvec[classnum](localworld->getDomain(), object(j));
156 localworld->addJob(o);
157 }
158 }
159
160 // now we actually need to run the jobs
161 // everybody will be executing their localworld's jobs
162 localworld->runJobs();
163 }
164
165 namespace escript
166 {
167
168 boost::python::object raw_buildDomains(boost::python::tuple t, boost::python::dict kwargs)
169 {
170 int l=len(t);
171 if (l<2)
172 {
173 throw DomainException("Insufficient parameters to buildDomains.");
174 }
175 extract<WorldSplitter&> exw(t[0]);
176 if (!exw.check())
177 {
178 throw DomainException("First parameter to buildDomains must be a WorldSplitter.");
179 }
180 WorldSplitter& ws=exw();
181 tuple ntup=tuple(t.slice(1,l)); // strip off the object param
182 return ws.buildDomains(ntup, kwargs);
183 }
184
185 }

  ViewVC Help
Powered by ViewVC 1.1.26