1 |
filter# $Id:$ |
2 |
|
3 |
# |
4 |
# COPYRIGHT ACcESS 2004 - All Rights Reserved |
5 |
# |
6 |
# This software is the property of ACcESS. No part of this code |
7 |
# may be copied in any form or by any means without the expressed written |
8 |
# consent of ACcESS. Copying, use or modification of this software |
9 |
# by any unauthorised person is illegal unless that |
10 |
# person has a software license agreement with ACcESS. |
11 |
# |
12 |
|
13 |
""" |
14 |
A simple framework to run benchmarks under OPENMP and to summarize the results in tables for instance in HTML |
15 |
|
16 |
@var __author__: name of author |
17 |
@var __licence__: licence agreement |
18 |
var __url__: url entry point on documentation |
19 |
@var __version__: version |
20 |
@var __date__: date of the version |
21 |
""" |
22 |
|
23 |
__author__="Lutz Gross, l.gross@uq.edu.au" |
24 |
__licence__="contact: esys@access.uq.edu.au" |
25 |
__url__="http://www.iservo.edu.au/esys/escript" |
26 |
__version__="$Revision:$" |
27 |
__date__="$Date:$" |
28 |
|
29 |
import os,socket,time,sys |
30 |
|
31 |
class BenchmarkSuite(object): |
32 |
""" |
33 |
framework to run a bunch of L{Benchmark}s with the object to create a table of statistics. |
34 |
@var MAX_LEVEL: maximum number of level in headers for output |
35 |
""" |
36 |
MAX_LEVEL=5 |
37 |
def __init__(self,name=None): |
38 |
""" |
39 |
sets up a suite of benchmarks |
40 |
|
41 |
@param name: name of the benchmark suite. If no name is given the class name is used. |
42 |
@type name: C{str} |
43 |
""" |
44 |
super(BenchmarkSuite,self).__init__() |
45 |
self.__benchmarks=[] |
46 |
self.__scale=1 |
47 |
if name==None: |
48 |
self.__name=self.__class__.__name__ |
49 |
else: |
50 |
self.__name=name |
51 |
|
52 |
def __str__(self): |
53 |
""" |
54 |
returns the name of the benchmark suite |
55 |
|
56 |
@return: name |
57 |
@rtype: C{str} |
58 |
""" |
59 |
return self.__name |
60 |
def addBenchmark(self,benchmark): |
61 |
""" |
62 |
adds a L{Benchmark} to the suite |
63 |
|
64 |
@param benchmark: adds a new L{Benchmark} to the suite |
65 |
@type benchmark: L{Benchmark} |
66 |
""" |
67 |
self.__benchmarks.append(benchmark) |
68 |
def __len__(self): |
69 |
""" |
70 |
returns the number of benchmarks in the suite |
71 |
|
72 |
@return: number of benchmarks |
73 |
@rtype: C{int} |
74 |
""" |
75 |
return len(self.__benchmarks) |
76 |
def __getitem__(self,i): |
77 |
""" |
78 |
returns the i-th benchmark in the suite through self[i] |
79 |
|
80 |
@param i: index of the requested benchmark |
81 |
@type i: C{int} |
82 |
@return: i-th benchmark |
83 |
@rtype: L{Benchmark} |
84 |
|
85 |
""" |
86 |
return self.__benchmarks[i] |
87 |
def run(self,scale=1): |
88 |
""" |
89 |
runs all benchmarks |
90 |
|
91 |
@param scale: defines the number of (OpenMP) threads to be used. If scale is a scalar all benchmarks |
92 |
are run with scale number of threads. If scale is a C{list}, the p-th problem in each of the benchmarks |
93 |
in the suite is run with scale[p] threads. If scale[p]<1 teh p-th problem is omitted. |
94 |
@type scale: C{int} or C{list} of C{int}s. |
95 |
""" |
96 |
self.__scale=scale |
97 |
for i in range(len(self)): self[i].run(scale=scale) |
98 |
def getHTML(self,filter,level=1): |
99 |
""" |
100 |
returns the results of the last benchmark run in HTML format. |
101 |
|
102 |
@param filter: filter to be applied to the results |
103 |
@type filter: L{BenchmarkFilter} |
104 |
@param level: level used in header <H?> tags |
105 |
@type level: C{int} |
106 |
@return: HTML document |
107 |
@rtype: C{str} |
108 |
""" |
109 |
out="" |
110 |
if level==1: out+="<HTML><HEAD><TITLE>Benchmark: %s</TITLE></HEAD><BODY>\n"%str(self) |
111 |
out+="<H%s>%s</H%s>\n"%(level,str(self),level) |
112 |
if level==1: |
113 |
m="" |
114 |
if isinstance(self.__scale,int): |
115 |
if self.__scale>1: |
116 |
m=" (%s threads)"%self.__scale |
117 |
out+="<p>platform: %s%s</p>\n"%(socket.gethostname(),m) |
118 |
for i in range(len(self)): |
119 |
out+="<p>\n" |
120 |
out+=self[i].getHTML(filter=filter,level=min(level+1,self.MAX_LEVEL)) |
121 |
out+="<p>\n" |
122 |
if level==1: |
123 |
try: |
124 |
name=os.getlogin() |
125 |
out+="<hr><p align=\"center\">by %s at %s</p>\n"%(name,time.strftime('%X %x %Z')) |
126 |
except OSError: |
127 |
out+="<hr><p align=\"center\">%s</p>\n"%(time.strftime('%X %x %Z')) |
128 |
|
129 |
out+="</BODY></HTML>\n" |
130 |
return out |
131 |
|
132 |
|
133 |
class Benchmark(object): |
134 |
""" |
135 |
runs a bunch of similar L{BenchmarkProblem}s with a bunch of L{Options} |
136 |
""" |
137 |
def __init__(self,name=None,description=None): |
138 |
""" |
139 |
sets up a benchmark |
140 |
|
141 |
@param name: name of the benchmark. If no name is given the class name is used. |
142 |
@type name: C{str} |
143 |
@param description: description of the benchmark. |
144 |
@type description: C{str} or C{None} |
145 |
""" |
146 |
super(Benchmark,self).__init__() |
147 |
self.__options=[] |
148 |
self.__problems=[] |
149 |
self.__results=[] |
150 |
self.__scale=1 |
151 |
if name==None: |
152 |
self.__name=self.__class__.__name__ |
153 |
else: |
154 |
self.__name=name |
155 |
self.__description=description |
156 |
|
157 |
def __str__(self): |
158 |
""" |
159 |
returns the name of the benchmark suite |
160 |
|
161 |
@return: name |
162 |
@rtype: C{str} |
163 |
""" |
164 |
return self.__name |
165 |
|
166 |
def addProblem(self,problem): |
167 |
""" |
168 |
adds a problem to the benchmark |
169 |
|
170 |
@param problem: adds a new problem to the bechmark |
171 |
@type problem: L{BenchmarkProblem} |
172 |
""" |
173 |
self.__problems.append(problem) |
174 |
|
175 |
def addOptions(self,Options): |
176 |
""" |
177 |
adds a options to the benchmark |
178 |
|
179 |
@param options: adds a new option to the bechmark |
180 |
@type problem: L{Options} |
181 |
""" |
182 |
self.__options.append(Options) |
183 |
|
184 |
def run(self,scale=1): |
185 |
""" |
186 |
runs all problems with all options. |
187 |
|
188 |
|
189 |
@param scale: defines the number of (OpenMP) threads to be used. If scale is a scalar all benchmarks |
190 |
are run with scale number of threads. If scale is a C{list}, the p-th problem in each of the benchmarks |
191 |
in the suite is run with scale[p] threads. If scale[p]<1 teh p-th problem is omitted. |
192 |
@type scale: C{int} or C{list} of C{int}s. |
193 |
""" |
194 |
if isinstance(scale,list): |
195 |
c_max=min(len(scale),len(self.__problems)) |
196 |
else: |
197 |
c_max=len(self.__problems) |
198 |
self.__filter=filter |
199 |
self.__scale=scale |
200 |
self.__results=[] |
201 |
for c in range(c_max): |
202 |
r=self.__problems[c] |
203 |
if isinstance(scale,list): |
204 |
s=scale[c] |
205 |
else: |
206 |
s=scale |
207 |
row=[] |
208 |
if s>0: |
209 |
for p in self.__options: |
210 |
os.environ['OMP_NUM_THREADS']=str(s) |
211 |
row.append(r.run(p)) |
212 |
self.__results.append(row) |
213 |
def getHTML(self,filter,level=1): |
214 |
""" |
215 |
returns the results of the last benchmark run in HTML format. |
216 |
|
217 |
@param filter: filter to be applied to the results |
218 |
@type filter: L{BenchmarkFilter} |
219 |
@param level: level used in header <H?> tags |
220 |
@type level: C{int} |
221 |
@return: HTML document |
222 |
@rtype: C{str} |
223 |
""" |
224 |
out="" |
225 |
if level==1: out+="<HTML><HEAD><TITLE>Benchmark: %s</TITLE></HEAD><BODY>\n"%str(self) |
226 |
out+="<H%s>%s</H%s>\n"%(level,str(self),level) |
227 |
if level==1: |
228 |
m="" |
229 |
if isinstance(self.__scale,int): |
230 |
if self.__scale>1: |
231 |
m=" (%s threads)"%self.__scale |
232 |
out+="<p>platform: %s%s</p>\n"%(socket.gethostname(),m) |
233 |
if self.__description: out+="<p>%s</p>\n"%str(self.__description) |
234 |
if len(self.__problems)>0: |
235 |
out+="<TABLE ALIGN=\"center\" BORDER=3 CELLPADDING=5 CELLSPACING=1>\n" |
236 |
h1_seg="" |
237 |
rn=filter.getResultNames() |
238 |
if len(rn)==0: |
239 |
h1_seg+="<TD></TD>" |
240 |
else: |
241 |
for n in rn: h1_seg+="<TD ALIGN=\"center\">%s</TD>"%n |
242 |
h0="<TR><TH ALIGN=\"center\" ROWSPAN=2>Case</TH>" |
243 |
h1="<TR>" |
244 |
if isinstance(self.__scale,list): h0+="<TH ALIGN=\"center\" ROWSPAN=2>Threads</TH>" |
245 |
for o in self.__options: |
246 |
if len(rn)==0: |
247 |
h0+="<TH ALIGN=\"center\">%s</TH>"%str(o) |
248 |
elif len(rn)==1: |
249 |
h0+="<TH ALIGN=\"center\">%s</TH>"%str(o) |
250 |
empty_h1=False |
251 |
else: |
252 |
h0+="<TH ALIGN=\"center\" COLSPAN=%s>%s</TH>"%(len(rn),str(o)) |
253 |
h1+=h1_seg |
254 |
out+=h0+"</TR>\n"+h1+"</TR>\n" |
255 |
c=0 |
256 |
for r in range(len(self.__results)): |
257 |
out+="<TR><TH ALIGN=\"right\">%s</TH>"%str(self.__problems[r]) |
258 |
if isinstance(self.__scale,list): out+="<TD ALIGN=\"right\">%s</TD>"%self.__scale[c] |
259 |
for col in self.__results[r]: |
260 |
for e in filter(col): out+="<TD ALIGN=\"right\">%s</TD>"%e |
261 |
out+="</TR>\n" |
262 |
c+=1 |
263 |
out+="</TABLE>" |
264 |
if level==1: |
265 |
out+="<hr><p align=\"center\">by %s at %s</p>\n"%(os.getlogin(),time.strftime('%X %x %Z')) |
266 |
out+="</BODY></HTML>\n" |
267 |
return out |
268 |
|
269 |
class BenchmarkProblem(object): |
270 |
""" |
271 |
something that can be run and returns a list of characteristics such as timing, Mflops, error, etc. |
272 |
""" |
273 |
def __init__(self,name=None): |
274 |
""" |
275 |
sets up a benchmark problem |
276 |
|
277 |
@param name: name of the problem. If no name is given the class name is used. |
278 |
@type name: C{str} |
279 |
""" |
280 |
super(BenchmarkProblem,self).__init__() |
281 |
if name==None: |
282 |
self.__name=self.__class__.__name__ |
283 |
else: |
284 |
self.__name=name |
285 |
|
286 |
|
287 |
def __str__(self): |
288 |
""" |
289 |
returns the name of the benchmark suite |
290 |
|
291 |
@return: name |
292 |
@rtype: C{str} |
293 |
""" |
294 |
return self.__name |
295 |
|
296 |
def run(self,options=None): |
297 |
""" |
298 |
runs the problem and returns a list of run characteristics |
299 |
|
300 |
|
301 |
@param options: the options that are used for the run. Note that the number of OpenMP threads is controlled |
302 |
by the L{Benchmark} the problem is run in. |
303 |
@type options: L{Options} |
304 |
@return: run characteristics |
305 |
@rtype: any type that can be read by the L{BenchmarkFilter} applied to it. |
306 |
@remark: this function has to overwritten by a particular problem |
307 |
""" |
308 |
raise NotImplementedError |
309 |
return [] |
310 |
|
311 |
class BenchmarkFilter(object): |
312 |
""" |
313 |
object to filter the characteristcs returned by Bechmark runs. |
314 |
|
315 |
""" |
316 |
def __init__(self): |
317 |
""" |
318 |
sets up a filter |
319 |
""" |
320 |
pass |
321 |
|
322 |
|
323 |
def getResultNames(self): |
324 |
""" |
325 |
return the names of the results produced when run() is called. |
326 |
|
327 |
@return: names the list of the names to be used when the results of the run() call are printed |
328 |
@rtype: C{list} of C{str} |
329 |
@remark: this function has to overwritten by a particular problem |
330 |
""" |
331 |
raise NotImplementedError |
332 |
return [] |
333 |
|
334 |
def __call__(self,result): |
335 |
""" |
336 |
filters out values results returned as characteristcs of a problem run |
337 |
|
338 |
@param result: values to be filtered |
339 |
@type result: any type that is produced by the L{BenchmarkProblem} it is applied to |
340 |
@return: a list of strings selected from result |
341 |
@rtype: C{list} of C{str} |
342 |
@remark: this function has to overwritten by a particular problem |
343 |
""" |
344 |
raise NotImplementedError |
345 |
return [] |
346 |
|
347 |
|
348 |
class Options(object): |
349 |
""" |
350 |
defines a set of options to be used to run a L{BenchmarkProblem} |
351 |
""" |
352 |
def __init__(self,name=None): |
353 |
""" |
354 |
sets up the options |
355 |
|
356 |
@param name: name of the option. If no name is given the class name is used. |
357 |
@type name: C{str} |
358 |
""" |
359 |
super(Options,self).__init__() |
360 |
if name==None: |
361 |
self.__name=self.__class__.__name__ |
362 |
else: |
363 |
self.__name=name |
364 |
def __str__(self): |
365 |
""" |
366 |
returns the name of the benchmark suite |
367 |
|
368 |
@return: name |
369 |
@rtype: C{str} |
370 |
""" |
371 |
return self.__name |
372 |
|
373 |
if __name__=="__main__": |
374 |
|
375 |
class OptionsTest1(Options): |
376 |
pass |
377 |
class OptionsTest2(Options): |
378 |
pass |
379 |
|
380 |
class BenchmarkProblemTest1(BenchmarkProblem): |
381 |
def __init__(self): |
382 |
super(BenchmarkProblemTest1,self).__init__(name="TEST1") |
383 |
def run(self,options=None): |
384 |
import time |
385 |
return time.time(),"A" |
386 |
|
387 |
class BenchmarkProblemTest2(BenchmarkProblem): |
388 |
def __init__(self): |
389 |
super(BenchmarkProblemTest2,self).__init__(name="TEST2") |
390 |
def run(self,options=None): |
391 |
import time |
392 |
return -time.time(),"B" |
393 |
|
394 |
class SimpleFilter(BenchmarkFilter): |
395 |
def getResultNames(self): |
396 |
return ["r0","r1"] |
397 |
def __call__(self,result): |
398 |
return [str(result[0]),str(result[1])] |
399 |
|
400 |
bm=Benchmark("Example") |
401 |
bm.addProblem(BenchmarkProblemTest1()) |
402 |
bm.addProblem(BenchmarkProblemTest2()) |
403 |
bm.addOptions(OptionsTest1()) |
404 |
bm.addOptions(OptionsTest2()) |
405 |
|
406 |
bms=BenchmarkSuite("A Test") |
407 |
bms.addBenchmark(bm) |
408 |
|
409 |
bms.run() |
410 |
print bms.getHTML(filter=SimpleFilter()) |
411 |
|
412 |
bms.run(scale=4) |
413 |
print bms.getHTML(filter=SimpleFilter()) |
414 |
|
415 |
bms.run(scale=[1,2]) |
416 |
print bms.getHTML(filter=SimpleFilter()) |