/[escript]/trunk/escript/py_src/benchmark.py
ViewVC logotype

Annotation of /trunk/escript/py_src/benchmark.py

Parent Directory Parent Directory | Revision Log Revision Log


Revision 2881 - (hide annotations)
Thu Jan 28 02:03:15 2010 UTC (9 years, 11 months ago) by jfenwick
File MIME type: text/x-python
File size: 14509 byte(s)
Don't panic.
Updating copyright stamps

1 ksteube 1809
2     ########################################################
3     #
4 jfenwick 2881 # Copyright (c) 2003-2010 by University of Queensland
5 ksteube 1809 # Earth Systems Science Computational Center (ESSCC)
6     # http://www.uq.edu.au/esscc
7     #
8     # Primary Business: Queensland, Australia
9     # Licensed under the Open Software License version 3.0
10     # http://www.opensource.org/licenses/osl-3.0.php
11     #
12     ########################################################
13    
14 jfenwick 2881 __copyright__="""Copyright (c) 2003-2010 by University of Queensland
15 ksteube 1809 Earth Systems Science Computational Center (ESSCC)
16     http://www.uq.edu.au/esscc
17     Primary Business: Queensland, Australia"""
18     __license__="""Licensed under the Open Software License version 3.0
19     http://www.opensource.org/licenses/osl-3.0.php"""
20 jfenwick 2344 __url__="https://launchpad.net/escript-finley"
21 ksteube 1809
22 gross 379 filter# $Id:$
23 gross 364
24     """
25 caltinay 2158 A simple framework to run benchmarks under OpenMP and to summarize the results
26     in tables for instance in HTML
27 gross 364
28 jfenwick 2625 :var __author__: name of author
29     :var __license__: licence agreement
30     :var __copyright__: copyrights
31     :var __url__: url entry point on documentation
32     :var __version__: version
33     :var __date__: date of the version
34 gross 364 """
35    
36     __author__="Lutz Gross, l.gross@uq.edu.au"
37    
38 gross 462 import os,socket,time,sys,traceback
39 gross 393 from esys.escript import setNumberOfThreads
40 gross 364
41     class BenchmarkSuite(object):
42     """
43 jfenwick 2625 Framework to run a bunch of `Benchmark` s using the object and creating a
44 caltinay 2158 table of statistics.
45 gross 720
46 jfenwick 2625 :cvar MAX_LEVEL: maximum number of level in headers for output
47 gross 364 """
48     MAX_LEVEL=5
49     def __init__(self,name=None):
50     """
51 caltinay 2169 Sets up a suite of benchmarks.
52 caltinay 2158
53 jfenwick 2625 :param name: name of the benchmark suite. If no name is given the class
54 caltinay 2158 name is used.
55 jfenwick 2625 :type name: ``str``
56 gross 364 """
57     super(BenchmarkSuite,self).__init__()
58     self.__benchmarks=[]
59     self.__scale=1
60     if name==None:
61     self.__name=self.__class__.__name__
62     else:
63     self.__name=name
64 caltinay 2158
65 gross 364 def __str__(self):
66     """
67 caltinay 2169 Returns the name of the benchmark suite.
68 caltinay 2158
69 jfenwick 2625 :return: the name
70     :rtype: ``str``
71 gross 364 """
72     return self.__name
73 caltinay 2158
74 gross 364 def addBenchmark(self,benchmark):
75     """
76 jfenwick 2625 Adds a new `Benchmark` to the suite.
77 gross 364
78 jfenwick 2625 :param benchmark: the benchmark to add
79     :type benchmark: `Benchmark`
80 gross 364 """
81 caltinay 2158 self.__benchmarks.append(benchmark)
82    
83 gross 364 def __len__(self):
84     """
85 caltinay 2169 Returns the number of benchmarks in the suite.
86 gross 364
87 jfenwick 2625 :return: number of benchmarks
88     :rtype: ``int``
89 gross 364 """
90     return len(self.__benchmarks)
91 caltinay 2158
92 gross 364 def __getitem__(self,i):
93     """
94 caltinay 2169 Returns the i-th benchmark in the suite through self[i].
95 caltinay 2158
96 jfenwick 2625 :param i: index of the requested benchmark
97     :type i: ``int``
98     :return: i-th benchmark
99     :rtype: `Benchmark`
100 gross 364
101     """
102     return self.__benchmarks[i]
103 caltinay 2158
104 gross 364 def run(self,scale=1):
105     """
106 caltinay 2169 Runs all benchmarks.
107 gross 364
108 jfenwick 2625 :param scale: defines the number of (OpenMP) threads to be used. If
109     ``scale`` is a scalar all benchmarks are run with ``scale``
110     number of threads. If ``scale`` is a ``list``, the p-th
111 caltinay 2169 problem in each of the benchmarks in the suite is run with
112 jfenwick 2625 ``scale[p]`` threads. If ``scale[p]`` <1 the p-th problem is
113 caltinay 2169 omitted.
114 jfenwick 2625 :type scale: ``int`` or ``list`` of ``int``
115 gross 364 """
116 caltinay 2158 self.__scale=scale
117 gross 379 for i in range(len(self)): self[i].run(scale=scale)
118 caltinay 2158
119 gross 379 def getHTML(self,filter,level=1):
120 gross 364 """
121 caltinay 2169 Returns the results of the last benchmark run in HTML format.
122 gross 364
123 jfenwick 2625 :param filter: filter to be applied to the results
124     :type filter: `BenchmarkFilter`
125     :param level: level used in header <H?> tags
126     :type level: ``int``
127     :return: HTML document
128     :rtype: ``str``
129 gross 364 """
130     out=""
131     if level==1: out+="<HTML><HEAD><TITLE>Benchmark: %s</TITLE></HEAD><BODY>\n"%str(self)
132     out+="<H%s>%s</H%s>\n"%(level,str(self),level)
133 caltinay 2158 if level==1:
134 gross 364 m=""
135     if isinstance(self.__scale,int):
136     if self.__scale>1:
137     m=" (%s threads)"%self.__scale
138     out+="<p>platform: %s%s</p>\n"%(socket.gethostname(),m)
139     for i in range(len(self)):
140     out+="<p>\n"
141 gross 379 out+=self[i].getHTML(filter=filter,level=min(level+1,self.MAX_LEVEL))
142 gross 364 out+="<p>\n"
143     if level==1:
144 caltinay 2158 try:
145 gross 390 name=os.getlogin()
146     out+="<hr><p align=\"center\">by %s at %s</p>\n"%(name,time.strftime('%X %x %Z'))
147     except OSError:
148     out+="<hr><p align=\"center\">%s</p>\n"%(time.strftime('%X %x %Z'))
149 caltinay 2158
150 gross 364 out+="</BODY></HTML>\n"
151     return out
152    
153    
154     class Benchmark(object):
155     """
156 jfenwick 2625 Runs a bunch of similar `BenchmarkProblem` s with a bunch of `Options`.
157 gross 364 """
158     def __init__(self,name=None,description=None):
159     """
160 caltinay 2169 Sets up a benchmark.
161 caltinay 2158
162 jfenwick 2625 :param name: name of the benchmark. If no name is given the class name
163 caltinay 2158 is used.
164 jfenwick 2625 :type name: ``str``
165     :param description: description of the benchmark
166     :type description: ``str`` or ``None``
167 gross 364 """
168     super(Benchmark,self).__init__()
169     self.__options=[]
170     self.__problems=[]
171     self.__results=[]
172     self.__scale=1
173     if name==None:
174     self.__name=self.__class__.__name__
175     else:
176     self.__name=name
177     self.__description=description
178 caltinay 2158
179 gross 364 def __str__(self):
180     """
181 caltinay 2169 Returns the name of the benchmark suite.
182 caltinay 2158
183 jfenwick 2625 :return: the name
184     :rtype: ``str``
185 gross 364 """
186     return self.__name
187 caltinay 2158
188 gross 364 def addProblem(self,problem):
189     """
190 caltinay 2169 Adds a problem to the benchmark.
191 gross 364
192 jfenwick 2625 :param problem: the problem to be added
193     :type problem: `BenchmarkProblem`
194 gross 364 """
195     self.__problems.append(problem)
196    
197 gross 458 def addOptions(self,options):
198 gross 364 """
199 caltinay 2169 Adds options to the benchmark.
200 gross 364
201 jfenwick 2625 :param options: the options to be added to the benchmark. If
202 caltinay 2169 options==None the options are left unchanged.
203 jfenwick 2625 :type options: `Options`
204 gross 364 """
205 gross 458 if options!=None: self.__options.append(options)
206 gross 364
207     def run(self,scale=1):
208     """
209 caltinay 2169 Runs all problems with all options.
210 gross 364
211 jfenwick 2625 :param scale: defines the number of (OpenMP) threads to be used. If
212     ``scale`` is a scalar all benchmarks are run with ``scale``
213     number of threads. If ``scale`` is a ``list`` , the p-th
214 caltinay 2169 problem in each of the benchmarks in the suite is run with
215 jfenwick 2625 ``scale[p]`` threads. If ``scale[p]`` <1 the p-th problem is
216 caltinay 2169 omitted.
217 jfenwick 2625 :type scale: ``int`` or ``list`` of ``int`` s
218 gross 364 """
219     if isinstance(scale,list):
220 gross 386 c_max=min(len(scale),len(self.__problems))
221     else:
222     c_max=len(self.__problems)
223 gross 379 self.__filter=filter
224 gross 364 self.__scale=scale
225     self.__results=[]
226 gross 386 for c in range(c_max):
227     r=self.__problems[c]
228 gross 364 if isinstance(scale,list):
229     s=scale[c]
230     else:
231     s=scale
232     row=[]
233 gross 387 if s>0:
234 gross 435 t0=time.time()
235     print "%s with %s threads started."%(r.__class__,s)
236 gross 387 for p in self.__options:
237 gross 393 setNumberOfThreads(s)
238 gross 451 try:
239     row.append(r.run(p))
240     except:
241 gross 462 traceback.print_exc(file=sys.stdout)
242 gross 451 row.append(None)
243 gross 435 t0=time.time()-t0
244 caltinay 2158 print "%s with %s threads finished (walltime=%s sec)."%(r.__class__,s,t0)
245 gross 364 self.__results.append(row)
246 caltinay 2158
247 gross 379 def getHTML(self,filter,level=1):
248 gross 364 """
249 caltinay 2169 Returns the results of the last benchmark run in HTML format.
250 gross 364
251 jfenwick 2625 :param filter: filter to be applied to the results
252     :type filter: `BenchmarkFilter`
253     :param level: level used in header <H?> tags
254     :type level: ``int``
255     :return: HTML document
256     :rtype: ``str``
257 gross 364 """
258     out=""
259     if level==1: out+="<HTML><HEAD><TITLE>Benchmark: %s</TITLE></HEAD><BODY>\n"%str(self)
260     out+="<H%s>%s</H%s>\n"%(level,str(self),level)
261 caltinay 2158 if level==1:
262 gross 364 m=""
263     if isinstance(self.__scale,int):
264     if self.__scale>1:
265     m=" (%s threads)"%self.__scale
266     out+="<p>platform: %s%s</p>\n"%(socket.gethostname(),m)
267 caltinay 2158 if self.__description: out+="<p>%s</p>\n"%str(self.__description)
268 gross 364 if len(self.__problems)>0:
269     out+="<TABLE ALIGN=\"center\" BORDER=3 CELLPADDING=5 CELLSPACING=1>\n"
270     h1_seg=""
271 gross 379 rn=filter.getResultNames()
272 gross 364 if len(rn)==0:
273     h1_seg+="<TD></TD>"
274     else:
275 caltinay 2158 for n in rn: h1_seg+="<TD ALIGN=\"center\">%s</TD>"%n
276 gross 364 h0="<TR><TH ALIGN=\"center\" ROWSPAN=2>Case</TH>"
277     h1="<TR>"
278 caltinay 2158 if isinstance(self.__scale,list): h0+="<TH ALIGN=\"center\" ROWSPAN=2>Threads</TH>"
279 gross 364 for o in self.__options:
280     if len(rn)==0:
281     h0+="<TH ALIGN=\"center\">%s</TH>"%str(o)
282 gross 451 colspan=1
283 gross 364 elif len(rn)==1:
284     h0+="<TH ALIGN=\"center\">%s</TH>"%str(o)
285 gross 451 colspan=1
286 gross 364 empty_h1=False
287     else:
288 gross 451 colspan=len(rn)
289     h0+="<TH ALIGN=\"center\" COLSPAN=%s>%s</TH>"%(colspan,str(o))
290 gross 364 h1+=h1_seg
291     out+=h0+"</TR>\n"+h1+"</TR>\n"
292     c=0
293     for r in range(len(self.__results)):
294     out+="<TR><TH ALIGN=\"right\">%s</TH>"%str(self.__problems[r])
295 caltinay 2158 if isinstance(self.__scale,list):
296     out+="<TD ALIGN=\"right\">%s</TD>"%self.__scale[c]
297 gross 385 for col in self.__results[r]:
298 gross 451 if col==None:
299     out+="<TD ALIGN=\"center\" COLSPAN=%s>failed.</TD>"%colspan
300     else:
301     for e in filter(col): out+="<TD ALIGN=\"right\">%s</TD>"%e
302 gross 364 out+="</TR>\n"
303 gross 385 c+=1
304 gross 364 out+="</TABLE>"
305     if level==1:
306     out+="<hr><p align=\"center\">by %s at %s</p>\n"%(os.getlogin(),time.strftime('%X %x %Z'))
307     out+="</BODY></HTML>\n"
308 caltinay 2158 return out
309    
310 gross 364 class BenchmarkProblem(object):
311     """
312 caltinay 2169 Represents a benchmark problem that can be run and which returns a list of
313 caltinay 2158 characteristics such as timing, MFlops, error, etc.
314 gross 364 """
315     def __init__(self,name=None):
316     """
317 caltinay 2169 Sets up a benchmark problem.
318 caltinay 2158
319 jfenwick 2625 :param name: name of the problem. If no name is given the class name
320 caltinay 2158 is used.
321 jfenwick 2625 :type name: ``str``
322 gross 364 """
323     super(BenchmarkProblem,self).__init__()
324     if name==None:
325     self.__name=self.__class__.__name__
326     else:
327     self.__name=name
328    
329     def __str__(self):
330     """
331 caltinay 2169 Returns the name of the benchmark suite.
332 caltinay 2158
333 jfenwick 2625 :return: the name
334     :rtype: ``str``
335 gross 364 """
336     return self.__name
337    
338     def run(self,options=None):
339     """
340 caltinay 2169 Runs the problem and returns a list of run characteristics.
341 gross 364
342 jfenwick 2625 :param options: the options that are used for the run. Note that the
343 caltinay 2158 number of OpenMP threads is controlled by the
344 jfenwick 2625 `Benchmark` the problem is run in.
345     :type options: `Options`
346     :return: run characteristics
347     :rtype: any type that can be read by the `BenchmarkFilter` applied
348 caltinay 2169 to it
349 jfenwick 2625 :note: this function has to be overwritten by a particular problem
350 gross 364 """
351     raise NotImplementedError
352     return []
353 caltinay 2158
354 gross 379 class BenchmarkFilter(object):
355     """
356 caltinay 2169 Object to filter the characteristics returned by Benchmark runs.
357 caltinay 2158
358 gross 379 """
359     def __init__(self):
360     """
361 caltinay 2169 Sets up a filter.
362 gross 379 """
363     pass
364    
365     def getResultNames(self):
366     """
367 jfenwick 2625 Returns the names of the results produced when ``run()`` is called.
368 caltinay 2158
369 jfenwick 2625 :return: the list of the names to be used when the results of
370     the ``run()`` call are printed
371     :rtype: ``list`` of ``str``
372     :note: this function has to overwritten by a particular problem
373 gross 379 """
374     raise NotImplementedError
375     return []
376    
377     def __call__(self,result):
378     """
379 caltinay 2169 Filters out results returned as characteristics of a problem run.
380 caltinay 2158
381 jfenwick 2625 :param result: values to be filtered
382     :type result: any type that is produced by the `BenchmarkProblem`
383 caltinay 2158 it is applied to
384 jfenwick 2625 :return: a list of strings selected from result
385     :rtype: ``list`` of ``str``
386     :note: this function has to be overwritten by a particular problem
387 gross 379 """
388     raise NotImplementedError
389     return []
390    
391    
392 gross 364 class Options(object):
393     """
394 jfenwick 2625 Defines a set of options to be used to run a `BenchmarkProblem`.
395 gross 364 """
396     def __init__(self,name=None):
397     """
398 caltinay 2169 Sets up the options.
399 caltinay 2158
400 jfenwick 2625 :param name: name of the option. If no name is given the class name
401 caltinay 2158 is used.
402 jfenwick 2625 :type name: ``str``
403 gross 364 """
404     super(Options,self).__init__()
405     if name==None:
406 gross 385 self.__name=self.__class__.__name__
407 gross 364 else:
408     self.__name=name
409 caltinay 2158
410 gross 364 def __str__(self):
411     """
412 caltinay 2169 Returns the name of this options object.
413 caltinay 2158
414 jfenwick 2625 :return: the name
415     :rtype: ``str``
416 gross 364 """
417     return self.__name
418 caltinay 2158
419 gross 364 if __name__=="__main__":
420    
421     class OptionsTest1(Options):
422     pass
423     class OptionsTest2(Options):
424     pass
425    
426 gross 379 class BenchmarkProblemTest1(BenchmarkProblem):
427 gross 364 def __init__(self):
428     super(BenchmarkProblemTest1,self).__init__(name="TEST1")
429     def run(self,options=None):
430     import time
431     return time.time(),"A"
432    
433 gross 379 class BenchmarkProblemTest2(BenchmarkProblem):
434 gross 364 def __init__(self):
435     super(BenchmarkProblemTest2,self).__init__(name="TEST2")
436     def run(self,options=None):
437     import time
438     return -time.time(),"B"
439    
440 gross 379 class SimpleFilter(BenchmarkFilter):
441     def getResultNames(self):
442 caltinay 2158 return ["r0","r1"]
443 gross 379 def __call__(self,result):
444     return [str(result[0]),str(result[1])]
445    
446 gross 364 bm=Benchmark("Example")
447     bm.addProblem(BenchmarkProblemTest1())
448     bm.addProblem(BenchmarkProblemTest2())
449     bm.addOptions(OptionsTest1())
450     bm.addOptions(OptionsTest2())
451    
452     bms=BenchmarkSuite("A Test")
453     bms.addBenchmark(bm)
454    
455     bms.run()
456 gross 379 print bms.getHTML(filter=SimpleFilter())
457 caltinay 2158
458 gross 364 bms.run(scale=4)
459 gross 379 print bms.getHTML(filter=SimpleFilter())
460 gross 364
461     bms.run(scale=[1,2])
462 gross 379 print bms.getHTML(filter=SimpleFilter())
463 caltinay 2158

  ViewVC Help
Powered by ViewVC 1.1.26