/[escript]/trunk/escript/py_src/benchmark.py
ViewVC logotype

Contents of /trunk/escript/py_src/benchmark.py

Parent Directory Parent Directory | Revision Log Revision Log


Revision 390 - (show annotations)
Tue Dec 20 03:39:24 2005 UTC (13 years, 9 months ago) by gross
File MIME type: text/x-python
File size: 13289 byte(s)
os.loginname does not work under pbs. the exception is caught now
1 filter# $Id:$
2
3 #
4 # COPYRIGHT ACcESS 2004 - All Rights Reserved
5 #
6 # This software is the property of ACcESS. No part of this code
7 # may be copied in any form or by any means without the expressed written
8 # consent of ACcESS. Copying, use or modification of this software
9 # by any unauthorised person is illegal unless that
10 # person has a software license agreement with ACcESS.
11 #
12
13 """
14 A simple framework to run benchmarks under OPENMP and to summarize the results in tables for instance in HTML
15
16 @var __author__: name of author
17 @var __licence__: licence agreement
18 var __url__: url entry point on documentation
19 @var __version__: version
20 @var __date__: date of the version
21 """
22
23 __author__="Lutz Gross, l.gross@uq.edu.au"
24 __licence__="contact: esys@access.uq.edu.au"
25 __url__="http://www.iservo.edu.au/esys/escript"
26 __version__="$Revision:$"
27 __date__="$Date:$"
28
29 import os,socket,time,sys
30
31 class BenchmarkSuite(object):
32 """
33 framework to run a bunch of L{Benchmark}s with the object to create a table of statistics.
34 @var MAX_LEVEL: maximum number of level in headers for output
35 """
36 MAX_LEVEL=5
37 def __init__(self,name=None):
38 """
39 sets up a suite of benchmarks
40
41 @param name: name of the benchmark suite. If no name is given the class name is used.
42 @type name: C{str}
43 """
44 super(BenchmarkSuite,self).__init__()
45 self.__benchmarks=[]
46 self.__scale=1
47 if name==None:
48 self.__name=self.__class__.__name__
49 else:
50 self.__name=name
51
52 def __str__(self):
53 """
54 returns the name of the benchmark suite
55
56 @return: name
57 @rtype: C{str}
58 """
59 return self.__name
60 def addBenchmark(self,benchmark):
61 """
62 adds a L{Benchmark} to the suite
63
64 @param benchmark: adds a new L{Benchmark} to the suite
65 @type benchmark: L{Benchmark}
66 """
67 self.__benchmarks.append(benchmark)
68 def __len__(self):
69 """
70 returns the number of benchmarks in the suite
71
72 @return: number of benchmarks
73 @rtype: C{int}
74 """
75 return len(self.__benchmarks)
76 def __getitem__(self,i):
77 """
78 returns the i-th benchmark in the suite through self[i]
79
80 @param i: index of the requested benchmark
81 @type i: C{int}
82 @return: i-th benchmark
83 @rtype: L{Benchmark}
84
85 """
86 return self.__benchmarks[i]
87 def run(self,scale=1):
88 """
89 runs all benchmarks
90
91 @param scale: defines the number of (OpenMP) threads to be used. If scale is a scalar all benchmarks
92 are run with scale number of threads. If scale is a C{list}, the p-th problem in each of the benchmarks
93 in the suite is run with scale[p] threads. If scale[p]<1 teh p-th problem is omitted.
94 @type scale: C{int} or C{list} of C{int}s.
95 """
96 self.__scale=scale
97 for i in range(len(self)): self[i].run(scale=scale)
98 def getHTML(self,filter,level=1):
99 """
100 returns the results of the last benchmark run in HTML format.
101
102 @param filter: filter to be applied to the results
103 @type filter: L{BenchmarkFilter}
104 @param level: level used in header <H?> tags
105 @type level: C{int}
106 @return: HTML document
107 @rtype: C{str}
108 """
109 out=""
110 if level==1: out+="<HTML><HEAD><TITLE>Benchmark: %s</TITLE></HEAD><BODY>\n"%str(self)
111 out+="<H%s>%s</H%s>\n"%(level,str(self),level)
112 if level==1:
113 m=""
114 if isinstance(self.__scale,int):
115 if self.__scale>1:
116 m=" (%s threads)"%self.__scale
117 out+="<p>platform: %s%s</p>\n"%(socket.gethostname(),m)
118 for i in range(len(self)):
119 out+="<p>\n"
120 out+=self[i].getHTML(filter=filter,level=min(level+1,self.MAX_LEVEL))
121 out+="<p>\n"
122 if level==1:
123 try:
124 name=os.getlogin()
125 out+="<hr><p align=\"center\">by %s at %s</p>\n"%(name,time.strftime('%X %x %Z'))
126 except OSError:
127 out+="<hr><p align=\"center\">%s</p>\n"%(time.strftime('%X %x %Z'))
128
129 out+="</BODY></HTML>\n"
130 return out
131
132
133 class Benchmark(object):
134 """
135 runs a bunch of similar L{BenchmarkProblem}s with a bunch of L{Options}
136 """
137 def __init__(self,name=None,description=None):
138 """
139 sets up a benchmark
140
141 @param name: name of the benchmark. If no name is given the class name is used.
142 @type name: C{str}
143 @param description: description of the benchmark.
144 @type description: C{str} or C{None}
145 """
146 super(Benchmark,self).__init__()
147 self.__options=[]
148 self.__problems=[]
149 self.__results=[]
150 self.__scale=1
151 if name==None:
152 self.__name=self.__class__.__name__
153 else:
154 self.__name=name
155 self.__description=description
156
157 def __str__(self):
158 """
159 returns the name of the benchmark suite
160
161 @return: name
162 @rtype: C{str}
163 """
164 return self.__name
165
166 def addProblem(self,problem):
167 """
168 adds a problem to the benchmark
169
170 @param problem: adds a new problem to the bechmark
171 @type problem: L{BenchmarkProblem}
172 """
173 self.__problems.append(problem)
174
175 def addOptions(self,Options):
176 """
177 adds a options to the benchmark
178
179 @param options: adds a new option to the bechmark
180 @type problem: L{Options}
181 """
182 self.__options.append(Options)
183
184 def run(self,scale=1):
185 """
186 runs all problems with all options.
187
188
189 @param scale: defines the number of (OpenMP) threads to be used. If scale is a scalar all benchmarks
190 are run with scale number of threads. If scale is a C{list}, the p-th problem in each of the benchmarks
191 in the suite is run with scale[p] threads. If scale[p]<1 teh p-th problem is omitted.
192 @type scale: C{int} or C{list} of C{int}s.
193 """
194 if isinstance(scale,list):
195 c_max=min(len(scale),len(self.__problems))
196 else:
197 c_max=len(self.__problems)
198 self.__filter=filter
199 self.__scale=scale
200 self.__results=[]
201 for c in range(c_max):
202 r=self.__problems[c]
203 if isinstance(scale,list):
204 s=scale[c]
205 else:
206 s=scale
207 row=[]
208 if s>0:
209 for p in self.__options:
210 os.environ['OMP_NUM_THREADS']=str(s)
211 row.append(r.run(p))
212 self.__results.append(row)
213 def getHTML(self,filter,level=1):
214 """
215 returns the results of the last benchmark run in HTML format.
216
217 @param filter: filter to be applied to the results
218 @type filter: L{BenchmarkFilter}
219 @param level: level used in header <H?> tags
220 @type level: C{int}
221 @return: HTML document
222 @rtype: C{str}
223 """
224 out=""
225 if level==1: out+="<HTML><HEAD><TITLE>Benchmark: %s</TITLE></HEAD><BODY>\n"%str(self)
226 out+="<H%s>%s</H%s>\n"%(level,str(self),level)
227 if level==1:
228 m=""
229 if isinstance(self.__scale,int):
230 if self.__scale>1:
231 m=" (%s threads)"%self.__scale
232 out+="<p>platform: %s%s</p>\n"%(socket.gethostname(),m)
233 if self.__description: out+="<p>%s</p>\n"%str(self.__description)
234 if len(self.__problems)>0:
235 out+="<TABLE ALIGN=\"center\" BORDER=3 CELLPADDING=5 CELLSPACING=1>\n"
236 h1_seg=""
237 rn=filter.getResultNames()
238 if len(rn)==0:
239 h1_seg+="<TD></TD>"
240 else:
241 for n in rn: h1_seg+="<TD ALIGN=\"center\">%s</TD>"%n
242 h0="<TR><TH ALIGN=\"center\" ROWSPAN=2>Case</TH>"
243 h1="<TR>"
244 if isinstance(self.__scale,list): h0+="<TH ALIGN=\"center\" ROWSPAN=2>Threads</TH>"
245 for o in self.__options:
246 if len(rn)==0:
247 h0+="<TH ALIGN=\"center\">%s</TH>"%str(o)
248 elif len(rn)==1:
249 h0+="<TH ALIGN=\"center\">%s</TH>"%str(o)
250 empty_h1=False
251 else:
252 h0+="<TH ALIGN=\"center\" COLSPAN=%s>%s</TH>"%(len(rn),str(o))
253 h1+=h1_seg
254 out+=h0+"</TR>\n"+h1+"</TR>\n"
255 c=0
256 for r in range(len(self.__results)):
257 out+="<TR><TH ALIGN=\"right\">%s</TH>"%str(self.__problems[r])
258 if isinstance(self.__scale,list): out+="<TD ALIGN=\"right\">%s</TD>"%self.__scale[c]
259 for col in self.__results[r]:
260 for e in filter(col): out+="<TD ALIGN=\"right\">%s</TD>"%e
261 out+="</TR>\n"
262 c+=1
263 out+="</TABLE>"
264 if level==1:
265 out+="<hr><p align=\"center\">by %s at %s</p>\n"%(os.getlogin(),time.strftime('%X %x %Z'))
266 out+="</BODY></HTML>\n"
267 return out
268
269 class BenchmarkProblem(object):
270 """
271 something that can be run and returns a list of characteristics such as timing, Mflops, error, etc.
272 """
273 def __init__(self,name=None):
274 """
275 sets up a benchmark problem
276
277 @param name: name of the problem. If no name is given the class name is used.
278 @type name: C{str}
279 """
280 super(BenchmarkProblem,self).__init__()
281 if name==None:
282 self.__name=self.__class__.__name__
283 else:
284 self.__name=name
285
286
287 def __str__(self):
288 """
289 returns the name of the benchmark suite
290
291 @return: name
292 @rtype: C{str}
293 """
294 return self.__name
295
296 def run(self,options=None):
297 """
298 runs the problem and returns a list of run characteristics
299
300
301 @param options: the options that are used for the run. Note that the number of OpenMP threads is controlled
302 by the L{Benchmark} the problem is run in.
303 @type options: L{Options}
304 @return: run characteristics
305 @rtype: any type that can be read by the L{BenchmarkFilter} applied to it.
306 @remark: this function has to overwritten by a particular problem
307 """
308 raise NotImplementedError
309 return []
310
311 class BenchmarkFilter(object):
312 """
313 object to filter the characteristcs returned by Bechmark runs.
314
315 """
316 def __init__(self):
317 """
318 sets up a filter
319 """
320 pass
321
322
323 def getResultNames(self):
324 """
325 return the names of the results produced when run() is called.
326
327 @return: names the list of the names to be used when the results of the run() call are printed
328 @rtype: C{list} of C{str}
329 @remark: this function has to overwritten by a particular problem
330 """
331 raise NotImplementedError
332 return []
333
334 def __call__(self,result):
335 """
336 filters out values results returned as characteristcs of a problem run
337
338 @param result: values to be filtered
339 @type result: any type that is produced by the L{BenchmarkProblem} it is applied to
340 @return: a list of strings selected from result
341 @rtype: C{list} of C{str}
342 @remark: this function has to overwritten by a particular problem
343 """
344 raise NotImplementedError
345 return []
346
347
348 class Options(object):
349 """
350 defines a set of options to be used to run a L{BenchmarkProblem}
351 """
352 def __init__(self,name=None):
353 """
354 sets up the options
355
356 @param name: name of the option. If no name is given the class name is used.
357 @type name: C{str}
358 """
359 super(Options,self).__init__()
360 if name==None:
361 self.__name=self.__class__.__name__
362 else:
363 self.__name=name
364 def __str__(self):
365 """
366 returns the name of the benchmark suite
367
368 @return: name
369 @rtype: C{str}
370 """
371 return self.__name
372
373 if __name__=="__main__":
374
375 class OptionsTest1(Options):
376 pass
377 class OptionsTest2(Options):
378 pass
379
380 class BenchmarkProblemTest1(BenchmarkProblem):
381 def __init__(self):
382 super(BenchmarkProblemTest1,self).__init__(name="TEST1")
383 def run(self,options=None):
384 import time
385 return time.time(),"A"
386
387 class BenchmarkProblemTest2(BenchmarkProblem):
388 def __init__(self):
389 super(BenchmarkProblemTest2,self).__init__(name="TEST2")
390 def run(self,options=None):
391 import time
392 return -time.time(),"B"
393
394 class SimpleFilter(BenchmarkFilter):
395 def getResultNames(self):
396 return ["r0","r1"]
397 def __call__(self,result):
398 return [str(result[0]),str(result[1])]
399
400 bm=Benchmark("Example")
401 bm.addProblem(BenchmarkProblemTest1())
402 bm.addProblem(BenchmarkProblemTest2())
403 bm.addOptions(OptionsTest1())
404 bm.addOptions(OptionsTest2())
405
406 bms=BenchmarkSuite("A Test")
407 bms.addBenchmark(bm)
408
409 bms.run()
410 print bms.getHTML(filter=SimpleFilter())
411
412 bms.run(scale=4)
413 print bms.getHTML(filter=SimpleFilter())
414
415 bms.run(scale=[1,2])
416 print bms.getHTML(filter=SimpleFilter())

  ViewVC Help
Powered by ViewVC 1.1.26