/[escript]/trunk/escript/py_src/benchmark.py
ViewVC logotype

Diff of /trunk/escript/py_src/benchmark.py

Parent Directory Parent Directory | Revision Log Revision Log | View Patch Patch

revision 2157 by ksteube, Thu Sep 25 06:43:44 2008 UTC revision 2158 by caltinay, Mon Dec 15 07:17:47 2008 UTC
# Line 22  __url__="http://www.uq.edu.au/esscc/escr Line 22  __url__="http://www.uq.edu.au/esscc/escr
22  filter# $Id:$  filter# $Id:$
23    
24  """  """
25  A simple framework to run benchmarks under OPENMP and to summarize the results in tables for instance in HTML  A simple framework to run benchmarks under OpenMP and to summarize the results
26    in tables for instance in HTML
27    
28  @var __author__: name of author  @var __author__: name of author
29  @var __license__: licence agreement  @var __license__: licence agreement
# Line 39  from esys.escript import setNumberOfThre Line 40  from esys.escript import setNumberOfThre
40    
41  class BenchmarkSuite(object):  class BenchmarkSuite(object):
42     """     """
43     framework to run a bunch of L{Benchmark}s with the object to create a table of statistics.     framework to run a bunch of L{Benchmark}s using the object and creating a
44       table of statistics.
45    
46     @cvar MAX_LEVEL: maximum number of level in headers for output     @cvar MAX_LEVEL: maximum number of level in headers for output
47     """     """
# Line 47  class BenchmarkSuite(object): Line 49  class BenchmarkSuite(object):
49     def __init__(self,name=None):     def __init__(self,name=None):
50         """         """
51         sets up a suite of benchmarks         sets up a suite of benchmarks
52    
53         @param name: name of the benchmark suite. If no name is given the class name is used.         @param name: name of the benchmark suite. If no name is given the class
54                        name is used.
55         @type name: C{str}         @type name: C{str}
56         """         """
57         super(BenchmarkSuite,self).__init__()         super(BenchmarkSuite,self).__init__()
# Line 58  class BenchmarkSuite(object): Line 61  class BenchmarkSuite(object):
61             self.__name=self.__class__.__name__             self.__name=self.__class__.__name__
62         else:         else:
63            self.__name=name            self.__name=name
64              
65     def __str__(self):     def __str__(self):
66         """         """
67         returns the name of the benchmark suite         returns the name of the benchmark suite
68          
69         @return:  name         @return:  name
70         @rtype: C{str}         @rtype: C{str}
71         """         """
72         return self.__name         return self.__name
73    
74     def addBenchmark(self,benchmark):     def addBenchmark(self,benchmark):
75         """         """
76         adds a L{Benchmark} to the suite         adds a L{Benchmark} to the suite
# Line 74  class BenchmarkSuite(object): Line 78  class BenchmarkSuite(object):
78         @param benchmark: adds a new L{Benchmark} to the suite         @param benchmark: adds a new L{Benchmark} to the suite
79         @type benchmark: L{Benchmark}         @type benchmark: L{Benchmark}
80         """         """
81         self.__benchmarks.append(benchmark)                 self.__benchmarks.append(benchmark)
82    
83     def __len__(self):     def __len__(self):
84         """         """
85         returns the number of benchmarks in the suite         returns the number of benchmarks in the suite
86    
87         @return:  number of benchmarks         @return:  number of benchmarks
88         @rtype: C{int}               @rtype: C{int}
89         """         """
90         return len(self.__benchmarks)         return len(self.__benchmarks)
91    
92     def __getitem__(self,i):     def __getitem__(self,i):
93         """         """
94         returns the i-th benchmark in the suite through self[i]         returns the i-th benchmark in the suite through self[i]
95          
96         @param i: index of the requested benchmark         @param i: index of the requested benchmark
97         @type i: C{int}         @type i: C{int}
98         @return:  i-th benchmark         @return:  i-th benchmark
99         @rtype: L{Benchmark}               @rtype: L{Benchmark}
100    
101         """         """
102         return self.__benchmarks[i]         return self.__benchmarks[i]
103    
104     def run(self,scale=1):     def run(self,scale=1):
105         """         """
106         runs all benchmarks         runs all benchmarks
107    
108         @param scale: defines the number of (OpenMP) threads to be used. If scale is a scalar all benchmarks         @param scale: defines the number of (OpenMP) threads to be used. If
109                       are run with scale number of threads. If scale is a C{list}, the p-th problem in each of the benchmarks                       scale is a scalar all benchmarks are run with scale number
110                       in the suite is run with scale[p] threads. If scale[p]<1 teh p-th problem is omitted.                       of threads. If scale is a C{list}, the p-th problem in
111         @type scale: C{int} or C{list} of C{int}s.                       each of the benchmarks in the suite is run with scale[p]
112                         threads. If scale[p]<1 the p-th problem is omitted.
113           @type scale: C{int} or C{list} of C{int}s.
114         """         """
115         self.__scale=scale               self.__scale=scale
116         for i in range(len(self)): self[i].run(scale=scale)         for i in range(len(self)): self[i].run(scale=scale)
117    
118     def getHTML(self,filter,level=1):     def getHTML(self,filter,level=1):
119         """         """
120         returns the results of the last benchmark run in HTML format.         returns the results of the last benchmark run in HTML format.
121    
122         @param filter: filter to be applied to the results         @param filter: filter to be applied to the results
123         @type filter: L{BenchmarkFilter}         @type filter: L{BenchmarkFilter}
124         @param level: level used in header <H?> tags         @param level: level used in header <H?> tags
125         @type level: C{int}         @type level: C{int}
126         @return: HTML document         @return: HTML document
127         @rtype: C{str}         @rtype: C{str}
# Line 119  class BenchmarkSuite(object): Line 129  class BenchmarkSuite(object):
129         out=""         out=""
130         if level==1: out+="<HTML><HEAD><TITLE>Benchmark: %s</TITLE></HEAD><BODY>\n"%str(self)         if level==1: out+="<HTML><HEAD><TITLE>Benchmark: %s</TITLE></HEAD><BODY>\n"%str(self)
131         out+="<H%s>%s</H%s>\n"%(level,str(self),level)         out+="<H%s>%s</H%s>\n"%(level,str(self),level)
132         if level==1:         if level==1:
133             m=""             m=""
134             if isinstance(self.__scale,int):             if isinstance(self.__scale,int):
135                if self.__scale>1:                if self.__scale>1:
# Line 130  class BenchmarkSuite(object): Line 140  class BenchmarkSuite(object):
140             out+=self[i].getHTML(filter=filter,level=min(level+1,self.MAX_LEVEL))             out+=self[i].getHTML(filter=filter,level=min(level+1,self.MAX_LEVEL))
141             out+="<p>\n"             out+="<p>\n"
142         if level==1:         if level==1:
143             try:             try:
144                 name=os.getlogin()                 name=os.getlogin()
145                 out+="<hr><p align=\"center\">by %s at %s</p>\n"%(name,time.strftime('%X %x %Z'))                 out+="<hr><p align=\"center\">by %s at %s</p>\n"%(name,time.strftime('%X %x %Z'))
146             except OSError:             except OSError:
147                 out+="<hr><p align=\"center\">%s</p>\n"%(time.strftime('%X %x %Z'))                 out+="<hr><p align=\"center\">%s</p>\n"%(time.strftime('%X %x %Z'))
148                  
149             out+="</BODY></HTML>\n"             out+="</BODY></HTML>\n"
150         return out         return out
151    
# Line 146  class Benchmark(object): Line 156  class Benchmark(object):
156     """     """
157     def __init__(self,name=None,description=None):     def __init__(self,name=None,description=None):
158         """         """
159         sets up a benchmark         sets up a benchmark
160    
161         @param name: name of the benchmark. If no name is given the class name is used.         @param name: name of the benchmark. If no name is given the class name
162                        is used.
163         @type name: C{str}         @type name: C{str}
164         @param description: description of the benchmark.         @param description: description of the benchmark.
165         @type description: C{str} or C{None}               @type description: C{str} or C{None}
166         """         """
167         super(Benchmark,self).__init__()         super(Benchmark,self).__init__()
168         self.__options=[]         self.__options=[]
# Line 163  class Benchmark(object): Line 174  class Benchmark(object):
174         else:         else:
175            self.__name=name            self.__name=name
176         self.__description=description         self.__description=description
177          
178     def __str__(self):     def __str__(self):
179         """         """
180         returns the name of the benchmark suite         returns the name of the benchmark suite
181          
182         @return:  name         @return:  name
183         @rtype: C{str}         @rtype: C{str}
184         """         """
185         return self.__name         return self.__name
186              
187     def addProblem(self,problem):     def addProblem(self,problem):
188         """         """
189         adds a problem to the benchmark         adds a problem to the benchmark
190    
191         @param problem: adds a new problem to the bechmark         @param problem: adds a new problem to the benchmark
192         @type problem: L{BenchmarkProblem}         @type problem: L{BenchmarkProblem}
193         """         """
194         self.__problems.append(problem)         self.__problems.append(problem)
195    
196     def addOptions(self,options):     def addOptions(self,options):
197         """         """
198         adds a options to the benchmark         adds options to the benchmark
199    
200         @param options: adds a new option to the bechmark. If options==None they are are ignored         @param options: the options to be added to the bechmark. If
201                           options==None they are ignored
202         @type options: L{Options}         @type options: L{Options}
203         """         """
204         if options!=None: self.__options.append(options)         if options!=None: self.__options.append(options)
205    
206     def run(self,scale=1):     def run(self,scale=1):
207         """         """
208         runs all problems with all options.         runs all problems with all options.
209    
210           @param scale: defines the number of (OpenMP) threads to be used. If
211         @param scale: defines the number of (OpenMP) threads to be used. If scale is a scalar all benchmarks                       scale is a scalar all benchmarks are run with scale number
212                       are run with scale number of threads. If scale is a C{list}, the p-th problem in each of the benchmarks                       of threads. If scale is a C{list}, the p-th problem in
213                       in the suite is run with scale[p] threads. If scale[p]<1 teh p-th problem is omitted.                       each of the benchmarks in the suite is run with scale[p]
214         @type scale: C{int} or C{list} of C{int}s.                       threads. If scale[p]<1 the p-th problem is omitted.
215           @type scale: C{int} or C{list} of C{int}s.
216         """         """
217         if isinstance(scale,list):         if isinstance(scale,list):
218             c_max=min(len(scale),len(self.__problems))             c_max=min(len(scale),len(self.__problems))
# Line 226  class Benchmark(object): Line 239  class Benchmark(object):
239                       traceback.print_exc(file=sys.stdout)                       traceback.print_exc(file=sys.stdout)
240                       row.append(None)                       row.append(None)
241                t0=time.time()-t0                t0=time.time()-t0
242                print "%s with %s threads finished (walltime =%s sec)."%(r.__class__,s,t0)                print "%s with %s threads finished (walltime=%s sec)."%(r.__class__,s,t0)
243            self.__results.append(row)            self.__results.append(row)
244    
245     def getHTML(self,filter,level=1):     def getHTML(self,filter,level=1):
246         """         """
247         returns the results of the last benchmark run in HTML format.         returns the results of the last benchmark run in HTML format.
248    
249         @param filter: filter to be applied to the results         @param filter: filter to be applied to the results
250         @type filter: L{BenchmarkFilter}         @type filter: L{BenchmarkFilter}
251         @param level: level used in header <H?> tags         @param level: level used in header <H?> tags
252         @type level: C{int}         @type level: C{int}
253         @return: HTML document         @return: HTML document
254         @rtype: C{str}         @rtype: C{str}
# Line 242  class Benchmark(object): Line 256  class Benchmark(object):
256         out=""         out=""
257         if level==1: out+="<HTML><HEAD><TITLE>Benchmark: %s</TITLE></HEAD><BODY>\n"%str(self)         if level==1: out+="<HTML><HEAD><TITLE>Benchmark: %s</TITLE></HEAD><BODY>\n"%str(self)
258         out+="<H%s>%s</H%s>\n"%(level,str(self),level)         out+="<H%s>%s</H%s>\n"%(level,str(self),level)
259         if level==1:         if level==1:
260           m=""           m=""
261           if isinstance(self.__scale,int):           if isinstance(self.__scale,int):
262              if self.__scale>1:              if self.__scale>1:
263                  m=" (%s threads)"%self.__scale                  m=" (%s threads)"%self.__scale
264           out+="<p>platform: %s%s</p>\n"%(socket.gethostname(),m)           out+="<p>platform: %s%s</p>\n"%(socket.gethostname(),m)
265         if self.__description: out+="<p>%s</p>\n"%str(self.__description)           if self.__description: out+="<p>%s</p>\n"%str(self.__description)
266         if len(self.__problems)>0:         if len(self.__problems)>0:
267            out+="<TABLE ALIGN=\"center\" BORDER=3 CELLPADDING=5 CELLSPACING=1>\n"            out+="<TABLE ALIGN=\"center\" BORDER=3 CELLPADDING=5 CELLSPACING=1>\n"
268            h1_seg=""            h1_seg=""
# Line 256  class Benchmark(object): Line 270  class Benchmark(object):
270            if len(rn)==0:            if len(rn)==0:
271               h1_seg+="<TD></TD>"               h1_seg+="<TD></TD>"
272            else:            else:
273               for n in rn: h1_seg+="<TD ALIGN=\"center\">%s</TD>"%n                       for n in rn: h1_seg+="<TD ALIGN=\"center\">%s</TD>"%n
274            h0="<TR><TH ALIGN=\"center\" ROWSPAN=2>Case</TH>"            h0="<TR><TH ALIGN=\"center\" ROWSPAN=2>Case</TH>"
275            h1="<TR>"            h1="<TR>"
276            if isinstance(self.__scale,list): h0+="<TH ALIGN=\"center\" ROWSPAN=2>Threads</TH>"            if isinstance(self.__scale,list): h0+="<TH ALIGN=\"center\" ROWSPAN=2>Threads</TH>"
277            for o in self.__options:            for o in self.__options:
278                   if len(rn)==0:                   if len(rn)==0:
279                       h0+="<TH ALIGN=\"center\">%s</TH>"%str(o)                       h0+="<TH ALIGN=\"center\">%s</TH>"%str(o)
# Line 276  class Benchmark(object): Line 290  class Benchmark(object):
290            c=0            c=0
291            for r in range(len(self.__results)):            for r in range(len(self.__results)):
292               out+="<TR><TH ALIGN=\"right\">%s</TH>"%str(self.__problems[r])               out+="<TR><TH ALIGN=\"right\">%s</TH>"%str(self.__problems[r])
293               if isinstance(self.__scale,list):               if isinstance(self.__scale,list):
294                   out+="<TD ALIGN=\"right\">%s</TD>"%self.__scale[c]                   out+="<TD ALIGN=\"right\">%s</TD>"%self.__scale[c]
295               for col in self.__results[r]:               for col in self.__results[r]:
296                     if col==None:                     if col==None:
297                        out+="<TD ALIGN=\"center\" COLSPAN=%s>failed.</TD>"%colspan                        out+="<TD ALIGN=\"center\" COLSPAN=%s>failed.</TD>"%colspan
# Line 289  class Benchmark(object): Line 303  class Benchmark(object):
303         if level==1:         if level==1:
304            out+="<hr><p align=\"center\">by %s at %s</p>\n"%(os.getlogin(),time.strftime('%X %x %Z'))            out+="<hr><p align=\"center\">by %s at %s</p>\n"%(os.getlogin(),time.strftime('%X %x %Z'))
305            out+="</BODY></HTML>\n"            out+="</BODY></HTML>\n"
306         return out         return out
307          
308  class BenchmarkProblem(object):  class BenchmarkProblem(object):
309     """     """
310     something that can be run and returns a list of characteristics such as timing, Mflops, error, etc.     a benchmark problem that can be run and which returns a list of
311       characteristics such as timing, MFlops, error, etc.
312     """     """
313     def __init__(self,name=None):     def __init__(self,name=None):
314         """         """
315         sets up a benchmark problem         sets up a benchmark problem
316    
317         @param name: name of the problem. If no name is given the class name is used.         @param name: name of the problem. If no name is given the class name
318                        is used.
319         @type name: C{str}         @type name: C{str}
320         """         """
321         super(BenchmarkProblem,self).__init__()         super(BenchmarkProblem,self).__init__()
# Line 308  class BenchmarkProblem(object): Line 324  class BenchmarkProblem(object):
324         else:         else:
325            self.__name=name            self.__name=name
326    
         
327     def __str__(self):     def __str__(self):
328         """         """
329         returns the name of the benchmark suite         returns the name of the benchmark suite
330          
331         @return:  name         @return:  name
332         @rtype: C{str}         @rtype: C{str}
333         """         """
# Line 322  class BenchmarkProblem(object): Line 337  class BenchmarkProblem(object):
337         """         """
338         runs the problem and returns a list of run characteristics         runs the problem and returns a list of run characteristics
339    
340           @param options: the options that are used for the run. Note that the
341         @param options: the options that are used for the run. Note that the number of OpenMP threads is controlled                         number of OpenMP threads is controlled by the
342                         by the L{Benchmark} the problem is run in.                         L{Benchmark} the problem is run in.
343         @type options: L{Options}         @type options: L{Options}
344         @return: run characteristics         @return: run characteristics
345         @rtype: any type that can be read by the L{BenchmarkFilter} applied to it.         @rtype: any type that can be read by the L{BenchmarkFilter} applied
346         @note: this function has to overwritten by a particular problem                 to it.
347           @note: this function has to be overwritten by a particular problem
348         """         """
349         raise NotImplementedError         raise NotImplementedError
350         return []         return []
351        
352  class BenchmarkFilter(object):  class BenchmarkFilter(object):
353     """     """
354     object to filter the characteristcs returned by Bechmark runs.     object to filter the characteristics returned by Benchmark runs.
355      
356     """     """
357     def __init__(self):     def __init__(self):
358         """         """
359         sets up a filter         sets up a filter
360         """         """
361         pass         pass
   
362    
363     def getResultNames(self):     def getResultNames(self):
364         """         """
365         return the names of the results produced when run() is called.         returns the names of the results produced when run() is called.
366          
367         @return: names the list of the names to be used when the results of the run() call are printed         @return: the list of the names to be used when the results of
368                    the run() call are printed
369         @rtype: C{list} of C{str}         @rtype: C{list} of C{str}
370         @note: this function has to overwritten by a particular problem         @note: this function has to overwritten by a particular problem
371         """         """
# Line 358  class BenchmarkFilter(object): Line 374  class BenchmarkFilter(object):
374    
375     def __call__(self,result):     def __call__(self,result):
376         """         """
377         filters out values results returned as characteristcs of a problem run         filters out results returned as characteristics of a problem run
378          
379         @param result: values to be filtered         @param result: values to be filtered
380         @type result: any type that is produced by the L{BenchmarkProblem} it is applied to         @type result: any type that is produced by the L{BenchmarkProblem}
381                         it is applied to
382         @return: a list of strings selected from result         @return: a list of strings selected from result
383         @rtype: C{list} of C{str}         @rtype: C{list} of C{str}
384         @note: this function has to overwritten by a particular problem         @note: this function has to be overwritten by a particular problem
385         """         """
386         raise NotImplementedError         raise NotImplementedError
387         return []         return []
# Line 372  class BenchmarkFilter(object): Line 389  class BenchmarkFilter(object):
389    
390  class Options(object):  class Options(object):
391      """      """
392      defines a set of options to be used to run a L{BenchmarkProblem}      defines a set of options to be used to run a L{BenchmarkProblem}
393      """      """
394      def __init__(self,name=None):      def __init__(self,name=None):
395         """         """
396         sets up the options         sets up the options
397    
398         @param name: name of the option. If no name is given the class name is used.         @param name: name of the option. If no name is given the class name
399                        is used.
400         @type name: C{str}         @type name: C{str}
401         """         """
402         super(Options,self).__init__()         super(Options,self).__init__()
# Line 386  class Options(object): Line 404  class Options(object):
404            self.__name=self.__class__.__name__            self.__name=self.__class__.__name__
405         else:         else:
406            self.__name=name            self.__name=name
407    
408      def __str__(self):      def __str__(self):
409         """         """
410         returns the name of the benchmark suite         returns the name of the benchmark suite
411          
412         @return:  name         @return:  name
413         @rtype: C{str}         @rtype: C{str}
414         """         """
415         return self.__name         return self.__name
416        
417  if __name__=="__main__":  if __name__=="__main__":
418    
419      class OptionsTest1(Options):      class OptionsTest1(Options):
# Line 418  if __name__=="__main__": Line 437  if __name__=="__main__":
437    
438      class SimpleFilter(BenchmarkFilter):      class SimpleFilter(BenchmarkFilter):
439         def getResultNames(self):         def getResultNames(self):
440              return ["r0","r1"]                return ["r0","r1"]
441         def __call__(self,result):         def __call__(self,result):
442              return [str(result[0]),str(result[1])]              return [str(result[0]),str(result[1])]
443    
# Line 433  if __name__=="__main__": Line 452  if __name__=="__main__":
452    
453      bms.run()      bms.run()
454      print bms.getHTML(filter=SimpleFilter())      print bms.getHTML(filter=SimpleFilter())
455        
456      bms.run(scale=4)      bms.run(scale=4)
457      print bms.getHTML(filter=SimpleFilter())      print bms.getHTML(filter=SimpleFilter())
458    
459      bms.run(scale=[1,2])      bms.run(scale=[1,2])
460      print bms.getHTML(filter=SimpleFilter())      print bms.getHTML(filter=SimpleFilter())
461    

Legend:
Removed from v.2157  
changed lines
  Added in v.2158

  ViewVC Help
Powered by ViewVC 1.1.26