/[escript]/trunk/escript/py_src/benchmark.py
ViewVC logotype

Diff of /trunk/escript/py_src/benchmark.py

Parent Directory Parent Directory | Revision Log Revision Log | View Patch Patch

revision 434 by gross, Thu Jan 19 01:53:16 2006 UTC revision 2549 by jfenwick, Mon Jul 20 06:43:47 2009 UTC
# Line 1  Line 1 
 filter# $Id:$  
1    
2    ########################################################
3  #  #
4  #      COPYRIGHT ACcESS 2004 -  All Rights Reserved  # Copyright (c) 2003-2009 by University of Queensland
5    # Earth Systems Science Computational Center (ESSCC)
6    # http://www.uq.edu.au/esscc
7  #  #
8  #   This software is the property of ACcESS.  No part of this code  # Primary Business: Queensland, Australia
9  #   may be copied in any form or by any means without the expressed written  # Licensed under the Open Software License version 3.0
10  #   consent of ACcESS.  Copying, use or modification of this software  # http://www.opensource.org/licenses/osl-3.0.php
 #   by any unauthorised person is illegal unless that  
 #   person has a software license agreement with ACcESS.  
11  #  #
12    ########################################################
13    
14    __copyright__="""Copyright (c) 2003-2009 by University of Queensland
15    Earth Systems Science Computational Center (ESSCC)
16    http://www.uq.edu.au/esscc
17    Primary Business: Queensland, Australia"""
18    __license__="""Licensed under the Open Software License version 3.0
19    http://www.opensource.org/licenses/osl-3.0.php"""
20    __url__="https://launchpad.net/escript-finley"
21    
22    filter# $Id:$
23    
24  """  """
25  A simple framework to run benchmarks under OPENMP and to summarize the results in tables for instance in HTML  A simple framework to run benchmarks under OpenMP and to summarize the results
26    in tables for instance in HTML
27    
28  @var __author__: name of author  @var __author__: name of author
29  @var __licence__: licence agreement  @var __license__: licence agreement
30  var __url__: url entry point on documentation  @var __copyright__: copyrights
31    @var __url__: url entry point on documentation
32  @var __version__: version  @var __version__: version
33  @var __date__: date of the version  @var __date__: date of the version
34  """  """
35    
36  __author__="Lutz Gross, l.gross@uq.edu.au"  __author__="Lutz Gross, l.gross@uq.edu.au"
 __licence__="contact: esys@access.uq.edu.au"  
 __url__="http://www.iservo.edu.au/esys/escript"  
 __version__="$Revision:$"  
 __date__="$Date:$"  
37    
38  import os,socket,time,sys  import os,socket,time,sys,traceback
39  from esys.escript import setNumberOfThreads  from esys.escript import setNumberOfThreads
40    
41  class BenchmarkSuite(object):  class BenchmarkSuite(object):
42     """     """
43     framework to run a bunch of L{Benchmark}s with the object to create a table of statistics.     Framework to run a bunch of L{Benchmark}s using the object and creating a
44     @var MAX_LEVEL: maximum number of level in headers for output     table of statistics.
45    
46       @cvar MAX_LEVEL: maximum number of level in headers for output
47     """     """
48     MAX_LEVEL=5     MAX_LEVEL=5
49     def __init__(self,name=None):     def __init__(self,name=None):
50         """         """
51         sets up a suite of benchmarks         Sets up a suite of benchmarks.
52    
53         @param name: name of the benchmark suite. If no name is given the class name is used.         @param name: name of the benchmark suite. If no name is given the class
54                        name is used.
55         @type name: C{str}         @type name: C{str}
56         """         """
57         super(BenchmarkSuite,self).__init__()         super(BenchmarkSuite,self).__init__()
# Line 49  class BenchmarkSuite(object): Line 61  class BenchmarkSuite(object):
61             self.__name=self.__class__.__name__             self.__name=self.__class__.__name__
62         else:         else:
63            self.__name=name            self.__name=name
64              
65     def __str__(self):     def __str__(self):
66         """         """
67         returns the name of the benchmark suite         Returns the name of the benchmark suite.
68          
69         @return:  name         @return: the name
70         @rtype: C{str}         @rtype: C{str}
71         """         """
72         return self.__name         return self.__name
73    
74     def addBenchmark(self,benchmark):     def addBenchmark(self,benchmark):
75         """         """
76         adds a L{Benchmark} to the suite         Adds a new L{Benchmark} to the suite.
77    
78         @param benchmark: adds a new L{Benchmark} to the suite         @param benchmark: the benchmark to add
79         @type benchmark: L{Benchmark}         @type benchmark: L{Benchmark}
80         """         """
81         self.__benchmarks.append(benchmark)                 self.__benchmarks.append(benchmark)
82    
83     def __len__(self):     def __len__(self):
84         """         """
85         returns the number of benchmarks in the suite         Returns the number of benchmarks in the suite.
86    
87         @return:  number of benchmarks         @return: number of benchmarks
88         @rtype: C{int}               @rtype: C{int}
89         """         """
90         return len(self.__benchmarks)         return len(self.__benchmarks)
91    
92     def __getitem__(self,i):     def __getitem__(self,i):
93         """         """
94         returns the i-th benchmark in the suite through self[i]         Returns the i-th benchmark in the suite through self[i].
95          
96         @param i: index of the requested benchmark         @param i: index of the requested benchmark
97         @type i: C{int}         @type i: C{int}
98         @return:  i-th benchmark         @return: i-th benchmark
99         @rtype: L{Benchmark}               @rtype: L{Benchmark}
100    
101         """         """
102         return self.__benchmarks[i]         return self.__benchmarks[i]
103    
104     def run(self,scale=1):     def run(self,scale=1):
105         """         """
106         runs all benchmarks         Runs all benchmarks.
107    
108         @param scale: defines the number of (OpenMP) threads to be used. If scale is a scalar all benchmarks         @param scale: defines the number of (OpenMP) threads to be used. If
109                       are run with scale number of threads. If scale is a C{list}, the p-th problem in each of the benchmarks                       C{scale} is a scalar all benchmarks are run with C{scale}
110                       in the suite is run with scale[p] threads. If scale[p]<1 teh p-th problem is omitted.                       number of threads. If C{scale} is a C{list}, the p-th
111         @type scale: C{int} or C{list} of C{int}s.                       problem in each of the benchmarks in the suite is run with
112                         C{scale[p]} threads. If C{scale[p]}<1 the p-th problem is
113                         omitted.
114           @type scale: C{int} or C{list} of C{int}s
115         """         """
116         self.__scale=scale               self.__scale=scale
117         for i in range(len(self)): self[i].run(scale=scale)         for i in range(len(self)): self[i].run(scale=scale)
118    
119     def getHTML(self,filter,level=1):     def getHTML(self,filter,level=1):
120         """         """
121         returns the results of the last benchmark run in HTML format.         Returns the results of the last benchmark run in HTML format.
122    
123         @param filter: filter to be applied to the results         @param filter: filter to be applied to the results
124         @type filter: L{BenchmarkFilter}         @type filter: L{BenchmarkFilter}
125         @param level: level used in header <H?> tags         @param level: level used in header <H?> tags
126         @type level: C{int}         @type level: C{int}
127         @return: HTML document         @return: HTML document
128         @rtype: C{str}         @rtype: C{str}
# Line 110  class BenchmarkSuite(object): Line 130  class BenchmarkSuite(object):
130         out=""         out=""
131         if level==1: out+="<HTML><HEAD><TITLE>Benchmark: %s</TITLE></HEAD><BODY>\n"%str(self)         if level==1: out+="<HTML><HEAD><TITLE>Benchmark: %s</TITLE></HEAD><BODY>\n"%str(self)
132         out+="<H%s>%s</H%s>\n"%(level,str(self),level)         out+="<H%s>%s</H%s>\n"%(level,str(self),level)
133         if level==1:         if level==1:
134             m=""             m=""
135             if isinstance(self.__scale,int):             if isinstance(self.__scale,int):
136                if self.__scale>1:                if self.__scale>1:
# Line 121  class BenchmarkSuite(object): Line 141  class BenchmarkSuite(object):
141             out+=self[i].getHTML(filter=filter,level=min(level+1,self.MAX_LEVEL))             out+=self[i].getHTML(filter=filter,level=min(level+1,self.MAX_LEVEL))
142             out+="<p>\n"             out+="<p>\n"
143         if level==1:         if level==1:
144             try:             try:
145                 name=os.getlogin()                 name=os.getlogin()
146                 out+="<hr><p align=\"center\">by %s at %s</p>\n"%(name,time.strftime('%X %x %Z'))                 out+="<hr><p align=\"center\">by %s at %s</p>\n"%(name,time.strftime('%X %x %Z'))
147             except OSError:             except OSError:
148                 out+="<hr><p align=\"center\">%s</p>\n"%(time.strftime('%X %x %Z'))                 out+="<hr><p align=\"center\">%s</p>\n"%(time.strftime('%X %x %Z'))
149                  
150             out+="</BODY></HTML>\n"             out+="</BODY></HTML>\n"
151         return out         return out
152    
153    
154  class Benchmark(object):  class Benchmark(object):
155     """     """
156     runs a bunch of similar L{BenchmarkProblem}s with a bunch of L{Options}     Runs a bunch of similar L{BenchmarkProblem}s with a bunch of L{Options}.
157     """     """
158     def __init__(self,name=None,description=None):     def __init__(self,name=None,description=None):
159         """         """
160         sets up a benchmark         Sets up a benchmark.
161    
162         @param name: name of the benchmark. If no name is given the class name is used.         @param name: name of the benchmark. If no name is given the class name
163                        is used.
164         @type name: C{str}         @type name: C{str}
165         @param description: description of the benchmark.         @param description: description of the benchmark
166         @type description: C{str} or C{None}               @type description: C{str} or C{None}
167         """         """
168         super(Benchmark,self).__init__()         super(Benchmark,self).__init__()
169         self.__options=[]         self.__options=[]
# Line 154  class Benchmark(object): Line 175  class Benchmark(object):
175         else:         else:
176            self.__name=name            self.__name=name
177         self.__description=description         self.__description=description
178          
179     def __str__(self):     def __str__(self):
180         """         """
181         returns the name of the benchmark suite         Returns the name of the benchmark suite.
182          
183         @return:  name         @return: the name
184         @rtype: C{str}         @rtype: C{str}
185         """         """
186         return self.__name         return self.__name
187              
188     def addProblem(self,problem):     def addProblem(self,problem):
189         """         """
190         adds a problem to the benchmark         Adds a problem to the benchmark.
191    
192         @param problem: adds a new problem to the bechmark         @param problem: the problem to be added
193         @type problem: L{BenchmarkProblem}         @type problem: L{BenchmarkProblem}
194         """         """
195         self.__problems.append(problem)         self.__problems.append(problem)
196    
197     def addOptions(self,Options):     def addOptions(self,options):
198         """         """
199         adds a options to the benchmark         Adds options to the benchmark.
200    
201         @param options: adds a new option to the bechmark         @param options: the options to be added to the benchmark. If
202         @type problem: L{Options}                         options==None the options are left unchanged.
203           @type options: L{Options}
204         """         """
205         self.__options.append(Options)         if options!=None: self.__options.append(options)
206    
207     def run(self,scale=1):     def run(self,scale=1):
208         """         """
209         runs all problems with all options.         Runs all problems with all options.
   
210    
211         @param scale: defines the number of (OpenMP) threads to be used. If scale is a scalar all benchmarks         @param scale: defines the number of (OpenMP) threads to be used. If
212                       are run with scale number of threads. If scale is a C{list}, the p-th problem in each of the benchmarks                       C{scale} is a scalar all benchmarks are run with C{scale}
213                       in the suite is run with scale[p] threads. If scale[p]<1 teh p-th problem is omitted.                       number of threads. If C{scale} is a C{list}, the p-th
214         @type scale: C{int} or C{list} of C{int}s.                       problem in each of the benchmarks in the suite is run with
215                         C{scale[p]} threads. If C{scale[p]}<1 the p-th problem is
216                         omitted.
217           @type scale: C{int} or C{list} of C{int}s
218         """         """
219         if isinstance(scale,list):         if isinstance(scale,list):
220             c_max=min(len(scale),len(self.__problems))             c_max=min(len(scale),len(self.__problems))
# Line 207  class Benchmark(object): Line 231  class Benchmark(object):
231               s=scale               s=scale
232            row=[]            row=[]
233            if s>0:            if s>0:
234                print "run %s with %s threads"%(r.__class__,s)                t0=time.time()
235                  print "%s with %s threads started."%(r.__class__,s)
236                for p in self.__options:                for p in self.__options:
237                    setNumberOfThreads(s)                    setNumberOfThreads(s)
238                    row.append(r.run(p))                    try:
239                         row.append(r.run(p))
240                      except:
241                         traceback.print_exc(file=sys.stdout)
242                         row.append(None)
243                  t0=time.time()-t0
244                  print "%s with %s threads finished (walltime=%s sec)."%(r.__class__,s,t0)
245            self.__results.append(row)            self.__results.append(row)
246    
247     def getHTML(self,filter,level=1):     def getHTML(self,filter,level=1):
248         """         """
249         returns the results of the last benchmark run in HTML format.         Returns the results of the last benchmark run in HTML format.
250    
251         @param filter: filter to be applied to the results         @param filter: filter to be applied to the results
252         @type filter: L{BenchmarkFilter}         @type filter: L{BenchmarkFilter}
253         @param level: level used in header <H?> tags         @param level: level used in header <H?> tags
254         @type level: C{int}         @type level: C{int}
255         @return: HTML document         @return: HTML document
256         @rtype: C{str}         @rtype: C{str}
# Line 226  class Benchmark(object): Line 258  class Benchmark(object):
258         out=""         out=""
259         if level==1: out+="<HTML><HEAD><TITLE>Benchmark: %s</TITLE></HEAD><BODY>\n"%str(self)         if level==1: out+="<HTML><HEAD><TITLE>Benchmark: %s</TITLE></HEAD><BODY>\n"%str(self)
260         out+="<H%s>%s</H%s>\n"%(level,str(self),level)         out+="<H%s>%s</H%s>\n"%(level,str(self),level)
261         if level==1:         if level==1:
262           m=""           m=""
263           if isinstance(self.__scale,int):           if isinstance(self.__scale,int):
264              if self.__scale>1:              if self.__scale>1:
265                  m=" (%s threads)"%self.__scale                  m=" (%s threads)"%self.__scale
266           out+="<p>platform: %s%s</p>\n"%(socket.gethostname(),m)           out+="<p>platform: %s%s</p>\n"%(socket.gethostname(),m)
267         if self.__description: out+="<p>%s</p>\n"%str(self.__description)           if self.__description: out+="<p>%s</p>\n"%str(self.__description)
268         if len(self.__problems)>0:         if len(self.__problems)>0:
269            out+="<TABLE ALIGN=\"center\" BORDER=3 CELLPADDING=5 CELLSPACING=1>\n"            out+="<TABLE ALIGN=\"center\" BORDER=3 CELLPADDING=5 CELLSPACING=1>\n"
270            h1_seg=""            h1_seg=""
# Line 240  class Benchmark(object): Line 272  class Benchmark(object):
272            if len(rn)==0:            if len(rn)==0:
273               h1_seg+="<TD></TD>"               h1_seg+="<TD></TD>"
274            else:            else:
275               for n in rn: h1_seg+="<TD ALIGN=\"center\">%s</TD>"%n                       for n in rn: h1_seg+="<TD ALIGN=\"center\">%s</TD>"%n
276            h0="<TR><TH ALIGN=\"center\" ROWSPAN=2>Case</TH>"            h0="<TR><TH ALIGN=\"center\" ROWSPAN=2>Case</TH>"
277            h1="<TR>"            h1="<TR>"
278            if isinstance(self.__scale,list): h0+="<TH ALIGN=\"center\" ROWSPAN=2>Threads</TH>"            if isinstance(self.__scale,list): h0+="<TH ALIGN=\"center\" ROWSPAN=2>Threads</TH>"
279            for o in self.__options:            for o in self.__options:
280                   if len(rn)==0:                   if len(rn)==0:
281                       h0+="<TH ALIGN=\"center\">%s</TH>"%str(o)                       h0+="<TH ALIGN=\"center\">%s</TH>"%str(o)
282                         colspan=1
283                   elif len(rn)==1:                   elif len(rn)==1:
284                       h0+="<TH ALIGN=\"center\">%s</TH>"%str(o)                       h0+="<TH ALIGN=\"center\">%s</TH>"%str(o)
285                         colspan=1
286                       empty_h1=False                       empty_h1=False
287                   else:                   else:
288                       h0+="<TH ALIGN=\"center\" COLSPAN=%s>%s</TH>"%(len(rn),str(o))                       colspan=len(rn)
289                         h0+="<TH ALIGN=\"center\" COLSPAN=%s>%s</TH>"%(colspan,str(o))
290                   h1+=h1_seg                   h1+=h1_seg
291            out+=h0+"</TR>\n"+h1+"</TR>\n"            out+=h0+"</TR>\n"+h1+"</TR>\n"
292            c=0            c=0
293            for r in range(len(self.__results)):            for r in range(len(self.__results)):
294               out+="<TR><TH ALIGN=\"right\">%s</TH>"%str(self.__problems[r])               out+="<TR><TH ALIGN=\"right\">%s</TH>"%str(self.__problems[r])
295               if isinstance(self.__scale,list): out+="<TD ALIGN=\"right\">%s</TD>"%self.__scale[c]               if isinstance(self.__scale,list):
296                     out+="<TD ALIGN=\"right\">%s</TD>"%self.__scale[c]
297               for col in self.__results[r]:               for col in self.__results[r]:
298                     for e in filter(col): out+="<TD ALIGN=\"right\">%s</TD>"%e                     if col==None:
299                          out+="<TD ALIGN=\"center\" COLSPAN=%s>failed.</TD>"%colspan
300                       else:
301                          for e in filter(col): out+="<TD ALIGN=\"right\">%s</TD>"%e
302               out+="</TR>\n"               out+="</TR>\n"
303               c+=1               c+=1
304            out+="</TABLE>"            out+="</TABLE>"
305         if level==1:         if level==1:
306            out+="<hr><p align=\"center\">by %s at %s</p>\n"%(os.getlogin(),time.strftime('%X %x %Z'))            out+="<hr><p align=\"center\">by %s at %s</p>\n"%(os.getlogin(),time.strftime('%X %x %Z'))
307            out+="</BODY></HTML>\n"            out+="</BODY></HTML>\n"
308         return out         return out
309          
310  class BenchmarkProblem(object):  class BenchmarkProblem(object):
311     """     """
312     something that can be run and returns a list of characteristics such as timing, Mflops, error, etc.     Represents a benchmark problem that can be run and which returns a list of
313       characteristics such as timing, MFlops, error, etc.
314     """     """
315     def __init__(self,name=None):     def __init__(self,name=None):
316         """         """
317         sets up a benchmark problem         Sets up a benchmark problem.
318    
319         @param name: name of the problem. If no name is given the class name is used.         @param name: name of the problem. If no name is given the class name
320                        is used.
321         @type name: C{str}         @type name: C{str}
322         """         """
323         super(BenchmarkProblem,self).__init__()         super(BenchmarkProblem,self).__init__()
# Line 285  class BenchmarkProblem(object): Line 326  class BenchmarkProblem(object):
326         else:         else:
327            self.__name=name            self.__name=name
328    
         
329     def __str__(self):     def __str__(self):
330         """         """
331         returns the name of the benchmark suite         Returns the name of the benchmark suite.
332          
333         @return:  name         @return: the name
334         @rtype: C{str}         @rtype: C{str}
335         """         """
336         return self.__name         return self.__name
337    
338     def run(self,options=None):     def run(self,options=None):
339         """         """
340         runs the problem and returns a list of run characteristics         Runs the problem and returns a list of run characteristics.
   
341    
342         @param options: the options that are used for the run. Note that the number of OpenMP threads is controlled         @param options: the options that are used for the run. Note that the
343                         by the L{Benchmark} the problem is run in.                         number of OpenMP threads is controlled by the
344                           L{Benchmark} the problem is run in.
345         @type options: L{Options}         @type options: L{Options}
346         @return: run characteristics         @return: run characteristics
347         @rtype: any type that can be read by the L{BenchmarkFilter} applied to it.         @rtype: any type that can be read by the L{BenchmarkFilter} applied
348         @remark: this function has to overwritten by a particular problem                 to it
349           @note: this function has to be overwritten by a particular problem
350         """         """
351         raise NotImplementedError         raise NotImplementedError
352         return []         return []
353        
354  class BenchmarkFilter(object):  class BenchmarkFilter(object):
355     """     """
356     object to filter the characteristcs returned by Bechmark runs.     Object to filter the characteristics returned by Benchmark runs.
357      
358     """     """
359     def __init__(self):     def __init__(self):
360         """         """
361         sets up a filter         Sets up a filter.
362         """         """
363         pass         pass
   
364    
365     def getResultNames(self):     def getResultNames(self):
366         """         """
367         return the names of the results produced when run() is called.         Returns the names of the results produced when C{run()} is called.
368          
369         @return: names the list of the names to be used when the results of the run() call are printed         @return: the list of the names to be used when the results of
370                    the C{run()} call are printed
371         @rtype: C{list} of C{str}         @rtype: C{list} of C{str}
372         @remark: this function has to overwritten by a particular problem         @note: this function has to overwritten by a particular problem
373         """         """
374         raise NotImplementedError         raise NotImplementedError
375         return []         return []
376    
377     def __call__(self,result):     def __call__(self,result):
378         """         """
379         filters out values results returned as characteristcs of a problem run         Filters out results returned as characteristics of a problem run.
380          
381         @param result: values to be filtered         @param result: values to be filtered
382         @type result: any type that is produced by the L{BenchmarkProblem} it is applied to         @type result: any type that is produced by the L{BenchmarkProblem}
383                         it is applied to
384         @return: a list of strings selected from result         @return: a list of strings selected from result
385         @rtype: C{list} of C{str}         @rtype: C{list} of C{str}
386         @remark: this function has to overwritten by a particular problem         @note: this function has to be overwritten by a particular problem
387         """         """
388         raise NotImplementedError         raise NotImplementedError
389         return []         return []
# Line 349  class BenchmarkFilter(object): Line 391  class BenchmarkFilter(object):
391    
392  class Options(object):  class Options(object):
393      """      """
394      defines a set of options to be used to run a L{BenchmarkProblem}      Defines a set of options to be used to run a L{BenchmarkProblem}.
395      """      """
396      def __init__(self,name=None):      def __init__(self,name=None):
397         """         """
398         sets up the options         Sets up the options.
399    
400         @param name: name of the option. If no name is given the class name is used.         @param name: name of the option. If no name is given the class name
401                        is used.
402         @type name: C{str}         @type name: C{str}
403         """         """
404         super(Options,self).__init__()         super(Options,self).__init__()
# Line 363  class Options(object): Line 406  class Options(object):
406            self.__name=self.__class__.__name__            self.__name=self.__class__.__name__
407         else:         else:
408            self.__name=name            self.__name=name
409    
410      def __str__(self):      def __str__(self):
411         """         """
412         returns the name of the benchmark suite         Returns the name of this options object.
413          
414         @return:  name         @return: the name
415         @rtype: C{str}         @rtype: C{str}
416         """         """
417         return self.__name         return self.__name
418        
419  if __name__=="__main__":  if __name__=="__main__":
420    
421      class OptionsTest1(Options):      class OptionsTest1(Options):
# Line 395  if __name__=="__main__": Line 439  if __name__=="__main__":
439    
440      class SimpleFilter(BenchmarkFilter):      class SimpleFilter(BenchmarkFilter):
441         def getResultNames(self):         def getResultNames(self):
442              return ["r0","r1"]                return ["r0","r1"]
443         def __call__(self,result):         def __call__(self,result):
444              return [str(result[0]),str(result[1])]              return [str(result[0]),str(result[1])]
445    
# Line 410  if __name__=="__main__": Line 454  if __name__=="__main__":
454    
455      bms.run()      bms.run()
456      print bms.getHTML(filter=SimpleFilter())      print bms.getHTML(filter=SimpleFilter())
457        
458      bms.run(scale=4)      bms.run(scale=4)
459      print bms.getHTML(filter=SimpleFilter())      print bms.getHTML(filter=SimpleFilter())
460    
461      bms.run(scale=[1,2])      bms.run(scale=[1,2])
462      print bms.getHTML(filter=SimpleFilter())      print bms.getHTML(filter=SimpleFilter())
463    

Legend:
Removed from v.434  
changed lines
  Added in v.2549

  ViewVC Help
Powered by ViewVC 1.1.26