/[escript]/trunk/escript/py_src/benchmark.py
ViewVC logotype

Diff of /trunk/escript/py_src/benchmark.py

Parent Directory Parent Directory | Revision Log Revision Log | View Patch Patch

revision 364 by gross, Thu Dec 15 04:38:28 2005 UTC revision 609 by elspeth, Tue Mar 21 09:46:39 2006 UTC
# Line 1  Line 1 
1  # $Id:$  filter# $Id:$
   
 #  
 #      COPYRIGHT ACcESS 2004 -  All Rights Reserved  
 #  
 #   This software is the property of ACcESS.  No part of this code  
 #   may be copied in any form or by any means without the expressed written  
 #   consent of ACcESS.  Copying, use or modification of this software  
 #   by any unauthorised person is illegal unless that  
 #   person has a software license agreement with ACcESS.  
 #  
2    
3  """  """
4  A simple framework to run benchmarks under OPENMP and to summarize the results in tables for instance in HTML  A simple framework to run benchmarks under OPENMP and to summarize the results in tables for instance in HTML
# Line 21  var __url__: url entry point on document Line 11  var __url__: url entry point on document
11  """  """
12    
13  __author__="Lutz Gross, l.gross@uq.edu.au"  __author__="Lutz Gross, l.gross@uq.edu.au"
14  __licence__="contact: esys@access.uq.edu.au"  __copyright__="""  Copyright (c) 2006 by ACcESS MNRF
15                        http://www.access.edu.au
16                    Primary Business: Queensland, Australia"""
17    __licence__="""Licensed under the Open Software License version 3.0
18                 http://www.opensource.org/licences/osl-3.0.php"""
19  __url__="http://www.iservo.edu.au/esys/escript"  __url__="http://www.iservo.edu.au/esys/escript"
20  __version__="$Revision:$"  __version__="$Revision:$"
21  __date__="$Date:$"  __date__="$Date:$"
22    
23  import os,socket,time,sys  import os,socket,time,sys,traceback
24    from esys.escript import setNumberOfThreads
25    
26  class BenchmarkSuite(object):  class BenchmarkSuite(object):
27     """     """
# Line 90  class BenchmarkSuite(object): Line 85  class BenchmarkSuite(object):
85    
86         @param scale: defines the number of (OpenMP) threads to be used. If scale is a scalar all benchmarks         @param scale: defines the number of (OpenMP) threads to be used. If scale is a scalar all benchmarks
87                       are run with scale number of threads. If scale is a C{list}, the p-th problem in each of the benchmarks                       are run with scale number of threads. If scale is a C{list}, the p-th problem in each of the benchmarks
88                       in the suite is run with scale[p] threads. In the case, len(scale) has to be less or equal to the                       in the suite is run with scale[p] threads. If scale[p]<1 teh p-th problem is omitted.
                      largest benchmark in the suite.  
89         @type scale: C{int} or C{list} of C{int}s.         @type scale: C{int} or C{list} of C{int}s.
90         """         """
91         self.__scale=scale               self.__scale=scale      
92         for i in range(len(self)): self[i].run(scale)         for i in range(len(self)): self[i].run(scale=scale)
93     def getHTML(self,level=1):     def getHTML(self,filter,level=1):
94         """         """
95         returns the results of the last benchmark run in HTML format.         returns the results of the last benchmark run in HTML format.
96    
97           @param filter: filter to be applied to the results
98           @type filter: L{BenchmarkFilter}
99         @param level: level used in header <H?> tags         @param level: level used in header <H?> tags
100         @type level: C{int}         @type level: C{int}
101         @return: HTML document         @return: HTML document
# Line 116  class BenchmarkSuite(object): Line 112  class BenchmarkSuite(object):
112             out+="<p>platform: %s%s</p>\n"%(socket.gethostname(),m)             out+="<p>platform: %s%s</p>\n"%(socket.gethostname(),m)
113         for i in range(len(self)):         for i in range(len(self)):
114             out+="<p>\n"             out+="<p>\n"
115             out+=self[i].getHTML(min(level+1,self.MAX_LEVEL))             out+=self[i].getHTML(filter=filter,level=min(level+1,self.MAX_LEVEL))
116             out+="<p>\n"             out+="<p>\n"
117         if level==1:         if level==1:
118             out+="<hr><p align=\"center\">by %s at %s</p>\n"%(os.getlogin(),time.strftime('%X %x %Z'))             try:
119                   name=os.getlogin()
120                   out+="<hr><p align=\"center\">by %s at %s</p>\n"%(name,time.strftime('%X %x %Z'))
121               except OSError:
122                   out+="<hr><p align=\"center\">%s</p>\n"%(time.strftime('%X %x %Z'))
123                  
124             out+="</BODY></HTML>\n"             out+="</BODY></HTML>\n"
125         return out         return out
126    
# Line 166  class Benchmark(object): Line 167  class Benchmark(object):
167         """         """
168         self.__problems.append(problem)         self.__problems.append(problem)
169    
170     def addOptions(self,Options):     def addOptions(self,options):
171         """         """
172         adds a options to the benchmark         adds a options to the benchmark
173    
174         @param options: adds a new option to the bechmark         @param options: adds a new option to the bechmark. If options==None they are are ignored
175         @type problem: L{Options}         @type problem: L{Options}
176         """         """
177         self.__options.append(Options)         if options!=None: self.__options.append(options)
178    
179     def run(self,scale=1):     def run(self,scale=1):
180         """         """
# Line 182  class Benchmark(object): Line 183  class Benchmark(object):
183    
184         @param scale: defines the number of (OpenMP) threads to be used. If scale is a scalar all benchmarks         @param scale: defines the number of (OpenMP) threads to be used. If scale is a scalar all benchmarks
185                       are run with scale number of threads. If scale is a C{list}, the p-th problem in each of the benchmarks                       are run with scale number of threads. If scale is a C{list}, the p-th problem in each of the benchmarks
186                       in the suite is run with scale[p] threads. In the case, len(scale) has to be less or equal to the                       in the suite is run with scale[p] threads. If scale[p]<1 teh p-th problem is omitted.
                      largest benchmark in the suite.  
187         @type scale: C{int} or C{list} of C{int}s.         @type scale: C{int} or C{list} of C{int}s.
188         """         """
189         if isinstance(scale,list):         if isinstance(scale,list):
190             if len(scale)<len(self.__problems):             c_max=min(len(scale),len(self.__problems))
191                raise ValueError,"scale list is too small. must be greater or equal to the number of problems in the benchmark"         else:
192               c_max=len(self.__problems)
193           self.__filter=filter
194         self.__scale=scale         self.__scale=scale
195         self.__results=[]         self.__results=[]
196         c=0         for c in range(c_max):
197         for r in self.__problems:            r=self.__problems[c]
198            if isinstance(scale,list):            if isinstance(scale,list):
199               s=scale[c]               s=scale[c]
200            else:            else:
201               s=scale               s=scale
202            row=[]            row=[]
203            for p in self.__options:            if s>0:
204                os.putenv("OMP_NUM_TREADS",str(s))                t0=time.time()
205                row.append(r.run(p))                print "%s with %s threads started."%(r.__class__,s)
206                  for p in self.__options:
207                      setNumberOfThreads(s)
208                      try:
209                         row.append(r.run(p))
210                      except:
211                         traceback.print_exc(file=sys.stdout)
212                         row.append(None)
213                  t0=time.time()-t0
214                  print "%s with %s threads finished (walltime =%s sec)."%(r.__class__,s,t0)
215            self.__results.append(row)            self.__results.append(row)
216            c+=1     def getHTML(self,filter,level=1):
    def getHTML(self,level=1):  
217         """         """
218         returns the results of the last benchmark run in HTML format.         returns the results of the last benchmark run in HTML format.
219    
220           @param filter: filter to be applied to the results
221           @type filter: L{BenchmarkFilter}
222         @param level: level used in header <H?> tags         @param level: level used in header <H?> tags
223         @type level: C{int}         @type level: C{int}
224         @return: HTML document         @return: HTML document
# Line 225  class Benchmark(object): Line 237  class Benchmark(object):
237         if len(self.__problems)>0:         if len(self.__problems)>0:
238            out+="<TABLE ALIGN=\"center\" BORDER=3 CELLPADDING=5 CELLSPACING=1>\n"            out+="<TABLE ALIGN=\"center\" BORDER=3 CELLPADDING=5 CELLSPACING=1>\n"
239            h1_seg=""            h1_seg=""
240            rn=self.__problems[0].getResultNames()            rn=filter.getResultNames()
241            if len(rn)==0:            if len(rn)==0:
242               h1_seg+="<TD></TD>"               h1_seg+="<TD></TD>"
243            else:            else:
# Line 236  class Benchmark(object): Line 248  class Benchmark(object):
248            for o in self.__options:            for o in self.__options:
249                   if len(rn)==0:                   if len(rn)==0:
250                       h0+="<TH ALIGN=\"center\">%s</TH>"%str(o)                       h0+="<TH ALIGN=\"center\">%s</TH>"%str(o)
251                         colspan=1
252                   elif len(rn)==1:                   elif len(rn)==1:
253                       h0+="<TH ALIGN=\"center\">%s</TH>"%str(o)                       h0+="<TH ALIGN=\"center\">%s</TH>"%str(o)
254                         colspan=1
255                       empty_h1=False                       empty_h1=False
256                   else:                   else:
257                       h0+="<TH ALIGN=\"center\" COLSPAN=%s>%s</TH>"%(len(rn),str(o))                       colspan=len(rn)
258                         h0+="<TH ALIGN=\"center\" COLSPAN=%s>%s</TH>"%(colspan,str(o))
259                   h1+=h1_seg                   h1+=h1_seg
260            out+=h0+"</TR>\n"+h1+"</TR>\n"            out+=h0+"</TR>\n"+h1+"</TR>\n"
261            c=0            c=0
262            for r in range(len(self.__results)):            for r in range(len(self.__results)):
263               out+="<TR><TH ALIGN=\"right\">%s</TH>"%str(self.__problems[r])               out+="<TR><TH ALIGN=\"right\">%s</TH>"%str(self.__problems[r])
264               if isinstance(self.__scale,list): out+="<TD ALIGN=\"right\">%s</TD>"%self.__scale[c]               if isinstance(self.__scale,list):
265               for col in self.__results[r]:                   out+="<TD ALIGN=\"right\">%s</TD>"%self.__scale[c]
266                     for e in col: out+="<TD ALIGN=\"right\">%s</TD>"%str(e)               for col in self.__results[r]:
267                       if col==None:
268                          out+="<TD ALIGN=\"center\" COLSPAN=%s>failed.</TD>"%colspan
269                       else:
270                          for e in filter(col): out+="<TD ALIGN=\"right\">%s</TD>"%e
271               out+="</TR>\n"               out+="</TR>\n"
272                 c+=1
273            out+="</TABLE>"            out+="</TABLE>"
           c+=1  
274         if level==1:         if level==1:
275            out+="<hr><p align=\"center\">by %s at %s</p>\n"%(os.getlogin(),time.strftime('%X %x %Z'))            out+="<hr><p align=\"center\">by %s at %s</p>\n"%(os.getlogin(),time.strftime('%X %x %Z'))
276            out+="</BODY></HTML>\n"            out+="</BODY></HTML>\n"
# Line 274  class BenchmarkProblem(object): Line 293  class BenchmarkProblem(object):
293         else:         else:
294            self.__name=name            self.__name=name
295    
    def getResultNames(self):  
        """  
        return the names of the results produced when run() is called.  
         
        @return: names the list of the names to be used when the results of the run() call are printed  
        @rtype: C{list} of C{str}  
        @remark: this function has to overwritten by a particular problem  
        """  
        raise NotImplementedError  
        return []  
296                
297     def __str__(self):     def __str__(self):
298         """         """
# Line 302  class BenchmarkProblem(object): Line 311  class BenchmarkProblem(object):
311         @param options: the options that are used for the run. Note that the number of OpenMP threads is controlled         @param options: the options that are used for the run. Note that the number of OpenMP threads is controlled
312                         by the L{Benchmark} the problem is run in.                         by the L{Benchmark} the problem is run in.
313         @type options: L{Options}         @type options: L{Options}
314         @return: list of run characteristics         @return: run characteristics
315         @rtype: C{list}         @rtype: any type that can be read by the L{BenchmarkFilter} applied to it.
316         @remark: this function has to overwritten by a particular problem         @remark: this function has to overwritten by a particular problem
317         """         """
318         raise NotImplementedError         raise NotImplementedError
319         return []         return []
320            
321    class BenchmarkFilter(object):
322       """
323       object to filter the characteristcs returned by Bechmark runs.
324      
325       """
326       def __init__(self):
327           """
328           sets up a filter
329           """
330           pass
331    
332    
333       def getResultNames(self):
334           """
335           return the names of the results produced when run() is called.
336          
337           @return: names the list of the names to be used when the results of the run() call are printed
338           @rtype: C{list} of C{str}
339           @remark: this function has to overwritten by a particular problem
340           """
341           raise NotImplementedError
342           return []
343    
344       def __call__(self,result):
345           """
346           filters out values results returned as characteristcs of a problem run
347          
348           @param result: values to be filtered
349           @type result: any type that is produced by the L{BenchmarkProblem} it is applied to
350           @return: a list of strings selected from result
351           @rtype: C{list} of C{str}
352           @remark: this function has to overwritten by a particular problem
353           """
354           raise NotImplementedError
355           return []
356    
357    
358  class Options(object):  class Options(object):
359      """      """
360      defines a set of options to be used to run a L{BenchmarkProblem}      defines a set of options to be used to run a L{BenchmarkProblem}
# Line 322  class Options(object): Line 368  class Options(object):
368         """         """
369         super(Options,self).__init__()         super(Options,self).__init__()
370         if name==None:         if name==None:
371             self.__name=self.__class__.__name__            self.__name=self.__class__.__name__
372         else:         else:
373            self.__name=name            self.__name=name
374      def __str__(self):      def __str__(self):
# Line 341  if __name__=="__main__": Line 387  if __name__=="__main__":
387      class OptionsTest2(Options):      class OptionsTest2(Options):
388          pass          pass
389    
390      class BenchmarkProblemTest(BenchmarkProblem):      class BenchmarkProblemTest1(BenchmarkProblem):
        def __init__(self,name=None):  
            super(BenchmarkProblemTest,self).__init__(name)  
        def getResultNames(self):  
             return ["r0","r1"]    
   
     class BenchmarkProblemTest1(BenchmarkProblemTest):  
391         def __init__(self):         def __init__(self):
392             super(BenchmarkProblemTest1,self).__init__(name="TEST1")             super(BenchmarkProblemTest1,self).__init__(name="TEST1")
393         def run(self,options=None):         def run(self,options=None):
394             import time             import time
395             return time.time(),"A"             return time.time(),"A"
396    
397      class BenchmarkProblemTest2(BenchmarkProblemTest):      class BenchmarkProblemTest2(BenchmarkProblem):
398         def __init__(self):         def __init__(self):
399             super(BenchmarkProblemTest2,self).__init__(name="TEST2")             super(BenchmarkProblemTest2,self).__init__(name="TEST2")
400         def run(self,options=None):         def run(self,options=None):
401             import time             import time
402             return -time.time(),"B"             return -time.time(),"B"
403    
404        class SimpleFilter(BenchmarkFilter):
405           def getResultNames(self):
406                return ["r0","r1"]  
407           def __call__(self,result):
408                return [str(result[0]),str(result[1])]
409    
410      bm=Benchmark("Example")      bm=Benchmark("Example")
411      bm.addProblem(BenchmarkProblemTest1())      bm.addProblem(BenchmarkProblemTest1())
412      bm.addProblem(BenchmarkProblemTest2())      bm.addProblem(BenchmarkProblemTest2())
# Line 371  if __name__=="__main__": Line 417  if __name__=="__main__":
417      bms.addBenchmark(bm)      bms.addBenchmark(bm)
418    
419      bms.run()      bms.run()
420      print bms.getHTML()      print bms.getHTML(filter=SimpleFilter())
421            
422      bms.run(scale=4)      bms.run(scale=4)
423      print bms.getHTML()      print bms.getHTML(filter=SimpleFilter())
424    
425      bms.run(scale=[1,2])      bms.run(scale=[1,2])
426      print bms.getHTML()      print bms.getHTML(filter=SimpleFilter())

Legend:
Removed from v.364  
changed lines
  Added in v.609

  ViewVC Help
Powered by ViewVC 1.1.26