/[escript]/trunk/site_scons/site_init.py
ViewVC logotype

Diff of /trunk/site_scons/site_init.py

Parent Directory Parent Directory | Revision Log Revision Log | View Patch Patch

temp_trunk_copy/scons/scons_extensions.py revision 1384 by phornby, Fri Jan 11 02:29:38 2008 UTC trunk/site_scons/site_init.py revision 5045 by caltinay, Fri Jun 13 02:53:39 2014 UTC
# Line 1  Line 1 
1    
2    ##############################################################################
3  # Extensions to Scons  #
4    # Copyright (c) 2003-2014 by University of Queensland
5  import py_compile  # http://www.uq.edu.au
6  import sys  #
7  import os  # Primary Business: Queensland, Australia
8  import time  # Licensed under the Open Software License version 3.0
9  import glob  # http://www.opensource.org/licenses/osl-3.0.php
10  import fnmatch  #
11  import types  # Development until 2012 by Earth Systems Science Computational Center (ESSCC)
12    # Development 2012-2013 by School of Earth Sciences
13  from SCons.Script.SConscript import SConsEnvironment  # Development from 2014 by Centre for Geoscience Computing (GeoComp)
14    #
15  ###############################################################################  ##############################################################################
16  def matchingFiles(env,directory,includes='*',excludes=None) :  
17    __copyright__="""Copyright (c) 2003-2014 by University of Queensland
18      # Pre-process for more convenient arguments  http://www.uq.edu.au
19    Primary Business: Queensland, Australia"""
20      if isinstance(includes,str) :  __license__="""Licensed under the Open Software License version 3.0
21          includes = env.Split(includes)  http://www.opensource.org/licenses/osl-3.0.php"""
22    __url__="https://launchpad.net/escript-finley"
23      if isinstance(excludes,str) :  
24          excludes = env.Split(excludes)  import sys, os, time, py_compile, re, subprocess
25    from SCons.Defaults import Chmod, Copy
26      def fn_filter(node):  from grouptest import *
27          fn = os.path.basename(str(node))  
28          match = False  def findLibWithHeader(env, libs, header, paths, lang='c++'):
29          for include in includes:      from SCons.Script.SConscript import Configure
30              if fnmatch.fnmatchcase( fn, include ):      inc_path=''
31                  match = True      lib_path=''
32        # 'paths' may be a prefix, so look for lib and include subdirectories
33        if type(paths)==str:
34            # find the header file first
35            for i in 'include','include64','include32','inc':
36                inc=os.path.join(paths, i)
37                if os.path.isfile(os.path.join(inc, header)):
38                    inc_path=inc
39                  break                  break
40            if inc_path=='':
41                raise RuntimeError('%s not found under %s'%(header,paths))
42    
43          if match and not excludes is None:          # now try to find a lib directory
44              for exclude in excludes:          for l in 'lib','lib64','lib32':
45                  if fnmatch.fnmatchcase( fn, exclude ):              lp=os.path.join(paths, l)
46                      match = False              if os.path.isdir(lp):
47                      break                  lib_path=lp
48                    break
49          return match          if lib_path=='':
50                raise RuntimeError('No lib directory found under %s'%paths)
51      def filter_nodes(where):      else:
52          contents = glob.glob( os.path.join(str(where),"*") )          if os.path.isfile(os.path.join(paths[0], header)):
53          children = [ x for x in contents if fn_filter(x) ]              inc_path=paths[0]
         nodes = []  
         for f in children:  
             nodes.append(gen_node(f))  
         return nodes  
   
     def gen_node(n):  
         """Checks first to see if the node is a file or a dir, then  
         creates the appropriate node. [code seems redundant, if the node  
         is a node, then shouldn't it just be left as is?  
         """  
         if type(n) in (type(''), type(u'')):  
             path = n  
54          else:          else:
55              path = n.abspath              raise RuntimeError('%s not found under %s'%(header,paths[0]))
56            if os.path.isdir(paths[1]):
57          if os.path.isdir(path):              lib_path=paths[1]
             return env.Dir(n)  
   
         return env.File(n)  
   
     here = env.Dir(directory)  
     nodes = filter_nodes(here)  
   
     node_srcs = [n.srcnode() for n in nodes]  
   
     src = here.srcnode()  
     if src is not here:  
         for s in filter_nodes(src):  
             if s not in node_srcs:  
                 # Probably need to check if this node is a directory  
                 nodes.append(  
                     gen_node(os.path.join(str(directory),  
                                           os.path.basename(str(s)))))  
   
     return nodes  
 #==============================================================================  
 SConsEnvironment.matchingFiles = matchingFiles  
 ###############################################################################  
   
   
   
 ###############################################################################  
 def InstallPyModule(env,target_dir,source,shlib=None,excludes=None) :  
   
     # put the .so/.dll over in  <target_dir>  
     if shlib :  
         shtarg = env.Install(target_dir, shlib)  
   
     # Gather the python sources  
     python_src = env.matchingFiles(source, "*.py",excludes=excludes)  
   
     # Here is a hack to deal with the (possibly not yet generated)  
     # __init__.py. The problem is that the python_src list is built before  
     # __init__.py is updated from __init__.in. The AlwaysBuild call  
     # ensures that this does not cause a problem, except on the first  
     # build after a clean checkout, in which case there is no old  
     # __init__.py in src, and hence it does not make it into python_src!  
   
     init_input = env.matchingFiles(source, "__init__.in")  
     if init_input :  
         if python_src :  
             names = [x.name for x in python_src]  
             if "__init__.py" not in names :  
                 python_src.append(env.File(os.path.join(str(source),  
                                                         "__init__.py")))  
58          else:          else:
59              python_src = [env.File(os.path.join(str(source),"__init__.py"))]              raise RuntimeError('%s is not a valid path.'%paths[1])
               
     # decide if we're doing py or pyc distribn and install.  
60    
61      if  env['distrib_py_src'] :      # now try the library
62          pytarg = env.Install(target_dir, python_src)      conf=Configure(env.Clone())
63      else:      conf.env.AppendUnique(CPPPATH = [inc_path])
64          pyc = env.PyCompile(python_src)      conf.env.AppendUnique(LIBPATH = [lib_path])
65          pytarg = env.Install(target_dir, pyc)      if type(libs)==str: libs=[libs]
66        if len(libs)==0: libs=['']
67        # we can't check for each library by itself since they may depend on each
68        # other, so we add all libraries to the link line and check only for one
69        conf.env.AppendUnique(LIBS = libs)
70        if not conf.CheckLibWithHeader(libs[0], header, lang):
71            conf.Finish()
72            raise RuntimeError('Unable to link against %s (paths: %s, %s)'%(libs,inc_path,lib_path))
73    
74        conf.Finish()
75        return inc_path, lib_path
76    
77    def detectModule(env, module):
78        from tempfile import TemporaryFile
79        p=subprocess.call([env['pythoncmd'],'-c','import %s'%module], stderr=TemporaryFile())
80        if p != 0:
81            env[module] = False
82            return False
83        env[module] = True
84        return True
85    
86    def write_buildvars(env):
87        buildvars=open(os.path.join(env['libinstall'], 'buildvars'), 'w')
88        for k,v in sorted(env['buildvars'].items()):
89            buildvars.write("%s=%s\n"%(k,v))
90        buildvars.close()
91    
92      if shlib :  def generateTestScripts(env, TestGroups):
93          targ_ret = env.Flatten([pytarg] + [shtarg])      try:
94      else:          utest=open('utest.sh','w')
95          targ_ret = pytarg          utest.write(GroupTest.makeHeader(env['PLATFORM'], env['prefix'], False))
96            for tests in TestGroups:
97      return targ_ret              utest.write(tests.makeString())
98  #==============================================================================          utest.close()
99  SConsEnvironment.InstallPyModule = InstallPyModule          env.Execute(Chmod('utest.sh', 0o755))
100  ###############################################################################          print("Generated utest.sh.")
101            # This version contains only python tests - I want this to be usable
102  ###############################################################################          # from a binary only install if you have the test files
103  def installDirectory(env,target,source,includes="*", excludes=None,          utest=open('itest.sh','w')
104                       recursive=False):          utest.write(GroupTest.makeHeader(env['PLATFORM'], env['prefix'], True))
105                for tests in TestGroups:
106                  if tests.exec_cmd=='$PYTHONRUNNER ':
107      if os.path.isfile(str(target)) :              utest.write(tests.makeString())
108          raise UserError("target must be a directory")          utest.close()
109            env.Execute(Chmod('itest.sh', 0o755))
110      if os.path.isfile(str(source)) :          print("Generated itest.sh.")        
111          raise UserError("source must be a directory")      except IOError:
112            env['warnings'].append("Error attempting to write unit test script(s).")
113      source_files = env.matchingFiles(source,includes,excludes)  
114        # delete scripts upon cleanup
115      ret_targ = []      env.Clean('target_init', 'utest.sh')
116        env.Clean('target_init', 'itest.sh')
117      for f in source_files :  
118          if f.isfile() :      # Make sure that the escript wrapper is in place
119              targ = env.Install(target,f)      if not os.path.isfile(os.path.join(env['bininstall'], 'run-escript')):
120              ret_targ.append(targ)          print("Copying escript wrapper.")
121            env.Execute(Copy(os.path.join(env['bininstall'],'run-escript'), 'bin/run-escript'))
         if f.isdir() and recursive :  
             x = os.path.basename(str(f))  
             t = env.Dir(os.path.join(str(target),x))  
             targ = env.installDirectory(t,f,includes,excludes,recursive)  
             ret_targ += targ  
       
     return ret_targ  
 #==============================================================================  
 SConsEnvironment.installDirectory = installDirectory  
 ###############################################################################  
122    
 ###############################################################################  
123  # Code to build .pyc from .py  # Code to build .pyc from .py
124  def build_py(target, source, env):  def build_py(target, source, env):
125      py_compile.compile(str(source[0]), str(target[0]))      try:
126      return 0         py_compile.compile(str(source[0]), str(target[0]), doraise=True)
127           return 0
128        except py_compile.PyCompileError, e:
129           print e
130           return 1
131    
 def doSubstitution(target,source,env) :  
     import product_info as PI  
     data = source[0].get_contents()  
     data = data.replace('$ProductName$',PI.PRODUCT_NAME)  
     data = data.replace('$LowerProductName$',PI.product_name)  
     data = data.replace('$ProductVersion$',PI.product_version)  
     data = data.replace('$VersionString$',PI.pkg_version_string)  
     data = data.replace('$SVNRevision$',PI.svn_revision)  
     open(str(target[0]),'w').write(data)  
     return 0  
132    
133  # Code to run unit_test executables  # Code to run unit_test executables
134  def runUnitTest(target, source, env):  def runUnitTest(target, source, env):
135    time_start = time.time()    time_start = time.time()
136    app = str(source[0].abspath)    app = str(source[0].abspath)
137    if env['useMPI']: app = env['mpi_run'] + ' ' + app    pn, sn= os.path.split(app)
138      if not os.name== "nt":
139         app = "cd "+pn+"; "+os.path.join(env['bininstall'], "run-escript")+" -bv "+os.path.join('.',sn)
140      else:
141          if env['usempi']:
142              app = "cd %s & mpiexec -np %s -genvlist PYTHONPATH,OMP_NUM_THREADS,"\
143                "FINLEY_TEST_DATA,PATH %s"\
144                %(pn,env['ENV']['ESCRIPT_NUM_NODES'], sn)
145          else:
146               app = "cd "+ pn +" & "+sn
147    print "Executing test: " + app    print "Executing test: " + app
148    if not env.Execute(app):    if not env.Execute(app):
149      open(str(target[0]),'w').write("PASSED\n")      open(str(target[0]),'w').write("PASSED\n")
# Line 189  def runUnitTest(target, source, env): Line 152  def runUnitTest(target, source, env):
152    print "Test execution time: ", round(time.time() - time_start, 1), " seconds wall time for " + str(source[0].abspath)    print "Test execution time: ", round(time.time() - time_start, 1), " seconds wall time for " + str(source[0].abspath)
153    return None    return None
154    
155    def binpath(env, name=None):
156        if not name:
157            return env['bininstall']
158        return os.path.join(env['bininstall'], name)
159    
160  def runPyUnitTest(target, source, env):  def runPyUnitTest(target, source, env):
161     time_start = time.time()     time_start = time.time()
162     app = str(source[0].abspath)     app = str(source[0].abspath)
163     if env['useMPI']:     pn, sn= os.path.split(app)
164       app = env['mpi_run'] +' lib/pythonMPI ' + app     if os.name=="nt":
165           if env['usempi']:
166               app = "cd %s & mpiexec -np %s -genvlist PYTHONPATH,OMP_NUM_THREADS,"\
167                  "FINLEY_TEST_DATA,PATH %s\pythonMPIredirect.exe %s"\
168                  %(pn,env['ENV']['ESCRIPT_NUM_NODES'],env['libinstall'],sn)
169           else:
170               app = "cd "+ pn +" & "+sys.executable + " " + sn
171     else:     else:
172       app = sys.executable + " " + app       skipfile = os.path.join(env['build_dir'], sn[:-3]) + ".skipped"
173     print "Executing test: " + app       try:
174             os.unlink(skipfile)
175         except Exception as e:
176            pass
177         app = "cd "+pn+"; "+binpath(env, "run-escript")+" -ov "+binpath(env,
178                "../tools/testrunner.py")+" -outputfile="+skipfile+" "+sn
179       print "Executing test: ",app
180     if env.Execute(app) == 0:     if env.Execute(app) == 0:
181        open(str(target[0]),'w').write("PASSED\n")        open(str(target[0]),'w').write("PASSED\n")
182     else:     else:
# Line 204  def runPyUnitTest(target, source, env): Line 184  def runPyUnitTest(target, source, env):
184     print "Test execution time: ", round(time.time() - time_start, 1), " seconds wall time for " + str(source[0].abspath)     print "Test execution time: ", round(time.time() - time_start, 1), " seconds wall time for " + str(source[0].abspath)
185     return None     return None
186    
187  def addBuilders(env) :  def runPyExample(target, source, env):
188      py_builder = env.Builder(action = build_py,     time_start = time.time()
189                               suffix = '.pyc',     app = str(source[0].abspath)
190                               src_suffix = '.py',     pn, sn= os.path.split(app)
191                               single_source=True)     if os.name=="nt":
192           if env['usempi']:
193      env.Append(BUILDERS = {'PyCompile' : py_builder});             app = "cd %s & mpiexec -np %s -genvlist PYTHONPATH,OMP_NUM_THREADS,"\
194                  "FINLEY_TEST_DATA,PATH %s\pythonMPIredirect.exe %s"\
195      substituter = env.Builder(action = doSubstitution,                %(pn,env['ENV']['ESCRIPT_NUM_NODES'],env['libinstall'],sn)
196                                suffix = '',         else:
197                                src_suffix = '.in',             app = "cd "+ pn +" & "+sys.executable + " " + sn
198                                single_source=True )     else:
   
     env.Append(BUILDERS = {'VariableSubstitution' : substituter});  
   
     runUnitTest_builder = env.Builder(action = runUnitTest,  
                                       suffix = '.passed',  
                                       src_suffix=env.get('PROGSUFFIX',''),  
                                       single_source=True)  
   
     env.Append(BUILDERS = {'RunUnitTest' : runUnitTest_builder});  
   
     runPyUnitTest_builder = env.Builder(action = runPyUnitTest,  
                                         suffix = '.passed',  
                                         src_suffix='.py',  
                                         single_source=True)  
   
     env.Append(BUILDERS = {'RunPyUnitTest' : runPyUnitTest_builder});  
     return  
 #==============================================================================  
 SConsEnvironment.addBuilders = addBuilders  
 ###############################################################################  
   
 ###############################################################################  
 def epydocAction(target, source, env):  
   
     doc_dir = os.path.dirname(str(target[0].abspath))  
   
     cmd = [  
         [env['epydoc_path'], "-qqqq", "-o", doc_dir, str(source[0].tpath)]  
           ]  
     print 'executing epydoc...'  
     return env.Execute(cmd,"Build epydoc documentation")  
   
   
 def epydocDepend(env, target, source,  
                  src_pattern="*.py",  
                  file_names=['index.html','epydoc.css'],  
                  subdirs=['private','public']) :  
     """  
     \brief add a dependency between the directory containing the doco and that  
            containing the source  
     \param target - the directory containing index.html for the doco,  
                     and the private and public sub-directories.  
     \param source - the python module source directory  
     """  
     the_subdirs = [os.path.join(target.abspath,sub) for sub in subdirs]  
     the_targets = [os.path.join(target.abspath,file) for file in file_names]  
   
   
     ret_target = target  
       
     # if absolutely anything goes wrong, turn this on.  
     force_build = False  
   
     dst_time = 0  
     src_time = 1  
   
     try:  
   
         # have a shot at digging out all the source file times.  
         src_time = max(  
             [os.path.getmtime(str(x))  
              for x in env.matchingFiles(source,src_pattern) +  
                       [source.abspath]]  
             )  
   
         # now try to find the target files and their mod time.  
         a = [os.path.getmtime(os.path.join(target.abspath,str(x)))  
              for x in file_names ]  
   
         for directory in the_subdirs :  
             # include the mod time of the directory  
             a += [os.path.getmtime(str(directory))]  
   
             # now go for the mod times of all files below the subdirs.  
             if os.path.isdir(str(directory)) :  
                 a += [os.path.getmtime(str(x))  
                       for x in env.matchingFiles(directory,"*")]  
   
             else:  
                 # if it is not a directory, and we expected it to be  
                 # do something nasty.  
                 # we're in a try anyway.  
                 force_build = True  
                 os.unlink(directory)  
   
         dst_time = max(a)  
               
     except:  
         # Force an unlink and re-build.  
         force_build = True  
           
     if src_time > dst_time or force_build :  
         for x in the_targets :  
             try:  
                 os.unlink(x)  
             except OSError:  
                 pass  
   
         ret_target = env.Command(the_targets,source,epydocAction)  
199            
200         app = "cd "+pn+"; pwd; "+binpath(env, "run-escript")+" -ov "+sn
201       print "Executing test: ",app
202       if env.Execute(app) == 0:
203          open(str(target[0]),'w').write("PASSED\n")
204       else:
205         return 1
206       print "Test execution time: ", round(time.time() - time_start, 1), " seconds wall time for " + str(source[0].abspath)
207       return None
208    
209      env.Clean(target, the_subdirs + file_names)  def eps2pdf(target, source, env):
210    #   if env.Execute("epstopdf "+str(source[0].abspath)+" -o "+str(target[0].abspath))!=0:
211      return ret_target     if env.Execute("ps2pdf -dEPSCrop "+str(source[0].abspath)+" "+str(target[0].abspath))!=0:
212  #==============================================================================         return 1
213  SConsEnvironment.epydocDepend = epydocDepend     return None
 ###############################################################################  
   
   
 ###############################################################################  
 def genSConscriptCalls(env,dir_list):  
   
     for d in dir_list :  
         print 'calling SConscript in "./%s"' %(d)  
         env.SConscript(dirs = [d],  
                        build_dir='build/$PLATFORM/' + str(d),  
                        duplicate=0)  
   
   
     return  
 #==============================================================================  
 SConsEnvironment.genSConscriptCalls = genSConscriptCalls  
 ###############################################################################  
   
   
 ###############################################################################  
 def print_all_nodes(env,dirnode, level=0):  
     """Print all the scons nodes that are children of this node, recursively."""  
     if type(dirnode)==type(''):  
         dirnode=env.Dir(dirnode)  
     dt = type(env.Dir('.'))  
     for f in dirnode.all_children():  
         if type(f) == dt:  
             print "%s%s: .............."%(level * ' ', str(f))  
             env.print_all_nodes(f, level+2)  
         print "%s%s"%(level * ' ', str(f))  
   
 #==============================================================================  
 SConsEnvironment.print_all_nodes = print_all_nodes  
 ###############################################################################  
     
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
 ###############################################################################  
 def Glob(env,dir='.',includes="*",excludes=None, scan_dir=True):  
   
     """Similar to glob.glob, except globs SCons nodes, and thus sees  
     generated files and files from build directories.  Basically, it sees  
     anything SCons knows about.  A key subtlety is that since this function  
     operates on generated nodes as well as source nodes on the filesystem,  
     it needs to be called after builders that generate files you want to  
     include.  
   
     It will return both Dir entries and File entries  
     """  
   
     # Pre-process for more convenient arguments  
   
     if isinstance(includes,str) :  
         includes = env.Split(includes)  
   
     if isinstance(excludes,str) :  
         excludes = env.Split(excludes)  
   
     def fn_filter(node):  
         fn = os.path.basename(str(node))  
         match = 0  
         for include in includes:  
             if fnmatch.fnmatchcase( fn, include ):  
                 match = 1  
                 break  
   
         if match == 1 and not excludes is None:  
             for exclude in excludes:  
                 if fnmatch.fnmatchcase( fn, exclude ):  
                     match = 0  
                     break  
   
         return match  
   
     def filter_nodes(where):  
         contents = where.all_children(scan=scan_dir)  
         children = [ x for x in contents if fn_filter(x) ]  
         nodes = []  
         for f in children:  
             nodes.append(gen_node(f))  
         return nodes  
   
     def gen_node(n):  
         """Checks first to see if the node is a file or a dir, then  
         creates the appropriate node. [code seems redundant, if the node  
         is a node, then shouldn't it just be left as is?  
         """  
         if type(n) in (type(''), type(u'')):  
             path = n  
         else:  
             path = n.abspath  
         if os.path.isdir(path):  
             return env.Dir(n)  
         else:  
             return env.File(n)  
   
     here = env.Dir(dir)  
     nodes = filter_nodes(here)  
   
     node_srcs = [n.srcnode() for n in nodes]  
   
     src = here.srcnode()  
     if src is not here:  
         for s in filter_nodes(src):  
             if s not in node_srcs:  
                 # Probably need to check if this node is a directory  
                 nodes.append(  
                     gen_node(os.path.join(dir,os.path.basename(str(s)))))  
214    
215      return nodes  def effectiveName(inname):
216        m=re.compile("^r1i[0-9]{1,2}n[0-9]{1,2}$")  # savanna names take the form r1i?n?
217        if m.match(inname):
218            return "savanna"
219        return inname

Legend:
Removed from v.1384  
changed lines
  Added in v.5045

  ViewVC Help
Powered by ViewVC 1.1.26