/[escript]/trunk/site_scons/site_init.py
ViewVC logotype

Diff of /trunk/site_scons/site_init.py

Parent Directory Parent Directory | Revision Log Revision Log | View Patch Patch

trunk/scons/scons_extensions.py revision 1240 by phornby, Mon Aug 13 06:39:29 2007 UTC trunk/site_scons/site_init.py revision 4300 by caltinay, Mon Mar 11 00:50:19 2013 UTC
# Line 1  Line 1 
1    
2    ##############################################################################
3  # Extensions to Scons  #
4    # Copyright (c) 2003-2013 by University of Queensland
5  import py_compile  # http://www.uq.edu.au
6  import sys  #
7  import os  # Primary Business: Queensland, Australia
8  import glob  # Licensed under the Open Software License version 3.0
9  import fnmatch  # http://www.opensource.org/licenses/osl-3.0.php
10  import types  #
11    # Development until 2012 by Earth Systems Science Computational Center (ESSCC)
12  from SCons.Script.SConscript import SConsEnvironment  # Development since 2012 by School of Earth Sciences
13    #
14  ###############################################################################  ##############################################################################
15  def matchingFiles(env,directory,includes='*',excludes=None) :  
16    __copyright__="""Copyright (c) 2003-2013 by University of Queensland
17      # Pre-process for more convenient arguments  http://www.uq.edu.au
18    Primary Business: Queensland, Australia"""
19      if isinstance(includes,str) :  __license__="""Licensed under the Open Software License version 3.0
20          includes = env.Split(includes)  http://www.opensource.org/licenses/osl-3.0.php"""
21    __url__="https://launchpad.net/escript-finley"
22      if isinstance(excludes,str) :  
23          excludes = env.Split(excludes)  import sys, os, time, py_compile, re, subprocess
24    from SCons.Defaults import Chmod
25      def fn_filter(node):  from grouptest import *
26          fn = os.path.basename(str(node))  
27          match = False  def findLibWithHeader(env, libs, header, paths, lang='c'):
28          for include in includes:      from SCons.Script.SConscript import Configure
29              if fnmatch.fnmatchcase( fn, include ):      inc_path=''
30                  match = True      lib_path=''
31        # 'paths' may be a prefix, so look for lib and include subdirectories
32        if type(paths)==str:
33            # find the header file first
34            for i in 'include','include64','include32','inc':
35                inc=os.path.join(paths, i)
36                if os.path.isfile(os.path.join(inc, header)):
37                    inc_path=inc
38                  break                  break
39            if inc_path=='':
40                raise RuntimeError('%s not found under %s'%(header,paths))
41    
42          if match and not excludes is None:          # now try to find a lib directory
43              for exclude in excludes:          for l in 'lib','lib64','lib32':
44                  if fnmatch.fnmatchcase( fn, exclude ):              lp=os.path.join(paths, l)
45                      match = False              if os.path.isdir(lp):
46                      break                  lib_path=lp
47                    break
48          return match          if lib_path=='':
49                raise RuntimeError('No lib directory found under %s'%paths)
50      def filter_nodes(where):      else:
51          contents = glob.glob( os.path.join(str(where),"*") )          if os.path.isfile(os.path.join(paths[0], header)):
52          children = [ x for x in contents if fn_filter(x) ]              inc_path=paths[0]
         nodes = []  
         for f in children:  
             nodes.append(gen_node(f))  
         return nodes  
   
     def gen_node(n):  
         """Checks first to see if the node is a file or a dir, then  
         creates the appropriate node. [code seems redundant, if the node  
         is a node, then shouldn't it just be left as is?  
         """  
         if type(n) in (type(''), type(u'')):  
             path = n  
53          else:          else:
54              path = n.abspath              raise RuntimeError('%s not found under %s'%(header,paths[0]))
55            if os.path.isdir(paths[1]):
56          if os.path.isdir(path):              lib_path=paths[1]
             return env.Dir(n)  
   
         return env.File(n)  
   
     here = env.Dir(directory)  
     nodes = filter_nodes(here)  
   
     node_srcs = [n.srcnode() for n in nodes]  
   
     src = here.srcnode()  
     if src is not here:  
         for s in filter_nodes(src):  
             if s not in node_srcs:  
                 # Probably need to check if this node is a directory  
                 nodes.append(  
                     gen_node(os.path.join(str(directory),  
                                           os.path.basename(str(s)))))  
   
     return nodes  
 #==============================================================================  
 SConsEnvironment.matchingFiles = matchingFiles  
 ###############################################################################  
   
   
   
 ###############################################################################  
 def InstallPyModule(env,target_dir,source,shlib=None,excludes=None) :  
   
     # put the .so/.dll over in  <target_dir>  
     if shlib :  
         shtarg = env.Install(target_dir, shlib)  
   
     # Gather the python sources  
     python_src = env.matchingFiles(source, "*.py",excludes=excludes)  
   
     # Here is a hack to deal with the (possibly not yet generated)  
     # __init__.py. The problem is that the python_src list is built before  
     # __init__.py is updated from __init__.in. The AlwaysBuild call  
     # ensures that this does not cause a problem, except on the first  
     # build after a clean checkout, in which case there is no old  
     # __init__.py in src, and hence it does not make it into python_src!  
   
     init_input = env.matchingFiles(source, "__init__.in")  
     if init_input :  
         if python_src :  
             names = [x.name for x in python_src]  
             if "__init__.py" not in names :  
                 python_src.append(env.File(os.path.join(str(source),  
                                                         "__init__.py")))  
57          else:          else:
58              python_src = [env.File(os.path.join(str(source),"__init__.py"))]              raise RuntimeError('%s is not a valid path.'%paths[1])
               
     # decide if we're doing py or pyc distribn and install.  
59    
60      if  env['distrib_py_src'] :      # now try the library
61          pytarg = env.Install(target_dir, python_src)      conf=Configure(env.Clone())
62      else:      conf.env.AppendUnique(CPPPATH = [inc_path])
63          pyc = env.PyCompile(python_src)      conf.env.AppendUnique(LIBPATH = [lib_path])
64          pytarg = env.Install(target_dir, pyc)      if type(libs)==str: libs=[libs]
65        # we can't check for each library by itself since they may depend on each
66      if shlib :      # other, so we add all libraries to the link line and check only for one
67          targ_ret = env.Flatten([pytarg] + [shtarg])      conf.env.AppendUnique(LIBS = libs)
68      else:      if not conf.CheckLibWithHeader(libs[0], header, lang):
69          targ_ret = pytarg          conf.Finish()
70            raise RuntimeError('Unable to link against %s (paths: %s, %s)'%(libs,inc_path,lib_path))
71    
72        conf.Finish()
73        return inc_path, lib_path
74    
75    def detectModule(env, module):
76        p=subprocess.call([env['pythoncmd'],'-c','import %s'%module])
77        if p != 0:
78            env[module] = False
79            return False
80        env[module] = True
81        return True
82    
83    def write_buildvars(env):
84        buildvars=open(os.path.join(env['libinstall'], 'buildvars'), 'w')
85        for k,v in sorted(env['buildvars'].items()):
86            buildvars.write("%s=%s\n"%(k,v))
87        buildvars.close()
88    
89      return targ_ret  def generateTestScripts(env, TestGroups):
90  #==============================================================================      try:
91  SConsEnvironment.InstallPyModule = InstallPyModule          utest=open('utest.sh','w')
92  ###############################################################################          utest.write(GroupTest.makeHeader(env['PLATFORM'], env['prefix'], False))
93            for tests in TestGroups:
94  ###############################################################################              utest.write(tests.makeString())
95  def installDirectory(env,target,source,includes="*", excludes=None,          utest.close()
96                       recursive=False):          env.Execute(Chmod('utest.sh', 0o755))
97                print("Generated utest.sh.")
98                # This version contains only python tests - I want this to be usable
99      if os.path.isfile(str(target)) :          # from a binary only install if you have the test files
100          raise UserError("target must be a directory")          utest=open('itest.sh','w')
101            utest.write(GroupTest.makeHeader(env['PLATFORM'], env['prefix'], True))
102      if os.path.isfile(str(source)) :          for tests in TestGroups:
103          raise UserError("source must be a directory")            if tests.exec_cmd=='$PYTHONRUNNER ':
104                utest.write(tests.makeString())
105      source_files = env.matchingFiles(source,includes,excludes)          utest.close()
106            env.Execute(Chmod('itest.sh', 0o755))
107      ret_targ = []          print("Generated itest.sh.")        
108        except IOError:
109      for f in source_files :          env['warnings'].append("Error attempting to write unit test script(s).")
110          if f.isfile() :  
111              targ = env.Install(target,f)      # delete scripts upon cleanup
112              ret_targ.append(targ)      env.Clean('target_init', 'utest.sh')
113        env.Clean('target_init', 'itest.sh')
114          if f.isdir() and recursive :  
115              x = os.path.basename(str(f))      # Make sure that the escript wrapper is in place
116              t = env.Dir(os.path.join(str(target),x))      if not os.path.isfile(os.path.join(env['bininstall'], 'run-escript')):
117              targ = env.installDirectory(t,f,includes,excludes,recursive)          print("Copying escript wrapper.")
118              ret_targ += targ          Execute(Copy(os.path.join(env['bininstall'],'run-escript'), 'bin/run-escript'))
       
     return ret_targ  
 #==============================================================================  
 SConsEnvironment.installDirectory = installDirectory  
 ###############################################################################  
119    
 ###############################################################################  
120  # Code to build .pyc from .py  # Code to build .pyc from .py
121  def build_py(target, source, env):  def build_py(target, source, env):
122      py_compile.compile(str(source[0]), str(target[0]))      try:
123      return 0         py_compile.compile(str(source[0]), str(target[0]), doraise=True)
124           return 0
125        except py_compile.PyCompileError, e:
126           print e
127           return 1
128    
 def doSubstitution(target,source,env) :  
     import product_info as PI  
     data = source[0].get_contents()  
     data = data.replace('$ProductName$',PI.PRODUCT_NAME)  
     data = data.replace('$LowerProductName$',PI.product_name)  
     data = data.replace('$ProductVersion$',PI.product_version)  
     data = data.replace('$VersionString$',PI.pkg_version_string)  
     data = data.replace('$SVNRevision$',PI.svn_revision)  
     open(str(target[0]),'w').write(data)  
     return 0  
129    
130  # Code to run unit_test executables  # Code to run unit_test executables
131  def runUnitTest(target, source, env):  def runUnitTest(target, source, env):
132      app = str(source[0].abspath)    time_start = time.time()
133      app = str(source[0].abspath)
134      olddir = os.getcwd()    pn, sn= os.path.split(app)
135      newdir = os.path.dirname(str(source[0]))    if not os.name== "nt":
136      os.chdir(newdir)       app = "cd "+pn+"; "+os.path.join(env['bininstall'], "run-escript")+" -bv "+os.path.join('.',sn)
137      else:
138      if env.Execute(app) != 0:        if env['usempi']:
139          os.chdir(olddir)            app = "cd %s & mpiexec -np %s -genvlist PYTHONPATH,OMP_NUM_THREADS,"\
140          return 1              "FINLEY_TEST_DATA,PYVISI_TEST_DATA_ROOT,PYVISI_WORKDIR,PATH %s"\
141                %(pn,env['ENV']['ESCRIPT_NUM_NODES'], sn)
142      os.chdir(olddir)        else:
143               app = "cd "+ pn +" & "+sn
144      print "Executing test: " + app
145      if not env.Execute(app):
146      open(str(target[0]),'w').write("PASSED\n")      open(str(target[0]),'w').write("PASSED\n")
147      return 0    else:
148        return 1
149      print "Test execution time: ", round(time.time() - time_start, 1), " seconds wall time for " + str(source[0].abspath)
150      return None
151    
152  def runPyUnitTest(target, source, env):  def runPyUnitTest(target, source, env):
153      app = env['python_cmd'] + ' "' + str(source[0].abspath) + '"'     time_start = time.time()
154       app = str(source[0].abspath)
155      olddir = os.getcwd()     pn, sn= os.path.split(app)
156      newdir = os.path.dirname(str(source[0]))     if os.name== "nt":
157      os.chdir(newdir)         if env['usempi']:
158               app = "cd %s & mpiexec -np %s -genvlist PYTHONPATH,OMP_NUM_THREADS,"\
159      if env.Execute(app)  != 0:                "FINLEY_TEST_DATA,PYVISI_TEST_DATA_ROOT,PYVISI_WORKDIR,PATH %s\pythonMPIredirect.exe %s"\
160          os.chdir(olddir)                %(pn,env['ENV']['ESCRIPT_NUM_NODES'],env['libinstall'],sn)
161          return 1         else:
162               app = "cd "+ pn +" & "+sys.executable + " " + sn
163      os.chdir(olddir)     else:
164      open(str(target[0]),'w').write("PASSED\n")       app = "cd "+pn+"; "+os.path.join(env['bininstall'], "run-escript")+" -ov "+sn
165      return 0     print "Executing test: ",app
166         if env.Execute(app) == 0:
167          open(str(target[0]),'w').write("PASSED\n")
168  def addBuilders(env) :     else:
169      py_builder = env.Builder(action = build_py,       return 1
170                               suffix = '.pyc',     print "Test execution time: ", round(time.time() - time_start, 1), " seconds wall time for " + str(source[0].abspath)
171                               src_suffix = '.py',     return None
172                               single_source=True)  
173    def eps2pdf(target, source, env):
174      env.Append(BUILDERS = {'PyCompile' : py_builder});  #   if env.Execute("epstopdf "+str(source[0].abspath)+" -o "+str(target[0].abspath))!=0:
175       if env.Execute("ps2pdf -dEPSCrop "+str(source[0].abspath)+" "+str(target[0].abspath))!=0:
176      substituter = env.Builder(action = doSubstitution,         return 1
177                                suffix = '',     return None
178                                src_suffix = '.in',  
179                                single_source=True )  def effectiveName(inname):
180       m=re.compile("^r1i[0-9]{1,2}n[0-9]{1,2}$")   # savanna names take the form r1i?n?
181      env.Append(BUILDERS = {'VariableSubstitution' : substituter});     if m.match(inname):
182        return "savanna"
183      runUnitTest_builder = env.Builder(action = runUnitTest,     return inname
                                       suffix = '.passed',  
                                       src_suffix=env.get('PROGSUFFIX',''),  
                                       single_source=True)  
   
     env.Append(BUILDERS = {'RunUnitTest' : runUnitTest_builder});  
   
     runPyUnitTest_builder = env.Builder(action = runPyUnitTest,  
                                         suffix = '.passed',  
                                         src_suffix='.py',  
                                         single_source=True)  
   
     env.Append(BUILDERS = {'RunPyUnitTest' : runPyUnitTest_builder});  
     return  
 #==============================================================================  
 SConsEnvironment.addBuilders = addBuilders  
 ###############################################################################  
   
 ###############################################################################  
 def epydocAction(target, source, env):  
   
     doc_dir = os.path.dirname(str(target[0].abspath))  
   
     cmd = [  
         [env['epydoc_path'], "-qqqq", "-o", doc_dir, str(source[0].tpath)]  
           ]  
     print 'executing epydoc...'  
     return env.Execute(cmd,"Build epydoc documentation")  
   
   
 def epydocDepend(env, target, source,  
                  src_pattern="*.py",  
                  file_names=['index.html','epydoc.css'],  
                  subdirs=['private','public']) :  
     """  
     \brief add a dependency between the directory containing the doco and that  
            containing the source  
     \param target - the directory containing index.html for the doco,  
                     and the private and public sub-directories.  
     \param source - the python module source directory  
     """  
     the_subdirs = [os.path.join(target.abspath,sub) for sub in subdirs]  
     the_targets = [os.path.join(target.abspath,file) for file in file_names]  
   
   
     ret_target = target  
       
     # if absolutely anything goes wrong, turn this on.  
     force_build = False  
   
     dst_time = 0  
     src_time = 1  
   
     try:  
   
         # have a shot at digging out all the source file times.  
         src_time = max(  
             [os.path.getmtime(str(x))  
              for x in env.matchingFiles(source,src_pattern) +  
                       [source.abspath]]  
             )  
   
         # now try to find the target files and their mod time.  
         a = [os.path.getmtime(os.path.join(target.abspath,str(x)))  
              for x in file_names ]  
   
         for directory in the_subdirs :  
             # include the mod time of the directory  
             a += [os.path.getmtime(str(directory))]  
   
             # now go for the mod times of all files below the subdirs.  
             if os.path.isdir(str(directory)) :  
                 a += [os.path.getmtime(str(x))  
                       for x in env.matchingFiles(directory,"*")]  
   
             else:  
                 # if it is not a directory, and we expected it to be  
                 # do something nasty.  
                 # we're in a try anyway.  
                 force_build = True  
                 os.unlink(directory)  
   
         dst_time = max(a)  
               
     except:  
         # Force an unlink and re-build.  
         force_build = True  
           
     if src_time > dst_time or force_build :  
         for x in the_targets :  
             try:  
                 os.unlink(x)  
             except OSError:  
                 pass  
   
         ret_target = env.Command(the_targets,source,epydocAction)  
       
   
     env.Clean(target, the_subdirs + file_names)  
   
     return ret_target  
 #==============================================================================  
 SConsEnvironment.epydocDepend = epydocDepend  
 ###############################################################################  
   
   
 ###############################################################################  
 def genSConscriptCalls(env,dir_list):  
   
     for d in dir_list :  
         print 'calling SConscript in "./%s"' %(d)  
         env.SConscript(dirs = [d],  
                        build_dir='build/$PLATFORM/' + str(d),  
                        duplicate=0)  
   
   
     return  
 #==============================================================================  
 SConsEnvironment.genSConscriptCalls = genSConscriptCalls  
 ###############################################################################  
   
   
 ###############################################################################  
 def print_all_nodes(env,dirnode, level=0):  
     """Print all the scons nodes that are children of this node, recursively."""  
     if type(dirnode)==type(''):  
         dirnode=env.Dir(dirnode)  
     dt = type(env.Dir('.'))  
     for f in dirnode.all_children():  
         if type(f) == dt:  
             print "%s%s: .............."%(level * ' ', str(f))  
             env.print_all_nodes(f, level+2)  
         print "%s%s"%(level * ' ', str(f))  
   
 #==============================================================================  
 SConsEnvironment.print_all_nodes = print_all_nodes  
 ###############################################################################  
     
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
 ###############################################################################  
 def Glob(env,dir='.',includes="*",excludes=None, scan_dir=True):  
   
     """Similar to glob.glob, except globs SCons nodes, and thus sees  
     generated files and files from build directories.  Basically, it sees  
     anything SCons knows about.  A key subtlety is that since this function  
     operates on generated nodes as well as source nodes on the filesystem,  
     it needs to be called after builders that generate files you want to  
     include.  
   
     It will return both Dir entries and File entries  
     """  
   
     # Pre-process for more convenient arguments  
   
     if isinstance(includes,str) :  
         includes = env.Split(includes)  
   
     if isinstance(excludes,str) :  
         excludes = env.Split(excludes)  
   
     def fn_filter(node):  
         fn = os.path.basename(str(node))  
         match = 0  
         for include in includes:  
             if fnmatch.fnmatchcase( fn, include ):  
                 match = 1  
                 break  
   
         if match == 1 and not excludes is None:  
             for exclude in excludes:  
                 if fnmatch.fnmatchcase( fn, exclude ):  
                     match = 0  
                     break  
   
         return match  
   
     def filter_nodes(where):  
         contents = where.all_children(scan=scan_dir)  
         children = [ x for x in contents if fn_filter(x) ]  
         nodes = []  
         for f in children:  
             nodes.append(gen_node(f))  
         return nodes  
   
     def gen_node(n):  
         """Checks first to see if the node is a file or a dir, then  
         creates the appropriate node. [code seems redundant, if the node  
         is a node, then shouldn't it just be left as is?  
         """  
         if type(n) in (type(''), type(u'')):  
             path = n  
         else:  
             path = n.abspath  
         if os.path.isdir(path):  
             return env.Dir(n)  
         else:  
             return env.File(n)  
   
     here = env.Dir(dir)  
     nodes = filter_nodes(here)  
   
     node_srcs = [n.srcnode() for n in nodes]  
   
     src = here.srcnode()  
     if src is not here:  
         for s in filter_nodes(src):  
             if s not in node_srcs:  
                 # Probably need to check if this node is a directory  
                 nodes.append(  
                     gen_node(os.path.join(dir,os.path.basename(str(s)))))  
   
     return nodes  

Legend:
Removed from v.1240  
changed lines
  Added in v.4300

  ViewVC Help
Powered by ViewVC 1.1.26