/[escript]/branches/4.0fordebian/site_scons/site_init.py
ViewVC logotype

Diff of /branches/4.0fordebian/site_scons/site_init.py

Parent Directory Parent Directory | Revision Log Revision Log | View Patch Patch

trunk/scons/scons_extensions.py revision 1233 by phornby, Mon Aug 13 00:20:22 2007 UTC branches/4.0fordebian/site_scons/site_init.py revision 5577 by jfenwick, Fri Apr 3 05:26:15 2015 UTC
# Line 1  Line 1 
1    
2    ##############################################################################
3  # Extensions to Scons  #
4    # Copyright (c) 2003-2014 by University of Queensland
5  import py_compile  # http://www.uq.edu.au
6  import sys  #
7  import os  # Primary Business: Queensland, Australia
8  import glob  # Licensed under the Open Software License version 3.0
9  import fnmatch  # http://www.opensource.org/licenses/osl-3.0.php
10  import types  #
11    # Development until 2012 by Earth Systems Science Computational Center (ESSCC)
12  from SCons.Script.SConscript import SConsEnvironment  # Development 2012-2013 by School of Earth Sciences
13    # Development from 2014 by Centre for Geoscience Computing (GeoComp)
14  ###############################################################################  #
15  def matchingFiles(env,directory,includes='*',excludes=None) :  ##############################################################################
16    
17      # Pre-process for more convenient arguments  __copyright__="""Copyright (c) 2003-2014 by University of Queensland
18    http://www.uq.edu.au
19      if isinstance(includes,str) :  Primary Business: Queensland, Australia"""
20          includes = env.Split(includes)  __license__="""Licensed under the Open Software License version 3.0
21    http://www.opensource.org/licenses/osl-3.0.php"""
22      if isinstance(excludes,str) :  __url__="https://launchpad.net/escript-finley"
23          excludes = env.Split(excludes)  
24    import sys, os, time, py_compile, re, subprocess
25      def fn_filter(node):  from SCons.Defaults import Chmod, Copy
26          fn = os.path.basename(str(node))  from grouptest import *
27          match = False  from extractdebbuild import *
28          for include in includes:  
29              if fnmatch.fnmatchcase( fn, include ):  def findLibWithHeader(env, libs, header, paths, lang='c++'):
30                  match = True      from SCons.Script.SConscript import Configure
31        inc_path=''
32        lib_path=''
33        # 'paths' may be a prefix, so look for lib and include subdirectories
34        if type(paths)==str:
35            # find the header file first
36            for i in 'include','include64','include32','inc':
37                inc=os.path.join(paths, i)
38                if os.path.isfile(os.path.join(inc, header)):
39                    inc_path=inc
40                  break                  break
41            if inc_path=='':
42                raise RuntimeError('%s not found under %s'%(header,paths))
43    
44          if match and not excludes is None:          # now try to find a lib directory
45              for exclude in excludes:          for l in 'lib','lib64','lib32':
46                  if fnmatch.fnmatchcase( fn, exclude ):              lp=os.path.join(paths, l)
47                      match = False              if os.path.isdir(lp):
48                      break                  lib_path=lp
49                    break
50          return match          if lib_path=='':
51                raise RuntimeError('No lib directory found under %s'%paths)
52      def filter_nodes(where):      else:
53          contents = glob.glob( os.path.join(str(where),"*") )          if os.path.isfile(os.path.join(paths[0], header)):
54          children = [ x for x in contents if fn_filter(x) ]              inc_path=paths[0]
         nodes = []  
         for f in children:  
             nodes.append(gen_node(f))  
         return nodes  
   
     def gen_node(n):  
         """Checks first to see if the node is a file or a dir, then  
         creates the appropriate node. [code seems redundant, if the node  
         is a node, then shouldn't it just be left as is?  
         """  
         if type(n) in (type(''), type(u'')):  
             path = n  
55          else:          else:
56              path = n.abspath              raise RuntimeError('%s not found under %s'%(header,paths[0]))
57            if os.path.isdir(paths[1]):
58          if os.path.isdir(path):              lib_path=paths[1]
             return env.Dir(n)  
   
         return env.File(n)  
   
     here = env.Dir(directory)  
     nodes = filter_nodes(here)  
   
     node_srcs = [n.srcnode() for n in nodes]  
   
     src = here.srcnode()  
     if src is not here:  
         for s in filter_nodes(src):  
             if s not in node_srcs:  
                 # Probably need to check if this node is a directory  
                 nodes.append(  
                     gen_node(os.path.join(str(directory),  
                                           os.path.basename(str(s)))))  
   
     return nodes  
 #==============================================================================  
 SConsEnvironment.matchingFiles = matchingFiles  
 ###############################################################################  
   
   
   
 ###############################################################################  
 def InstallPyModule(env,target_dir,source,shlib=None,excludes=None) :  
   
     # put the .so/.dll over in  <target_dir>  
     if shlib :  
         shtarg = env.Install(target_dir, shlib)  
   
     # Gather the python sources  
     python_src = env.matchingFiles(source, "*.py",excludes=excludes)  
   
     # Here is a hack to deal with the (possibly not yet generated)  
     # __init__.py. The problem is that the python_src list is built before  
     # __init__.py is updated from __init__.in. The AlwaysBuild call  
     # ensures that this does not cause a problem, except on the first  
     # build after a clean checkout, in which case there is no old  
     # __init__.py in src, and hence it does not make it into python_src!  
   
     init_input = env.matchingFiles(source, "__init__.in")  
     if init_input :  
         if python_src :  
             names = [x.name for x in python_src]  
             if "__init__.py" not in names :  
                 python_src.append(env.File(os.path.join(str(source),  
                                                         "__init__.py")))  
59          else:          else:
60              python_src = [env.File(os.path.join(str(source),"__init__.py"))]              raise RuntimeError('%s is not a valid path.'%paths[1])
               
     # decide if we're doing py or pyc distribn and install.  
61    
62      if  env['distrib_py_src'] :      # now try the library
63          pytarg = env.Install(target_dir, python_src)      conf=Configure(env.Clone())
64      else:      conf.env.AppendUnique(CPPPATH = [inc_path])
65          pyc = env.PyCompile(python_src)      conf.env.AppendUnique(LIBPATH = [lib_path])
66          pytarg = env.Install(target_dir, pyc)      if type(libs)==str: libs=[libs]
67        if len(libs)==0: libs=['']
68      if shlib :      # we can't check for each library by itself since they may depend on each
69          targ_ret = env.Flatten([pytarg] + [shtarg])      # other, so we add all libraries to the link line and check only for one
70      else:      conf.env.AppendUnique(LIBS = libs)
71          targ_ret = pytarg      if not conf.CheckLibWithHeader(libs[0], header, lang):
72            conf.Finish()
73      return targ_ret          raise RuntimeError('Unable to link against %s (paths: %s, %s)'%(libs,inc_path,lib_path))
74  #==============================================================================  
75  SConsEnvironment.InstallPyModule = InstallPyModule      conf.Finish()
76  ###############################################################################      return inc_path, lib_path
77    
78  ###############################################################################  def detectModule(env, module):
79  def installDirectory(env,target,source,includes="*", excludes=None,      from tempfile import TemporaryFile
80                       recursive=False):      p=subprocess.call([env['pythoncmd'],'-c','import %s'%module], stderr=TemporaryFile())
81            if p != 0:
82                env[module] = False
83      if os.path.isfile(str(target)) :          return False
84          raise UserError("target must be a directory")      env[module] = True
85        return True
86      if os.path.isfile(str(source)) :  
87          raise UserError("source must be a directory")  def write_buildvars(env):
88        buildvars=open(os.path.join(env['libinstall'], 'buildvars'), 'w')
89      source_files = env.matchingFiles(source,includes,excludes)      for k,v in sorted(env['buildvars'].items()):
90            buildvars.write("%s=%s\n"%(k,v))
91        buildvars.close()
92    
93    def write_launcher(env):
94        reps={'%n':'${ESCRIPT_NUM_NODES}', '%p':'${ESCRIPT_NUM_PROCS}',
95              '%N':'${TOTPROC}', '%t':'${ESCRIPT_NUM_THREADS}', '%f':'${HOSTFILE}',
96              '%h':'${HOSTLIST}', '%e':'${EXPORT_ENV}', '%b':'${EXEC_CMD}'}
97        pre=env['prelaunch']
98        cmd=env['launcher']
99        post=env['postlaunch']
100        # %b should be present in launcher at least
101        if not '%b' in cmd:
102            raise RuntimeError('option "launcher" must contain %b!')
103    
104        for k, v in reps.iteritems():
105            pre = pre.replace(k, v)
106            cmd = cmd.replace(k, v)
107            post = post.replace(k, v)
108        try:
109            launchscript = os.path.join(env['bininstall'], 'run-escript')
110            launcher=open(launchscript, 'w')
111            for line in open('run-escript.in','r').readlines():
112                launcher.write(line.replace('@@PRELAUNCH', pre).replace('@@LAUNCH', cmd).replace('@@POSTLAUNCH', post))
113            launcher.close()
114            env.Execute(Chmod(launchscript, 0o755))
115        except IOError:
116            env['warnings'].append("Error attempting to write launcher script.")
117    
118      ret_targ = []  def generateTestScripts(env, TestGroups):
119        try:
120      for f in source_files :          utest=open('utest.sh','w')
121          if f.isfile() :          utest.write(GroupTest.makeHeader(env['PLATFORM'], env['prefix'], False))
122              targ = env.Install(target,f)          for tests in TestGroups:
123              ret_targ.append(targ)              utest.write(tests.makeString())
124            utest.close()
125          if f.isdir() and recursive :          env.Execute(Chmod('utest.sh', 0o755))
126              x = os.path.basename(str(f))          print("Generated utest.sh.")
127              t = env.Dir(os.path.join(str(target),x))          # This version contains only python tests - I want this to be usable
128              targ = env.installDirectory(t,f,includes,excludes,recursive)          # from a binary only install if you have the test files
129              ret_targ += targ          utest=open('itest.sh','w')
130                utest.write(GroupTest.makeHeader(env['PLATFORM'], env['prefix'], True))
131      return ret_targ          for tests in TestGroups:
132  #==============================================================================            if tests.exec_cmd=='$PYTHONRUNNER ':
133  SConsEnvironment.installDirectory = installDirectory              utest.write(tests.makeString())
134  ###############################################################################          utest.close()
135            env.Execute(Chmod('itest.sh', 0o755))
136            print("Generated itest.sh.")        
137        except IOError:
138            env['warnings'].append("Error attempting to write unit test script(s).")
139    
140        # delete scripts upon cleanup
141        env.Clean('target_init', 'utest.sh')
142        env.Clean('target_init', 'itest.sh')
143    
 ###############################################################################  
144  # Code to build .pyc from .py  # Code to build .pyc from .py
145  def build_py(target, source, env):  def build_py(target, source, env):
146      py_compile.compile(str(source[0]), str(target[0]))      try:
147      return 0         py_compile.compile(str(source[0]), str(target[0]), doraise=True)
148           return 0
149        except py_compile.PyCompileError, e:
150           print e
151           return 1
152    
 def doSubstitution(target,source,env) :  
     import product_info as PI  
     data = source[0].get_contents()  
     data = data.replace('$ProductName$',PI.PRODUCT_NAME)  
     data = data.replace('$LowerProductName$',PI.product_name)  
     data = data.replace('$ProductVersion$',PI.product_version)  
     data = data.replace('$VersionString$',PI.pkg_version_string)  
     data = data.replace('$SVNRevision$',PI.svn_revision)  
     open(str(target[0]),'w').write(data)  
     return 0  
153    
154  # Code to run unit_test executables  # Code to run unit_test executables
155  def runUnitTest(target, source, env):  def runUnitTest(target, source, env):
156      app = str(source[0].abspath)    time_start = time.time()
157      app = str(source[0].abspath)
158      olddir = os.getcwd()    pn, sn= os.path.split(app)
159      newdir = os.path.dirname(str(source[0]))    if not os.name== "nt":
160      os.chdir(newdir)       app = "cd "+pn+"; "+os.path.join(env['bininstall'], "run-escript")+" -bv "+os.path.join('.',sn)
161      else:
162      if env.Execute(app) != 0:        if env['usempi']:
163          os.chdir(olddir)            app = "cd %s & mpiexec -np %s -genvlist PYTHONPATH,OMP_NUM_THREADS,"\
164          return 1              "FINLEY_TEST_DATA,PATH %s"\
165                %(pn,env['ENV']['ESCRIPT_NUM_NODES'], sn)
166      os.chdir(olddir)        else:
167               app = "cd "+ pn +" & "+sn
168      print "Executing test: " + app
169      if not env.Execute(app):
170      open(str(target[0]),'w').write("PASSED\n")      open(str(target[0]),'w').write("PASSED\n")
171      return 0    else:
172        return 1
173      print "Test execution time: ", round(time.time() - time_start, 1), " seconds wall time for " + str(source[0].abspath)
174      return None
175    
176    def binpath(env, name=None):
177        if not name:
178            return env['bininstall']
179        return os.path.join(env['bininstall'], name)
180    
181  def runPyUnitTest(target, source, env):  def runPyUnitTest(target, source, env):
182      app = env['python_path'] + ' "' + str(source[0].abspath) + '"'     time_start = time.time()
183       app = str(source[0].abspath)
184      olddir = os.getcwd()     pn, sn= os.path.split(app)
185      newdir = os.path.dirname(str(source[0]))     if os.name=="nt":
186      os.chdir(newdir)         if env['usempi']:
187               app = "cd %s & mpiexec -np %s -genvlist PYTHONPATH,OMP_NUM_THREADS,"\
188      if env.Execute(app)  != 0:                "FINLEY_TEST_DATA,PATH %s\pythonMPIredirect.exe %s"\
189          os.chdir(olddir)                %(pn,env['ENV']['ESCRIPT_NUM_NODES'],env['libinstall'],sn)
190          return 1         else:
191               app = "cd "+ pn +" & "+sys.executable + " " + sn
192      os.chdir(olddir)     else:
193      open(str(target[0]),'w').write("PASSED\n")       skipfile = os.path.join(env['BUILD_DIR'], sn[:-3]) + ".skipped"
194      return 0       try:
195               os.unlink(skipfile)
196         except Exception as e:
197  def addBuilders(env) :          pass
198      py_builder = env.Builder(action = build_py,       app = "cd "+pn+"; "+binpath(env, "run-escript")+" -ov "+binpath(env,
199                               suffix = '.pyc',              "../tools/testrunner.py")+" -outputfile="+skipfile+" "+sn
200                               src_suffix = '.py',     print "Executing test: ",app
201                               single_source=True)     if env.Execute(app) == 0:
202          open(str(target[0]),'w').write("PASSED\n")
203      env.Append(BUILDERS = {'PyCompile' : py_builder});     else:
204         return 1
205      substituter = env.Builder(action = doSubstitution,     print "Test execution time: ", round(time.time() - time_start, 1), " seconds wall time for " + str(source[0].abspath)
206                                suffix = '',     return None
207                                src_suffix = '.in',  
208                                single_source=True )  def runPyExample(target, source, env):
209       time_start = time.time()
210      env.Append(BUILDERS = {'VariableSubstitution' : substituter});     app = str(source[0].abspath)
211       pn, sn= os.path.split(app)
212      runUnitTest_builder = env.Builder(action = runUnitTest,     if os.name=="nt":
213                                        suffix = '.passed',         if env['usempi']:
214                                        src_suffix=env.get('PROGSUFFIX',''),             app = "cd %s & mpiexec -np %s -genvlist PYTHONPATH,OMP_NUM_THREADS,"\
215                                        single_source=True)                "FINLEY_TEST_DATA,PATH %s\pythonMPIredirect.exe %s"\
216                  %(pn,env['ENV']['ESCRIPT_NUM_NODES'],env['libinstall'],sn)
217      env.Append(BUILDERS = {'RunUnitTest' : runUnitTest_builder});         else:
218               app = "cd "+ pn +" & "+sys.executable + " " + sn
219      runPyUnitTest_builder = env.Builder(action = runPyUnitTest,     else:
                                         suffix = '.passed',  
                                         src_suffix='.py',  
                                         single_source=True)  
   
     env.Append(BUILDERS = {'RunPyUnitTest' : runPyUnitTest_builder});  
     return  
 #==============================================================================  
 SConsEnvironment.addBuilders = addBuilders  
 ###############################################################################  
   
 ###############################################################################  
 def epydocAction(target, source, env):  
   
     doc_dir = os.path.dirname(str(target[0].abspath))  
   
     cmd = [  
         [env['epydoc_path'], "-qqqq", "-o", doc_dir, str(source[0].tpath)]  
           ]  
     print 'executing epydoc...'  
     return env.Execute(cmd,"Build epydoc documentation")  
   
   
 def epydocDepend(env, target, source,  
                  src_pattern="*.py",  
                  file_names=['index.html','epydoc.css'],  
                  subdirs=['private','public']) :  
     """  
     \brief add a dependency between the directory containing the doco and that  
            containing the source  
     \param target - the directory containing index.html for the doco,  
                     and the private and public sub-directories.  
     \param source - the python module source directory  
     """  
     the_subdirs = [os.path.join(target.abspath,sub) for sub in subdirs]  
     the_targets = [os.path.join(target.abspath,file) for file in file_names]  
   
   
     ret_target = target  
       
     # if absolutely anything goes wrong, turn this on.  
     force_build = False  
   
     dst_time = 0  
     src_time = 1  
   
     try:  
   
         # have a shot at digging out all the source file times.  
         src_time = max(  
             [os.path.getmtime(str(x))  
              for x in env.matchingFiles(source,src_pattern) +  
                       [source.abspath]]  
             )  
   
         # now try to find the target files and their mod time.  
         a = [os.path.getmtime(os.path.join(target.abspath,str(x)))  
              for x in file_names ]  
   
         for directory in the_subdirs :  
             # include the mod time of the directory  
             a += [os.path.getmtime(str(directory))]  
   
             # now go for the mod times of all files below the subdirs.  
             if os.path.isdir(str(directory)) :  
                 a += [os.path.getmtime(str(x))  
                       for x in env.matchingFiles(directory,"*")]  
   
             else:  
                 # if it is not a directory, and we expected it to be  
                 # do something nasty.  
                 # we're in a try anyway.  
                 force_build = True  
                 os.unlink(directory)  
   
         dst_time = max(a)  
               
     except:  
         # Force an unlink and re-build.  
         force_build = True  
           
     if src_time > dst_time or force_build :  
         for x in the_targets :  
             try:  
                 os.unlink(x)  
             except OSError:  
                 pass  
   
         ret_target = env.Command(the_targets,source,epydocAction)  
220            
221         app = "cd "+pn+"; pwd; "+binpath(env, "run-escript")+" -ov "+sn
222      env.Clean(target, the_subdirs + file_names)     print "Executing test: ",app
223       if env.Execute(app) == 0:
224      return ret_target        open(str(target[0]),'w').write("PASSED\n")
225  #==============================================================================     else:
226  SConsEnvironment.epydocDepend = epydocDepend       return 1
227  ###############################################################################     print "Test execution time: ", round(time.time() - time_start, 1), " seconds wall time for " + str(source[0].abspath)
228       return None
229    
230  ###############################################################################  def eps2pdf(target, source, env):
231  def genSConscriptCalls(env,dir_list):  #   if env.Execute("epstopdf "+str(source[0].abspath)+" -o "+str(target[0].abspath))!=0:
232       if env.Execute("ps2pdf -dEPSCrop "+str(source[0].abspath)+" "+str(target[0].abspath))!=0:
233      for d in dir_list :         return 1
234          print 'calling SConscript in "./%s"' %(d)     return None
235          env.SConscript(dirs = [d],  
236                         build_dir='build/$PLATFORM/' + str(d),  def effectiveName(inname):
237                         duplicate=0)      m=re.compile("^r1i[0-9]{1,2}n[0-9]{1,2}$")  # savanna names take the form r1i?n?
238        if m.match(inname):
239            return "savanna"
240      return      return inname
 #==============================================================================  
 SConsEnvironment.genSConscriptCalls = genSConscriptCalls  
 ###############################################################################  
   
   
 ###############################################################################  
 def print_all_nodes(env,dirnode, level=0):  
     """Print all the scons nodes that are children of this node, recursively."""  
     if type(dirnode)==type(''):  
         dirnode=env.Dir(dirnode)  
     dt = type(env.Dir('.'))  
     for f in dirnode.all_children():  
         if type(f) == dt:  
             print "%s%s: .............."%(level * ' ', str(f))  
             env.print_all_nodes(f, level+2)  
         print "%s%s"%(level * ' ', str(f))  
   
 #==============================================================================  
 SConsEnvironment.print_all_nodes = print_all_nodes  
 ###############################################################################  
     
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
   
 ###############################################################################  
 def Glob(env,dir='.',includes="*",excludes=None, scan_dir=True):  
   
     """Similar to glob.glob, except globs SCons nodes, and thus sees  
     generated files and files from build directories.  Basically, it sees  
     anything SCons knows about.  A key subtlety is that since this function  
     operates on generated nodes as well as source nodes on the filesystem,  
     it needs to be called after builders that generate files you want to  
     include.  
   
     It will return both Dir entries and File entries  
     """  
   
     # Pre-process for more convenient arguments  
   
     if isinstance(includes,str) :  
         includes = env.Split(includes)  
   
     if isinstance(excludes,str) :  
         excludes = env.Split(excludes)  
   
     def fn_filter(node):  
         fn = os.path.basename(str(node))  
         match = 0  
         for include in includes:  
             if fnmatch.fnmatchcase( fn, include ):  
                 match = 1  
                 break  
   
         if match == 1 and not excludes is None:  
             for exclude in excludes:  
                 if fnmatch.fnmatchcase( fn, exclude ):  
                     match = 0  
                     break  
   
         return match  
   
     def filter_nodes(where):  
         contents = where.all_children(scan=scan_dir)  
         children = [ x for x in contents if fn_filter(x) ]  
         nodes = []  
         for f in children:  
             nodes.append(gen_node(f))  
         return nodes  
   
     def gen_node(n):  
         """Checks first to see if the node is a file or a dir, then  
         creates the appropriate node. [code seems redundant, if the node  
         is a node, then shouldn't it just be left as is?  
         """  
         if type(n) in (type(''), type(u'')):  
             path = n  
         else:  
             path = n.abspath  
         if os.path.isdir(path):  
             return env.Dir(n)  
         else:  
             return env.File(n)  
   
     here = env.Dir(dir)  
     nodes = filter_nodes(here)  
   
     node_srcs = [n.srcnode() for n in nodes]  
   
     src = here.srcnode()  
     if src is not here:  
         for s in filter_nodes(src):  
             if s not in node_srcs:  
                 # Probably need to check if this node is a directory  
                 nodes.append(  
                     gen_node(os.path.join(dir,os.path.basename(str(s)))))  
   
     return nodes  

Legend:
Removed from v.1233  
changed lines
  Added in v.5577

  ViewVC Help
Powered by ViewVC 1.1.26