trunk/scons/scons_extensions.py



# Extensions to Scons

import py_compile
import sys
import os
import glob
import fnmatch
import types

from SCons.Script.SConscript import SConsEnvironment

###############################################################################
def matchingFiles(env,directory,includes='*',excludes=None) :

    # Pre-process for more convenient arguments

    if isinstance(includes,str) :
        includes = env.Split(includes)

    if isinstance(excludes,str) :
        excludes = env.Split(excludes)

    def fn_filter(node):
        fn = os.path.basename(str(node))
        match = False
        for include in includes:
            if fnmatch.fnmatchcase( fn, include ):
                match = True
                break

        if match and not excludes is None:
            for exclude in excludes:
                if fnmatch.fnmatchcase( fn, exclude ):
                    match = False
                    break

        return match

    def filter_nodes(where):
        contents = glob.glob( os.path.join(str(where),"*") )
        children = [ x for x in contents if fn_filter(x) ]
        nodes = []
        for f in children:
            nodes.append(gen_node(f))
        return nodes

    def gen_node(n):
        """Checks first to see if the node is a file or a dir, then
        creates the appropriate node. [code seems redundant, if the node
        is a node, then shouldn't it just be left as is?
        """
        if type(n) in (type(''), type(u'')):
            path = n
        else:
            path = n.abspath

        if os.path.isdir(path):
            return env.Dir(n)

        return env.File(n)

    here = env.Dir(directory)
    nodes = filter_nodes(here)

    node_srcs = [n.srcnode() for n in nodes]

    src = here.srcnode()
    if src is not here:
        for s in filter_nodes(src):
            if s not in node_srcs:
                # Probably need to check if this node is a directory
                nodes.append(
                    gen_node(os.path.join(str(directory),
                                          os.path.basename(str(s)))))

    return nodes
#==============================================================================
SConsEnvironment.matchingFiles = matchingFiles
###############################################################################


###############################################################################
def InstallPyModule(env,target_dir,source,shlib=None,excludes=None) :

    # put the .so/.dll over in  <target_dir>
    if shlib :
        shtarg = env.Install(target_dir, shlib)

    # Gather the python sources
    python_src = env.matchingFiles(source, "*.py",excludes=excludes)

    # Here is a hack to deal with the (possibly not yet generated)
    # __init__.py. The problem is that the python_src list is built before
    # __init__.py is updated from __init__.in. The AlwaysBuild call
    # ensures that this does not cause a problem, except on the first
    # build after a clean checkout, in which case there is no old
    # __init__.py in src, and hence it does not make it into python_src!

    init_input = env.matchingFiles(source, "__init__.in")
    if init_input :
        if python_src :
            names = [x.name for x in python_src]
            if "__init__.py" not in names :
                python_src.append(env.File(os.path.join(str(source),
                                                        "__init__.py")))
        else:
            python_src = [env.File(os.path.join(str(source),"__init__.py"))]
            
    # decide if we're doing py or pyc distribn and install.

    if  env['distrib_py_src'] :
        pytarg = env.Install(target_dir, python_src)
    else:
        pyc = env.PyCompile(python_src)
        pytarg = env.Install(target_dir, pyc)

    if shlib :
        targ_ret = env.Flatten([pytarg] + [shtarg])
    else:
        targ_ret = pytarg

    return targ_ret
#==============================================================================
SConsEnvironment.InstallPyModule = InstallPyModule
###############################################################################

###############################################################################
def installDirectory(env,target,source,includes="*", excludes=None,
                     recursive=False):
    
    
    if os.path.isfile(str(target)) :
        raise UserError("target must be a directory")

    if os.path.isfile(str(source)) :
        raise UserError("source must be a directory")

    source_files = env.matchingFiles(source,includes,excludes)

    ret_targ = []

    for f in source_files :
        if f.isfile() :
            targ = env.Install(target,f)
            ret_targ.append(targ)

        if f.isdir() and recursive :
            x = os.path.basename(str(f))
            t = env.Dir(os.path.join(str(target),x))
            targ = env.installDirectory(t,f,includes,excludes,recursive)
            ret_targ += targ
    
    return ret_targ
#==============================================================================
SConsEnvironment.installDirectory = installDirectory
###############################################################################

###############################################################################
# Code to build .pyc from .py
def build_py(target, source, env):
    py_compile.compile(str(source[0]), str(target[0]))
    return 0

def doSubstitution(target,source,env) :
    import product_info as PI
    data = source[0].get_contents()
    data = data.replace('$ProductName$',PI.PRODUCT_NAME)
    data = data.replace('$LowerProductName$',PI.product_name)
    data = data.replace('$ProductVersion$',PI.product_version)
    data = data.replace('$VersionString$',PI.pkg_version_string)
    data = data.replace('$SVNRevision$',PI.svn_revision)
    open(str(target[0]),'w').write(data)
    return 0

# Code to run unit_test executables
def runUnitTest(target, source, env):
    app = str(source[0].abspath)

    olddir = os.getcwd()
    newdir = os.path.dirname(str(source[0]))
    os.chdir(newdir)

    if env.Execute(app) != 0:
        os.chdir(olddir)
        return 1

    os.chdir(olddir)
    open(str(target[0]),'w').write("PASSED\n")
    return 0

def runPyUnitTest(target, source, env): 
    app = env['python_path'] + ' "' + str(source[0].abspath) + '"'

    olddir = os.getcwd()
    newdir = os.path.dirname(str(source[0]))
    os.chdir(newdir)

    if env.Execute(app)  != 0:
        os.chdir(olddir)
        return 1

    os.chdir(olddir)
    open(str(target[0]),'w').write("PASSED\n")
    return 0
  

def addBuilders(env) :
    py_builder = env.Builder(action = build_py,
                             suffix = '.pyc',
                             src_suffix = '.py',
                             single_source=True)

    env.Append(BUILDERS = {'PyCompile' : py_builder});

    substituter = env.Builder(action = doSubstitution,
                              suffix = '',
                              src_suffix = '.in',
                              single_source=True )

    env.Append(BUILDERS = {'VariableSubstitution' : substituter});

    runUnitTest_builder = env.Builder(action = runUnitTest,
                                      suffix = '.passed',
                                      src_suffix=env.get('PROGSUFFIX',''),
                                      single_source=True)

    env.Append(BUILDERS = {'RunUnitTest' : runUnitTest_builder});

    runPyUnitTest_builder = env.Builder(action = runPyUnitTest,
                                        suffix = '.passed',
                                        src_suffix='.py',
                                        single_source=True)

    env.Append(BUILDERS = {'RunPyUnitTest' : runPyUnitTest_builder});
    return
#==============================================================================
SConsEnvironment.addBuilders = addBuilders
###############################################################################

###############################################################################
def epydocAction(target, source, env):

    doc_dir = os.path.dirname(str(target[0].abspath))

    cmd = [
        [env['epydoc_path'], "-qqqq", "-o", doc_dir, str(source[0].tpath)]
          ]
    print 'executing epydoc...'
    return env.Execute(cmd,"Build epydoc documentation")


def epydocDepend(env, target, source,
                 src_pattern="*.py",
                 file_names=['index.html','epydoc.css'],
                 subdirs=['private','public']) :
    """
    \brief add a dependency between the directory containing the doco and that
           containing the source
    \param target - the directory containing index.html for the doco,
                    and the private and public sub-directories.
    \param source - the python module source directory
    """
    the_subdirs = [os.path.join(target.abspath,sub) for sub in subdirs]
    the_targets = [os.path.join(target.abspath,file) for file in file_names]


    ret_target = target
    
    # if absolutely anything goes wrong, turn this on.
    force_build = False

    dst_time = 0
    src_time = 1

    try:

        # have a shot at digging out all the source file times.
        src_time = max(
            [os.path.getmtime(str(x))
             for x in env.matchingFiles(source,src_pattern) +
                      [source.abspath]]
            )

        # now try to find the target files and their mod time.
        a = [os.path.getmtime(os.path.join(target.abspath,str(x)))
             for x in file_names ] 

        for directory in the_subdirs :
            # include the mod time of the directory
            a += [os.path.getmtime(str(directory))]

            # now go for the mod times of all files below the subdirs.
            if os.path.isdir(str(directory)) :
                a += [os.path.getmtime(str(x))
                      for x in env.matchingFiles(directory,"*")]

            else:
                # if it is not a directory, and we expected it to be
                # do something nasty.
                # we're in a try anyway.
                force_build = True
                os.unlink(directory)

        dst_time = max(a)
            
    except:
        # Force an unlink and re-build.
        force_build = True
        
    if src_time > dst_time or force_build :
        for x in the_targets :
            try:
                os.unlink(x)
            except OSError:
                pass

        ret_target = env.Command(the_targets,source,epydocAction)
    

    env.Clean(target, the_subdirs + file_names)

    return ret_target
#==============================================================================
SConsEnvironment.epydocDepend = epydocDepend
###############################################################################


###############################################################################
def genSConscriptCalls(env,dir_list):

    for d in dir_list :
        print 'calling SConscript in "./%s"' %(d)
        env.SConscript(dirs = [d],
                       build_dir='build/$PLATFORM/' + str(d),
                       duplicate=0)


    return
#==============================================================================
SConsEnvironment.genSConscriptCalls = genSConscriptCalls
###############################################################################


###############################################################################
def print_all_nodes(env,dirnode, level=0):
    """Print all the scons nodes that are children of this node, recursively."""
    if type(dirnode)==type(''):
        dirnode=env.Dir(dirnode)
    dt = type(env.Dir('.'))
    for f in dirnode.all_children():
        if type(f) == dt:
            print "%s%s: .............."%(level * ' ', str(f))
            env.print_all_nodes(f, level+2)
        print "%s%s"%(level * ' ', str(f))

#==============================================================================
SConsEnvironment.print_all_nodes = print_all_nodes
###############################################################################
  

###############################################################################
def Glob(env,dir='.',includes="*",excludes=None, scan_dir=True):

    """Similar to glob.glob, except globs SCons nodes, and thus sees
    generated files and files from build directories.  Basically, it sees
    anything SCons knows about.  A key subtlety is that since this function
    operates on generated nodes as well as source nodes on the filesystem,
    it needs to be called after builders that generate files you want to
    include.

    It will return both Dir entries and File entries
    """

    # Pre-process for more convenient arguments

    if isinstance(includes,str) :
        includes = env.Split(includes)

    if isinstance(excludes,str) :
        excludes = env.Split(excludes)

    def fn_filter(node):
        fn = os.path.basename(str(node))
        match = 0
        for include in includes:
            if fnmatch.fnmatchcase( fn, include ):
                match = 1
                break

        if match == 1 and not excludes is None:
            for exclude in excludes:
                if fnmatch.fnmatchcase( fn, exclude ):
                    match = 0
                    break

        return match

    def filter_nodes(where):
        contents = where.all_children(scan=scan_dir)
        children = [ x for x in contents if fn_filter(x) ]
        nodes = []
        for f in children:
            nodes.append(gen_node(f))
        return nodes

    def gen_node(n):
        """Checks first to see if the node is a file or a dir, then
        creates the appropriate node. [code seems redundant, if the node
        is a node, then shouldn't it just be left as is?
        """
        if type(n) in (type(''), type(u'')):
            path = n
        else:
            path = n.abspath
        if os.path.isdir(path):
            return env.Dir(n)
        else:
            return env.File(n)

    here = env.Dir(dir)
    nodes = filter_nodes(here)

    node_srcs = [n.srcnode() for n in nodes]

    src = here.srcnode()
    if src is not here:
        for s in filter_nodes(src):
            if s not in node_srcs:
                # Probably need to check if this node is a directory
                nodes.append(
                    gen_node(os.path.join(dir,os.path.basename(str(s)))))

    return nodes
1	elspeth	645
2
3	jgs	268	# Extensions to Scons
4
5			import py_compile
6			import sys
7	jgs	297	import os
8	phornby	1233	import glob
9			import fnmatch
10			import types
11	jgs	268
12	phornby	1233	from SCons.Script.SConscript import SConsEnvironment
13
14			###############################################################################
15			def matchingFiles(env,directory,includes='*',excludes=None) :
16
17			# Pre-process for more convenient arguments
18
19			if isinstance(includes,str) :
20			includes = env.Split(includes)
21
22			if isinstance(excludes,str) :
23			excludes = env.Split(excludes)
24
25			def fn_filter(node):
26			fn = os.path.basename(str(node))
27			match = False
28			for include in includes:
29			if fnmatch.fnmatchcase( fn, include ):
30			match = True
31			break
32
33			if match and not excludes is None:
34			for exclude in excludes:
35			if fnmatch.fnmatchcase( fn, exclude ):
36			match = False
37			break
38
39			return match
40
41			def filter_nodes(where):
42			contents = glob.glob( os.path.join(str(where),"*") )
43			children = [ x for x in contents if fn_filter(x) ]
44			nodes = []
45			for f in children:
46			nodes.append(gen_node(f))
47			return nodes
48
49			def gen_node(n):
50			"""Checks first to see if the node is a file or a dir, then
51			creates the appropriate node. [code seems redundant, if the node
52			is a node, then shouldn't it just be left as is?
53			"""
54			if type(n) in (type(''), type(u'')):
55			path = n
56			else:
57			path = n.abspath
58
59			if os.path.isdir(path):
60			return env.Dir(n)
61
62			return env.File(n)
63
64			here = env.Dir(directory)
65			nodes = filter_nodes(here)
66
67			node_srcs = [n.srcnode() for n in nodes]
68
69			src = here.srcnode()
70			if src is not here:
71			for s in filter_nodes(src):
72			if s not in node_srcs:
73			# Probably need to check if this node is a directory
74			nodes.append(
75			gen_node(os.path.join(str(directory),
76			os.path.basename(str(s)))))
77
78			return nodes
79			#==============================================================================
80			SConsEnvironment.matchingFiles = matchingFiles
81			###############################################################################
82
83
84
85			###############################################################################
86			def InstallPyModule(env,target_dir,source,shlib=None,excludes=None) :
87
88			# put the .so/.dll over in <target_dir>
89			if shlib :
90			shtarg = env.Install(target_dir, shlib)
91
92			# Gather the python sources
93			python_src = env.matchingFiles(source, "*.py",excludes=excludes)
94
95			# Here is a hack to deal with the (possibly not yet generated)
96			# __init__.py. The problem is that the python_src list is built before
97			# __init__.py is updated from __init__.in. The AlwaysBuild call
98			# ensures that this does not cause a problem, except on the first
99			# build after a clean checkout, in which case there is no old
100			# __init__.py in src, and hence it does not make it into python_src!
101
102			init_input = env.matchingFiles(source, "__init__.in")
103			if init_input :
104			if python_src :
105			names = [x.name for x in python_src]
106			if "__init__.py" not in names :
107			python_src.append(env.File(os.path.join(str(source),
108			"__init__.py")))
109			else:
110			python_src = [env.File(os.path.join(str(source),"__init__.py"))]
111
112			# decide if we're doing py or pyc distribn and install.
113
114			if env['distrib_py_src'] :
115			pytarg = env.Install(target_dir, python_src)
116			else:
117			pyc = env.PyCompile(python_src)
118			pytarg = env.Install(target_dir, pyc)
119
120			if shlib :
121			targ_ret = env.Flatten([pytarg] + [shtarg])
122			else:
123			targ_ret = pytarg
124
125			return targ_ret
126			#==============================================================================
127			SConsEnvironment.InstallPyModule = InstallPyModule
128			###############################################################################
129
130			###############################################################################
131			def installDirectory(env,target,source,includes="*", excludes=None,
132			recursive=False):
133
134
135			if os.path.isfile(str(target)) :
136			raise UserError("target must be a directory")
137
138			if os.path.isfile(str(source)) :
139			raise UserError("source must be a directory")
140
141			source_files = env.matchingFiles(source,includes,excludes)
142
143			ret_targ = []
144
145			for f in source_files :
146			if f.isfile() :
147			targ = env.Install(target,f)
148			ret_targ.append(targ)
149
150			if f.isdir() and recursive :
151			x = os.path.basename(str(f))
152			t = env.Dir(os.path.join(str(target),x))
153			targ = env.installDirectory(t,f,includes,excludes,recursive)
154			ret_targ += targ
155
156			return ret_targ
157			#==============================================================================
158			SConsEnvironment.installDirectory = installDirectory
159			###############################################################################
160
161			###############################################################################
162	jgs	268	# Code to build .pyc from .py
163			def build_py(target, source, env):
164	phornby	1233	py_compile.compile(str(source[0]), str(target[0]))
165			return 0
166	jgs	297
167	phornby	1233	def doSubstitution(target,source,env) :
168			import product_info as PI
169			data = source[0].get_contents()
170			data = data.replace('$ProductName$',PI.PRODUCT_NAME)
171			data = data.replace('$LowerProductName$',PI.product_name)
172			data = data.replace('$ProductVersion$',PI.product_version)
173			data = data.replace('$VersionString$',PI.pkg_version_string)
174			data = data.replace('$SVNRevision$',PI.svn_revision)
175			open(str(target[0]),'w').write(data)
176			return 0
177
178	jgs	297	# Code to run unit_test executables
179			def runUnitTest(target, source, env):
180	phornby	1233	app = str(source[0].abspath)
181
182			olddir = os.getcwd()
183			newdir = os.path.dirname(str(source[0]))
184			os.chdir(newdir)
185
186			if env.Execute(app) != 0:
187			os.chdir(olddir)
188			return 1
189
190			os.chdir(olddir)
191	jgs	297	open(str(target[0]),'w').write("PASSED\n")
192	phornby	1233	return 0
193	cochrane	370
194	robwdcock	682	def runPyUnitTest(target, source, env):
195	phornby	1233	app = env['python_path'] + ' "' + str(source[0].abspath) + '"'
196
197			olddir = os.getcwd()
198			newdir = os.path.dirname(str(source[0]))
199			os.chdir(newdir)
200
201			if env.Execute(app) != 0:
202			os.chdir(olddir)
203			return 1
204
205			os.chdir(olddir)
206			open(str(target[0]),'w').write("PASSED\n")
207			return 0
208
209
210			def addBuilders(env) :
211			py_builder = env.Builder(action = build_py,
212			suffix = '.pyc',
213			src_suffix = '.py',
214			single_source=True)
215
216			env.Append(BUILDERS = {'PyCompile' : py_builder});
217
218			substituter = env.Builder(action = doSubstitution,
219			suffix = '',
220			src_suffix = '.in',
221			single_source=True )
222
223			env.Append(BUILDERS = {'VariableSubstitution' : substituter});
224
225			runUnitTest_builder = env.Builder(action = runUnitTest,
226			suffix = '.passed',
227			src_suffix=env.get('PROGSUFFIX',''),
228			single_source=True)
229
230			env.Append(BUILDERS = {'RunUnitTest' : runUnitTest_builder});
231
232			runPyUnitTest_builder = env.Builder(action = runPyUnitTest,
233			suffix = '.passed',
234			src_suffix='.py',
235			single_source=True)
236
237			env.Append(BUILDERS = {'RunPyUnitTest' : runPyUnitTest_builder});
238			return
239			#==============================================================================
240			SConsEnvironment.addBuilders = addBuilders
241			###############################################################################
242
243			###############################################################################
244			def epydocAction(target, source, env):
245
246			doc_dir = os.path.dirname(str(target[0].abspath))
247
248			cmd = [
249			[env['epydoc_path'], "-qqqq", "-o", doc_dir, str(source[0].tpath)]
250			]
251			print 'executing epydoc...'
252			return env.Execute(cmd,"Build epydoc documentation")
253
254
255			def epydocDepend(env, target, source,
256			src_pattern="*.py",
257			file_names=['index.html','epydoc.css'],
258			subdirs=['private','public']) :
259			"""
260			\brief add a dependency between the directory containing the doco and that
261			containing the source
262			\param target - the directory containing index.html for the doco,
263			and the private and public sub-directories.
264			\param source - the python module source directory
265			"""
266			the_subdirs = [os.path.join(target.abspath,sub) for sub in subdirs]
267			the_targets = [os.path.join(target.abspath,file) for file in file_names]
268
269
270			ret_target = target
271
272			# if absolutely anything goes wrong, turn this on.
273			force_build = False
274
275			dst_time = 0
276			src_time = 1
277
278			try:
279
280			# have a shot at digging out all the source file times.
281			src_time = max(
282			[os.path.getmtime(str(x))
283			for x in env.matchingFiles(source,src_pattern) +
284			[source.abspath]]
285			)
286
287			# now try to find the target files and their mod time.
288			a = [os.path.getmtime(os.path.join(target.abspath,str(x)))
289			for x in file_names ]
290
291			for directory in the_subdirs :
292			# include the mod time of the directory
293			a += [os.path.getmtime(str(directory))]
294
295			# now go for the mod times of all files below the subdirs.
296			if os.path.isdir(str(directory)) :
297			a += [os.path.getmtime(str(x))
298			for x in env.matchingFiles(directory,"*")]
299
300			else:
301			# if it is not a directory, and we expected it to be
302			# do something nasty.
303			# we're in a try anyway.
304			force_build = True
305			os.unlink(directory)
306
307			dst_time = max(a)
308
309			except:
310			# Force an unlink and re-build.
311			force_build = True
312
313			if src_time > dst_time or force_build :
314			for x in the_targets :
315			try:
316			os.unlink(x)
317			except OSError:
318			pass
319
320			ret_target = env.Command(the_targets,source,epydocAction)
321
322
323			env.Clean(target, the_subdirs + file_names)
324
325			return ret_target
326			#==============================================================================
327			SConsEnvironment.epydocDepend = epydocDepend
328			###############################################################################
329
330
331			###############################################################################
332			def genSConscriptCalls(env,dir_list):
333
334			for d in dir_list :
335			print 'calling SConscript in "./%s"' %(d)
336			env.SConscript(dirs = [d],
337			build_dir='build/$PLATFORM/' + str(d),
338			duplicate=0)
339
340
341			return
342			#==============================================================================
343			SConsEnvironment.genSConscriptCalls = genSConscriptCalls
344			###############################################################################
345
346
347			###############################################################################
348			def print_all_nodes(env,dirnode, level=0):
349			"""Print all the scons nodes that are children of this node, recursively."""
350			if type(dirnode)==type(''):
351			dirnode=env.Dir(dirnode)
352			dt = type(env.Dir('.'))
353			for f in dirnode.all_children():
354			if type(f) == dt:
355			print "%s%s: .............."%(level * ' ', str(f))
356			env.print_all_nodes(f, level+2)
357			print "%s%s"%(level * ' ', str(f))
358
359			#==============================================================================
360			SConsEnvironment.print_all_nodes = print_all_nodes
361			###############################################################################
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383			###############################################################################
384			def Glob(env,dir='.',includes="*",excludes=None, scan_dir=True):
385
386			"""Similar to glob.glob, except globs SCons nodes, and thus sees
387			generated files and files from build directories. Basically, it sees
388			anything SCons knows about. A key subtlety is that since this function
389			operates on generated nodes as well as source nodes on the filesystem,
390			it needs to be called after builders that generate files you want to
391			include.
392
393			It will return both Dir entries and File entries
394			"""
395
396			# Pre-process for more convenient arguments
397
398			if isinstance(includes,str) :
399			includes = env.Split(includes)
400
401			if isinstance(excludes,str) :
402			excludes = env.Split(excludes)
403
404			def fn_filter(node):
405			fn = os.path.basename(str(node))
406			match = 0
407			for include in includes:
408			if fnmatch.fnmatchcase( fn, include ):
409			match = 1
410			break
411
412			if match == 1 and not excludes is None:
413			for exclude in excludes:
414			if fnmatch.fnmatchcase( fn, exclude ):
415			match = 0
416			break
417
418			return match
419
420			def filter_nodes(where):
421			contents = where.all_children(scan=scan_dir)
422			children = [ x for x in contents if fn_filter(x) ]
423			nodes = []
424			for f in children:
425			nodes.append(gen_node(f))
426			return nodes
427
428			def gen_node(n):
429			"""Checks first to see if the node is a file or a dir, then
430			creates the appropriate node. [code seems redundant, if the node
431			is a node, then shouldn't it just be left as is?
432			"""
433			if type(n) in (type(''), type(u'')):
434			path = n
435			else:
436			path = n.abspath
437			if os.path.isdir(path):
438			return env.Dir(n)
439			else:
440			return env.File(n)
441
442			here = env.Dir(dir)
443			nodes = filter_nodes(here)
444
445			node_srcs = [n.srcnode() for n in nodes]
446
447			src = here.srcnode()
448			if src is not here:
449			for s in filter_nodes(src):
450			if s not in node_srcs:
451			# Probably need to check if this node is a directory
452			nodes.append(
453			gen_node(os.path.join(dir,os.path.basename(str(s)))))
454
455			return nodes