/[escript]/trunk/escript/py_src/pythfilter
ViewVC logotype

Annotation of /trunk/escript/py_src/pythfilter

Parent Directory Parent Directory | Revision Log Revision Log


Revision 155 - (hide annotations)
Wed Nov 9 02:02:19 2005 UTC (13 years, 11 months ago) by jgs
File size: 19899 byte(s)
move all directories from trunk/esys2 into trunk and remove esys2

1 jgs 82 #!/usr/bin/env python
2    
3     # pythfilter.py v1.5.5, written by Matthias Baas (baas@ira.uka.de)
4    
5     # Doxygen filter which can be used to document Python source code.
6     # Classes (incl. methods) and functions can be documented.
7     # Every comment that begins with ## is literally turned into an
8     # Doxygen comment. Consecutive comment lines are turned into
9     # comment blocks (-> /** ... */).
10     # All the stuff is put inside a namespace with the same name as
11     # the source file.
12    
13     # Conversions:
14     # ============
15     # ##-blocks -> /** ... */
16     # "class name(base): ..." -> "class name : public base {...}"
17     # "def name(params): ..." -> "name(params) {...}"
18    
19     # Changelog:
20     # 21.01.2003: Raw (r"") or unicode (u"") doc string will now be properly
21     # handled. (thanks to Richard Laager for the patch)
22     # 22.12.2003: Fixed a bug where no function names would be output for "def"
23     # blocks that were not in a class.
24     # (thanks to Richard Laager for the patch)
25     # 12.12.2003: Implemented code to handle static and class methods with
26     # this logic: Methods with "self" as the first argument are
27     # non-static. Methods with "cls" are Python class methods,
28     # which translate into static methods for Doxygen. Other
29     # methods are assumed to be static methods. As should be
30     # obvious, this logic doesn't take into account if the method
31     # is actually setup as a classmethod() or a staticmethod(),
32     # just if it follows the normal conventions.
33     # (thanks to Richard Laager for the patch)
34     # 11.12.2003: Corrected #includes to use os.path.sep instead of ".". Corrected
35     # namespace code to use "::" instead of ".".
36     # (thanks to Richard Laager for the patch)
37     # 11.12.2003: Methods beginning with two underscores that end with
38     # something other than two underscores are considered private
39     # and are handled accordingly.
40     # (thanks to Richard Laager for the patch)
41     # 03.12.2003: The first parameter of class methods (self) is removed from
42     # the documentation.
43     # 03.11.2003: The module docstring will be used as namespace documentation
44     # (thanks to Joe Bronkema for the patch)
45     # 08.07.2003: Namespaces get a default documentation so that the namespace
46     # and its contents will show up in the generated documentation.
47     # 05.02.2003: Directories will be delted during synchronization.
48     # 31.01.2003: -f option & filtering entire directory trees.
49     # 10.08.2002: In base classes the '.' will be replaced by '::'
50     # 18.07.2002: * and ** will be translated into arguments
51     # 18.07.2002: Argument lists may contain default values using constructors.
52     # 18.06.2002: Support for ## public:
53     # 21.01.2002: from ... import will be translated to "using namespace ...;"
54     # TODO: "from ... import *" vs "from ... import names"
55     # TODO: Using normal imports: name.name -> name::name
56     # 20.01.2002: #includes will be placed in front of the namespace
57    
58     ######################################################################
59    
60     # The program is written as a state machine with the following states:
61     #
62     # - OUTSIDE The current position is outside any comment,
63     # class definition or function.
64     #
65     # - BUILD_COMMENT Begins with first "##".
66     # Ends with the first token that is no "##"
67     # at the same column as before.
68     #
69     # - BUILD_CLASS_DECL Begins with "class".
70     # Ends with ":"
71     # - BUILD_CLASS_BODY Begins just after BUILD_CLASS_DECL.
72     # The first following token (which is no comment)
73     # determines indentation depth.
74     # Ends with a token that has a smaller indendation.
75     #
76     # - BUILD_DEF_DECL Begins with "def".
77     # Ends with ":".
78     # - BUILD_DEF_BODY Begins just after BUILD_DEF_DECL.
79     # The first following token (which is no comment)
80     # determines indentation depth.
81     # Ends with a token that has a smaller indendation.
82    
83     import getopt
84     import glob
85     import os.path
86     import shutil
87     import string
88     import sys
89     import token
90     import tokenize
91    
92     from stat import *
93    
94     OUTSIDE = 0
95     BUILD_COMMENT = 1
96     BUILD_CLASS_DECL = 2
97     BUILD_CLASS_BODY = 3
98     BUILD_DEF_DECL = 4
99     BUILD_DEF_BODY = 5
100     IMPORT = 6
101     IMPORT_OP = 7
102     IMPORT_APPEND = 8
103    
104     # Output file stream
105     outfile = sys.stdout
106    
107     # Output buffer
108     outbuffer = []
109    
110     out_row = 0
111     out_col = 0
112    
113     # Variables used by rec_name_n_param()
114     name = ""
115     param = ""
116     doc_string = ""
117     record_state = 0
118     bracket_counter = 0
119    
120     # Tuple: (row,column)
121     class_spos = (0,0)
122     def_spos = (0,0)
123     import_spos = (0,0)
124    
125     # Which import was used? ("import" or "from")
126     import_token = ""
127    
128     # Comment block buffer
129     comment_block = []
130     comment_finished = 0
131    
132     # Imported modules
133     modules = []
134    
135     # Program state
136     stateStack = [OUTSIDE]
137    
138     # Keep track of whether module has a docstring
139     module_has_docstring = False
140    
141     # Keep track of member protection
142     protection_level = "public"
143     private_member = False
144    
145     # Keep track of the module namespace
146     namespace = ""
147    
148     ######################################################################
149     # Output string s. '\n' may only be at the end of the string (not
150     # somewhere in the middle).
151     #
152     # In: s - String
153     # spos - Startpos
154     ######################################################################
155     def output(s,spos, immediate=0):
156     global outbuffer, out_row, out_col, outfile
157    
158     os = string.rjust(s,spos[1]-out_col+len(s))
159     if immediate:
160     outfile.write(os)
161     else:
162     outbuffer.append(os)
163     if (s[-1:]=="\n"):
164     out_row = out_row+1
165     out_col = 0
166     else:
167     out_col = spos[1]+len(s)
168    
169    
170     ######################################################################
171     # Records a name and parameters. The name is either a class name or
172     # a function name. Then the parameter is either the base class or
173     # the function parameters.
174     # The name is stored in the global variable "name", the parameters
175     # in "param".
176     # The variable "record_state" holds the current state of this internal
177     # state machine.
178     # The recording is started by calling start_recording().
179     #
180     # In: type, tok
181     ######################################################################
182     def rec_name_n_param(type, tok):
183     global record_state,name,param,doc_string,bracket_counter
184     s = record_state
185     # State 0: Do nothing.
186     if (s==0):
187     return
188     # State 1: Remember name.
189     elif (s==1):
190     name = tok
191     record_state = 2
192     # State 2: Wait for opening bracket or colon
193     elif (s==2):
194     if (tok=='('):
195     bracket_counter = 1
196     record_state=3
197     if (tok==':'): record_state=4
198     # State 3: Store parameter (or base class) and wait for an ending bracket
199     elif (s==3):
200     if (tok=='*' or tok=='**'):
201     tok=''
202     if (tok=='('):
203     bracket_counter = bracket_counter+1
204     if (tok==')'):
205     bracket_counter = bracket_counter-1
206     if bracket_counter==0:
207     record_state=4
208     else:
209     param=param+tok
210     # State 4: Look for doc string
211     elif (s==4):
212     if (type==token.NEWLINE or type==token.INDENT or type==token.SLASHEQUAL):
213     return
214     elif (tok==":"):
215     return
216     elif (type==token.STRING):
217     while tok[:1]=='r' or tok[:1]=='u':
218     tok=tok[1:]
219     while tok[:1]=='"':
220     tok=tok[1:]
221     while tok[-1:]=='"':
222     tok=tok[:-1]
223     doc_string=tok
224     record_state=0
225    
226     ######################################################################
227     # Starts the recording of a name & param part.
228     # The function rec_name_n_param() has to be fed with tokens. After
229     # the necessary tokens are fed the name and parameters can be found
230     # in the global variables "name" und "param".
231     ######################################################################
232     def start_recording():
233     global record_state,param,name, doc_string
234     record_state=1
235     name=""
236     param=""
237     doc_string=""
238    
239     ######################################################################
240     # Test if recording is finished
241     ######################################################################
242     def is_recording_finished():
243     global record_state
244     return record_state==0
245    
246     ######################################################################
247     ## Gather comment block
248     ######################################################################
249     def gather_comment(type,tok,spos):
250     global comment_block,comment_finished
251     if (type!=tokenize.COMMENT):
252     comment_finished = 1
253     else:
254     # Output old comment block if a new one is started.
255     if (comment_finished):
256     print_comment(spos)
257     comment_finished=0
258     if (tok[0:2]=="##" and tok[0:3]!="###"):
259     comment_block.append(tok[2:])
260    
261     ######################################################################
262     ## Output comment block and empty buffer.
263     ######################################################################
264     def print_comment(spos):
265     global comment_block,comment_finished
266     if (comment_block!=[]):
267     output("/**\n",spos)
268     for c in comment_block:
269     output(c,spos)
270     output("*/\n",spos)
271     comment_block = []
272     comment_finished = 0
273    
274     ######################################################################
275     def set_state(s):
276     global stateStack
277     stateStack[len(stateStack)-1]=s
278    
279     ######################################################################
280     def get_state():
281     global stateStack
282     return stateStack[len(stateStack)-1]
283    
284     ######################################################################
285     def push_state(s):
286     global stateStack
287     stateStack.append(s)
288    
289     ######################################################################
290     def pop_state():
291     global stateStack
292     stateStack.pop()
293    
294    
295     ######################################################################
296     def tok_eater(type, tok, spos, epos, line):
297     global stateStack,name,param,class_spos,def_spos,import_spos
298     global doc_string, modules, import_token, module_has_docstring
299     global protection_level, private_member
300    
301     rec_name_n_param(type,tok)
302     if (string.replace(string.strip(tok)," ","")=="##private:"):
303     protection_level = "private"
304     output("private:\n",spos)
305     elif (string.replace(string.strip(tok)," ","")=="##protected:"):
306     protection_level = "protected"
307     output("protected:\n",spos)
308     elif (string.replace(string.strip(tok)," ","")=="##public:"):
309     protection_level = "public"
310     output("public:\n",spos)
311     else:
312     gather_comment(type,tok,spos)
313    
314     state = get_state()
315    
316     # sys.stderr.write("%d: %s\n"%(state, tok))
317    
318     # OUTSIDE
319     if (state==OUTSIDE):
320     if (tok=="class"):
321     start_recording()
322     class_spos = spos
323     push_state(BUILD_CLASS_DECL)
324     elif (tok=="def"):
325     start_recording()
326     def_spos = spos
327     push_state(BUILD_DEF_DECL)
328     elif (tok=="import") or (tok=="from"):
329     import_token = tok
330     import_spos = spos
331     modules = []
332     push_state(IMPORT)
333     elif (spos[1] == 0 and tok[:3] == '"""'):
334     # Capture module docstring as namespace documentation
335     module_has_docstring = True
336     comment_block.append("\\namespace %s\n" % namespace)
337     comment_block.append(tok[3:-3])
338     print_comment(spos)
339    
340     # IMPORT
341     elif (state==IMPORT):
342     if (type==token.NAME):
343     modules.append(tok)
344     set_state(IMPORT_OP)
345     # IMPORT_OP
346     elif (state==IMPORT_OP):
347     if (tok=="."):
348     set_state(IMPORT_APPEND)
349     elif (tok==","):
350     set_state(IMPORT)
351     else:
352     for m in modules:
353     output('#include "'+m.replace('.',os.path.sep)+'.py"\n', import_spos, immediate=1)
354     if import_token=="from":
355     output('using namespace '+m.replace('.', '::')+';\n', import_spos)
356     pop_state()
357     # IMPORT_APPEND
358     elif (state==IMPORT_APPEND):
359     if (type==token.NAME):
360     modules[len(modules)-1]+="."+tok
361     set_state(IMPORT_OP)
362     # BUILD_CLASS_DECL
363     elif (state==BUILD_CLASS_DECL):
364     if (is_recording_finished()):
365     s = "class "+name
366     if (param!=""): s = s+" : public "+param.replace('.','::')
367     if (doc_string!=""): comment_block.append(doc_string)
368     print_comment(class_spos)
369     output(s+"\n",class_spos)
370     output("{\n",(class_spos[0]+1,class_spos[1]))
371     protection_level = "public"
372     output(" public:\n",(class_spos[0]+2,class_spos[1]))
373     set_state(BUILD_CLASS_BODY)
374     # BUILD_CLASS_BODY
375     elif (state==BUILD_CLASS_BODY):
376     if (type!=token.INDENT and type!=token.NEWLINE and type!=40 and
377     type!=tokenize.NL and type!=tokenize.COMMENT and
378     (spos[1]<=class_spos[1])):
379     output("}; // end of class\n",(out_row+1,class_spos[1]))
380     pop_state()
381     elif (tok=="def"):
382     start_recording()
383     def_spos = spos
384     push_state(BUILD_DEF_DECL)
385     # BUILD_DEF_DECL
386     elif (state==BUILD_DEF_DECL):
387     if (is_recording_finished()):
388     s = ''
389     # Do we document a class method? then remove the 'self' parameter
390     if BUILD_CLASS_BODY in stateStack:
391     params = param.split(",")
392     if params[0] == 'self':
393     param = string.join(params[1:], ",")
394     else:
395     s = 'static '
396     if params[0] == 'cls':
397     param = string.join(params[1:], ",")
398     s = s+name+"("+param+");\n"
399     if len(name) > 1 \
400     and name[0:2] == '__' \
401     and name[len(name)-2:len(name)] != '__' \
402     and protection_level != 'private':
403     private_member = True
404     output(" private:\n",(def_spos[0]+2,def_spos[1]))
405     else:
406     s = name+"("+param+");\n"
407     if (doc_string!=""): comment_block.append(doc_string)
408     print_comment(def_spos)
409     output(s,def_spos)
410     # output("{\n",(def_spos[0]+1,def_spos[1]))
411     set_state(BUILD_DEF_BODY)
412     # BUILD_DEF_BODY
413     elif (state==BUILD_DEF_BODY):
414     if (type!=token.INDENT and type!=token.NEWLINE \
415     and type!=40 and type!=tokenize.NL \
416     and (spos[1]<=def_spos[1])):
417     # output("} // end of method/function\n",(out_row+1,def_spos[1]))
418     if private_member and protection_level != 'private':
419     private_member = False
420     output(" " + protection_level + ":\n",(def_spos[0]+2,def_spos[1]))
421     pop_state()
422     # else:
423     # output(tok,spos)
424    
425    
426     def dump(filename):
427     f = open(filename)
428     r = f.readlines()
429     for s in r:
430     sys.stdout.write(s)
431    
432     def filter(filename):
433     global name, module_has_docstring
434    
435     path,name = os.path.split(filename)
436     root,ext = os.path.splitext(name)
437    
438     output("namespace "+root+" {\n",(0,0))
439    
440     # set module name for tok_eater to use if there's a module doc string
441     name = root
442    
443     sys.stderr.write('Filtering "'+filename+'"...')
444     f = open(filename)
445     tokenize.tokenize(f.readline, tok_eater)
446     f.close()
447     print_comment((0,0))
448    
449     output("\n",(0,0))
450     output("} // end of namespace\n",(0,0))
451    
452     if not module_has_docstring:
453     # Put in default namespace documentation
454     output('/** \\namespace '+root+' \n',(0,0))
455     output(' \\brief Module "%s" */\n'%(root),(0,0))
456    
457     for s in outbuffer:
458     outfile.write(s)
459    
460    
461     def filterFile(filename, out=sys.stdout):
462     global outfile
463    
464     outfile = out
465    
466     try:
467     root,ext = os.path.splitext(filename)
468    
469     if ext==".py":
470     filter(filename)
471     else:
472     dump(filename)
473    
474     sys.stderr.write("OK\n")
475     except IOError,e:
476     sys.stderr.write(e[1]+"\n")
477    
478    
479     ######################################################################
480    
481     # preparePath
482     def preparePath(path):
483     """Prepare a path.
484    
485     Checks if the path exists and creates it if it does not exist.
486     """
487     if not os.path.exists(path):
488     parent = os.path.dirname(path)
489     if parent!="":
490     preparePath(parent)
491     os.mkdir(path)
492    
493     # isNewer
494     def isNewer(file1,file2):
495     """Check if file1 is newer than file2.
496    
497     file1 must be an existing file.
498     """
499     if not os.path.exists(file2):
500     return True
501     return os.stat(file1)[ST_MTIME]>os.stat(file2)[ST_MTIME]
502    
503     # convert
504     def convert(srcpath, destpath):
505     """Convert a Python source tree into a C+ stub tree.
506    
507     All *.py files in srcpath (including sub-directories) are filtered
508     and written to destpath. If destpath exists, only the files
509     that have been modified are filtered again. Files that were deleted
510     from srcpath are also deleted in destpath if they are still present.
511     The function returns the number of processed *.py files.
512     """
513     count=0
514     sp = os.path.join(srcpath,"*")
515     sfiles = glob.glob(sp)
516     dp = os.path.join(destpath,"*")
517     dfiles = glob.glob(dp)
518     leftovers={}
519     for df in dfiles:
520     leftovers[os.path.basename(df)]=1
521    
522     for srcfile in sfiles:
523     basename = os.path.basename(srcfile)
524     if basename in leftovers:
525     del leftovers[basename]
526    
527     # Is it a subdirectory?
528     if os.path.isdir(srcfile):
529     sdir = os.path.join(srcpath,basename)
530     ddir = os.path.join(destpath,basename)
531     count+=convert(sdir, ddir)
532     continue
533     # Check the extension (only *.py will be converted)
534     root, ext = os.path.splitext(srcfile)
535     if ext.lower()!=".py":
536     continue
537    
538     destfile = os.path.join(destpath,basename)
539     if destfile==srcfile:
540     print "WARNING: Input and output names are identical!"
541     sys.exit(1)
542    
543     count+=1
544     # sys.stdout.write("%s\015"%(srcfile))
545    
546     if isNewer(srcfile, destfile):
547     preparePath(os.path.dirname(destfile))
548     # out=open(destfile,"w")
549     # filterFile(srcfile, out)
550     # out.close()
551     os.system("python %s -f %s>%s"%(sys.argv[0],srcfile,destfile))
552    
553     # Delete obsolete files in destpath
554     for df in leftovers:
555     dname=os.path.join(destpath,df)
556     if os.path.isdir(dname):
557     try:
558     shutil.rmtree(dname)
559     except:
560     print "Can't remove obsolete directory '%s'"%dname
561     else:
562     try:
563     os.remove(dname)
564     except:
565     print "Can't remove obsolete file '%s'"%dname
566    
567     return count
568    
569    
570     ######################################################################
571     ######################################################################
572     ######################################################################
573    
574     filter_file = False
575    
576     try:
577     opts, args = getopt.getopt(sys.argv[1:], "hf", ["help"])
578     except getopt.GetoptError,e:
579     print e
580     sys.exit(1)
581    
582     for o,a in opts:
583     if o=="-f":
584     filter_file = True
585    
586     if filter_file:
587     # Filter the specified file and print the result to stdout
588     filename = string.join(args)
589     filterFile(filename)
590     else:
591    
592     if len(args)!=2:
593     sys.stderr.write("%s options input output\n"%(os.path.basename(sys.argv[0])))
594     sys.exit(1)
595    
596     # Filter an entire Python source tree
597     print '"%s" -> "%s"\n'%(args[0],args[1])
598     c=convert(args[0],args[1])
599     print "%d files"%(c)
600    

Properties

Name Value
svn:eol-style native
svn:keywords Author Date Id Revision

  ViewVC Help
Powered by ViewVC 1.1.26