1
2
3
4
5
6
7
8
9 """
10 Epydoc parser for ReStructuredText strings. ReStructuredText is the
11 standard markup language used by the Docutils project.
12 L{parse_docstring()} provides the primary interface to this module; it
13 returns a L{ParsedRstDocstring}, which supports all of the methods
14 defined by L{ParsedDocstring}.
15
16 L{ParsedRstDocstring} is basically just a L{ParsedDocstring} wrapper
17 for the C{docutils.nodes.document} class.
18
19 Creating C{ParsedRstDocstring}s
20 ===============================
21
22 C{ParsedRstDocstring}s are created by the C{parse_document} function,
23 using the C{docutils.core.publish_string()} method, with the following
24 helpers:
25
26 - An L{_EpydocReader} is used to capture all error messages as it
27 parses the docstring.
28 - A L{_DocumentPseudoWriter} is used to extract the document itself,
29 without actually writing any output. The document is saved for
30 further processing. The settings for the writer are copied from
31 C{docutils.writers.html4css1.Writer}, since those settings will
32 be used when we actually write the docstring to html.
33
34 Using C{ParsedRstDocstring}s
35 ============================
36
37 C{ParsedRstDocstring}s support all of the methods defined by
38 C{ParsedDocstring}; but only the following four methods have
39 non-default behavior:
40
41 - L{to_html()<ParsedRstDocstring.to_html>} uses an
42 L{_EpydocHTMLTranslator} to translate the C{ParsedRstDocstring}'s
43 document into an HTML segment.
44 - L{split_fields()<ParsedRstDocstring.split_fields>} uses a
45 L{_SplitFieldsTranslator} to divide the C{ParsedRstDocstring}'s
46 document into its main body and its fields. Special handling
47 is done to account for consolidated fields.
48 - L{summary()<ParsedRstDocstring.summary>} uses a
49 L{_SummaryExtractor} to extract the first sentence from
50 the C{ParsedRstDocstring}'s document.
51 - L{to_plaintext()<ParsedRstDocstring.to_plaintext>} uses
52 C{document.astext()} to convert the C{ParsedRstDocstring}'s
53 document to plaintext.
54
55 @todo: Add ParsedRstDocstring.to_latex()
56 @var CONSOLIDATED_FIELDS: A dictionary encoding the set of
57 'consolidated fields' that can be used. Each consolidated field is
58 marked by a single tag, and contains a single bulleted list, where
59 each list item starts with an identifier, marked as interpreted text
60 (C{`...`}). This module automatically splits these consolidated
61 fields into individual fields. The keys of C{CONSOLIDATED_FIELDS} are
62 the names of possible consolidated fields; and the values are the
63 names of the field tags that should be used for individual entries in
64 the list.
65 """
66 __docformat__ = 'epytext en'
67
68
69 import re, os, os.path
70 from xml.dom.minidom import *
71
72 from docutils.core import publish_string
73 from docutils.writers import Writer
74 from docutils.writers.html4css1 import HTMLTranslator, Writer as HTMLWriter
75 from docutils.writers.latex2e import LaTeXTranslator, Writer as LaTeXWriter
76 from docutils.readers.standalone import Reader as StandaloneReader
77 from docutils.utils import new_document
78 from docutils.nodes import NodeVisitor, Text, SkipChildren
79 from docutils.nodes import SkipNode, TreeCopyVisitor
80 from docutils.frontend import OptionParser
81 from docutils.parsers.rst import directives, roles
82 import docutils.nodes
83 import docutils.transforms.frontmatter
84 import docutils.transforms
85 import docutils.utils
86
87 from epydoc.compat import *
88 from epydoc.markup import *
89 from epydoc.apidoc import ModuleDoc, ClassDoc
90 from epydoc.docwriter.dotgraph import *
91 from epydoc.docwriter.xlink import ApiLinkReader
92 from epydoc.markup.doctest import doctest_to_html, doctest_to_latex, \
93 HTMLDoctestColorizer
94
95
96
97
98 CONSOLIDATED_FIELDS = {
99 'parameters': 'param',
100 'arguments': 'arg',
101 'exceptions': 'except',
102 'variables': 'var',
103 'ivariables': 'ivar',
104 'cvariables': 'cvar',
105 'groups': 'group',
106 'types': 'type',
107 'keywords': 'keyword',
108 }
109
110
111
112
113
114 CONSOLIDATED_DEFLIST_FIELDS = ['param', 'arg', 'var', 'ivar', 'cvar', 'keyword']
115
117 """
118 Parse the given docstring, which is formatted using
119 ReStructuredText; and return a L{ParsedDocstring} representation
120 of its contents.
121 @param docstring: The docstring to parse
122 @type docstring: C{string}
123 @param errors: A list where any errors generated during parsing
124 will be stored.
125 @type errors: C{list} of L{ParseError}
126 @param options: Extra options. Unknown options are ignored.
127 Currently, no extra options are defined.
128 @rtype: L{ParsedDocstring}
129 """
130 writer = _DocumentPseudoWriter()
131 reader = _EpydocReader(errors)
132 publish_string(docstring, writer=writer, reader=reader,
133 settings_overrides={'report_level':10000,
134 'halt_level':10000,
135 'warning_stream':None})
136 return ParsedRstDocstring(writer.document)
137
139 """A reporter that ignores all debug messages. This is used to
140 shave a couple seconds off of epydoc's run time, since docutils
141 isn't very fast about processing its own debug messages."""
142 - def debug(self, *args, **kwargs): pass
143
145 """
146 An encoded version of a ReStructuredText docstring. The contents
147 of the docstring are encoded in the L{_document} instance
148 variable.
149
150 @ivar _document: A ReStructuredText document, encoding the
151 docstring.
152 @type _document: C{docutils.nodes.document}
153 """
165
167
168 visitor = _SplitFieldsTranslator(self._document, errors)
169 self._document.walk(visitor)
170 if len(self._document.children) > 0:
171 return self, visitor.fields
172 else:
173 return None, visitor.fields
174
176
177 visitor = _SummaryExtractor(self._document)
178 try: self._document.walk(visitor)
179 except docutils.nodes.NodeFound: pass
180 return visitor.summary, bool(visitor.other_docs)
181
182
183
184
185
186
187
188
189
190
191 - def to_html(self, docstring_linker, directory=None,
192 docindex=None, context=None, **options):
193
194 visitor = _EpydocHTMLTranslator(self._document, docstring_linker,
195 directory, docindex, context)
196 self._document.walkabout(visitor)
197 return ''.join(visitor.body)
198
199 - def to_latex(self, docstring_linker, **options):
200
201 visitor = _EpydocLaTeXTranslator(self._document, docstring_linker)
202 self._document.walkabout(visitor)
203 return ''.join(visitor.body)
204
205 - def to_plaintext(self, docstring_linker, **options):
206
207 return self._document.astext()
208
209 - def __repr__(self): return '<ParsedRstDocstring: ...>'
210
212 visitor = _TermsExtractor(self._document)
213 self._document.walkabout(visitor)
214 return visitor.terms
215
217 """
218 A reader that captures all errors that are generated by parsing,
219 and appends them to a list.
220 """
221
222
223
224
225
226 version = [int(v) for v in docutils.__version__.split('.')]
227 version += [ 0 ] * (3 - len(version))
228 if version < [0,4,0]:
229 default_transforms = list(ApiLinkReader.default_transforms)
230 try: default_transforms.remove(docutils.transforms.frontmatter.DocInfo)
231 except ValueError: pass
232 else:
236 del version
237
241
251
262
264 """
265 A pseudo-writer for the docutils framework, that can be used to
266 access the document itself. The output of C{_DocumentPseudoWriter}
267 is just an empty string; but after it has been used, the most
268 recently processed document is available as the instance variable
269 C{document}
270
271 @type document: C{docutils.nodes.document}
272 @ivar document: The most recently processed document.
273 """
277
280
282 """
283 A docutils node visitor that extracts the first sentence from
284 the first paragraph in a document.
285 """
290
293
295 if self.summary is not None:
296
297 self.other_docs = True
298 raise docutils.nodes.NodeFound('Found summary')
299
300 summary_pieces = []
301
302
303 for child in node:
304 if isinstance(child, docutils.nodes.Text):
305 m = re.match(r'(\s*[\w\W]*?\.)(\s|$)', child.data)
306 if m:
307 summary_pieces.append(docutils.nodes.Text(m.group(1)))
308 other = child.data[m.end():]
309 if other and not other.isspace():
310 self.other_docs = True
311 break
312 summary_pieces.append(child)
313
314 summary_doc = self.document.copy()
315 summary_para = node.copy()
316 summary_doc[:] = [summary_para]
317 summary_para[:] = summary_pieces
318 self.summary = ParsedRstDocstring(summary_doc)
319
322
324 'Ignore all unknown nodes'
325
327 """
328 A docutils node visitor that extracts the terms from documentation.
329
330 Terms are created using the C{:term:} interpreted text role.
331 """
333 NodeVisitor.__init__(self, document)
334
335 self.terms = None
336 """
337 The terms currently found.
338 @type: C{list}
339 """
340
342 self.terms = []
343 self._in_term = False
344
346 if 'term' in node.get('classes'):
347 self._in_term = True
348
350 if 'term' in node.get('classes'):
351 self._in_term = False
352
354 if self._in_term:
355 doc = self.document.copy()
356 doc[:] = [node.copy()]
357 self.terms.append(ParsedRstDocstring(doc))
358
360 'Ignore all unknown nodes'
361
363 'Ignore all unknown nodes'
364
366 """
367 A docutils translator that removes all fields from a document, and
368 collects them into the instance variable C{fields}
369
370 @ivar fields: The fields of the most recently walked document.
371 @type fields: C{list} of L{Field<markup.Field>}
372 """
373
374 ALLOW_UNMARKED_ARG_IN_CONSOLIDATED_FIELD = True
375 """If true, then consolidated fields are not required to mark
376 arguments with C{`backticks`}. (This is currently only
377 implemented for consolidated fields expressed as definition lists;
378 consolidated fields expressed as unordered lists still require
379 backticks for now."""
380
382 NodeVisitor.__init__(self, document)
383 self._errors = errors
384 self.fields = []
385 self._newfields = {}
386
389
391
392 node.parent.remove(node)
393
394
395 tag = node[0].astext().split(None, 1)
396 tagname = tag[0]
397 if len(tag)>1: arg = tag[1]
398 else: arg = None
399
400
401 fbody = node[1]
402 if arg is None:
403 for (list_tag, entry_tag) in CONSOLIDATED_FIELDS.items():
404 if tagname.lower() == list_tag:
405 try:
406 self.handle_consolidated_field(fbody, entry_tag)
407 return
408 except ValueError, e:
409 estr = 'Unable to split consolidated field '
410 estr += '"%s" - %s' % (tagname, e)
411 self._errors.append(ParseError(estr, node.line,
412 is_fatal=0))
413
414
415 if not self._newfields.has_key(tagname.lower()):
416 newfield = Field('newfield', tagname.lower(),
417 parse(tagname, 'plaintext'))
418 self.fields.append(newfield)
419 self._newfields[tagname.lower()] = 1
420
421 self._add_field(tagname, arg, fbody)
422
424 field_doc = self.document.copy()
425 for child in fbody: field_doc.append(child)
426 field_pdoc = ParsedRstDocstring(field_doc)
427 self.fields.append(Field(tagname, arg, field_pdoc))
428
430
431
432 node.parent.remove(node)
433
450
452
453
454
455 n = 0
456 _BAD_ITEM = ("list item %d is not well formed. Each item must "
457 "consist of a single marked identifier (e.g., `x`), "
458 "optionally followed by a colon or dash and a "
459 "description.")
460 for item in items:
461 n += 1
462 if item.tagname != 'list_item' or len(item) == 0:
463 raise ValueError('bad bulleted list (bad child %d).' % n)
464 if item[0].tagname != 'paragraph':
465 if item[0].tagname == 'definition_list':
466 raise ValueError(('list item %d contains a definition '+
467 'list (it\'s probably indented '+
468 'wrong).') % n)
469 else:
470 raise ValueError(_BAD_ITEM % n)
471 if len(item[0]) == 0:
472 raise ValueError(_BAD_ITEM % n)
473 if item[0][0].tagname != 'title_reference':
474 raise ValueError(_BAD_ITEM % n)
475
476
477 for item in items:
478
479 arg = item[0][0].astext()
480
481
482 fbody = item[:]
483 fbody[0] = fbody[0].copy()
484 fbody[0][:] = item[0][1:]
485
486
487 if (len(fbody[0]) > 0 and
488 isinstance(fbody[0][0], docutils.nodes.Text)):
489 child = fbody[0][0]
490 if child.data[:1] in ':-':
491 child.data = child.data[1:].lstrip()
492 elif child.data[:2] in (' -', ' :'):
493 child.data = child.data[2:].lstrip()
494
495
496 self._add_field(tagname, arg, fbody)
497
499
500 n = 0
501 _BAD_ITEM = ("item %d is not well formed. Each item's term must "
502 "consist of a single marked identifier (e.g., `x`), "
503 "optionally followed by a space, colon, space, and "
504 "a type description.")
505 for item in items:
506 n += 1
507 if (item.tagname != 'definition_list_item' or len(item) < 2 or
508 item[0].tagname != 'term' or
509 item[-1].tagname != 'definition'):
510 raise ValueError('bad definition list (bad child %d).' % n)
511 if len(item) > 3:
512 raise ValueError(_BAD_ITEM % n)
513 if not ((item[0][0].tagname == 'title_reference') or
514 (self.ALLOW_UNMARKED_ARG_IN_CONSOLIDATED_FIELD and
515 isinstance(item[0][0], docutils.nodes.Text))):
516 raise ValueError(_BAD_ITEM % n)
517 for child in item[0][1:]:
518 if child.astext() != '':
519 raise ValueError(_BAD_ITEM % n)
520
521
522 for item in items:
523
524 arg = item[0][0].astext()
525 fbody = item[-1]
526 self._add_field(tagname, arg, fbody)
527
528 if len(item) == 3:
529 type_descr = item[1]
530 self._add_field('type', arg, type_descr)
531
533 'Ignore all unknown nodes'
534
539
541 settings = None
543
544 if self.settings is None:
545 settings = OptionParser([LaTeXWriter()]).get_default_values()
546 self.__class__.settings = settings
547 document.settings = self.settings
548
549 LaTeXTranslator.__init__(self, document)
550 self._linker = docstring_linker
551
552
553
554
555 self.section_level = 3
556 self._section_number = [0]*self.section_level
557
558
560 target = self.encode(node.astext())
561 xref = self._linker.translate_identifier_xref(target, target)
562 self.body.append(xref)
563 raise SkipNode()
564
567
568
570 log.warning("Ignoring dotgraph in latex output (dotgraph "
571 "rendering for latex not implemented yet).")
572 raise SkipNode()
573
575 self.body.append(doctest_to_latex(node[0].astext()))
576 raise SkipNode()
577
579 settings = None
580 - def __init__(self, document, docstring_linker, directory,
581 docindex, context):
595
596
598 target = self.encode(node.astext())
599 xref = self._linker.translate_identifier_xref(target, target)
600 self.body.append(xref)
601 raise SkipNode()
602
608
611
612 - def starttag(self, node, tagname, suffix='\n', **attributes):
613 """
614 This modified version of starttag makes a few changes to HTML
615 tags, to prevent them from conflicting with epydoc. In particular:
616 - existing class attributes are prefixed with C{'rst-'}
617 - existing names are prefixed with C{'rst-'}
618 - hrefs starting with C{'#'} are prefixed with C{'rst-'}
619 - hrefs not starting with C{'#'} are given target='_top'
620 - all headings (C{<hM{n}>}) are given the css class C{'heading'}
621 """
622
623 attr_dicts = [attributes]
624 if isinstance(node, docutils.nodes.Node):
625 attr_dicts.append(node.attributes)
626 if isinstance(node, dict):
627 attr_dicts.append(node)
628
629
630
631 for attr_dict in attr_dicts:
632 for (key, val) in attr_dict.items():
633
634
635 if key.lower() in ('class', 'id', 'name'):
636 attr_dict[key] = 'rst-%s' % val
637 elif key.lower() in ('classes', 'ids', 'names'):
638 attr_dict[key] = ['rst-%s' % cls for cls in val]
639 elif key.lower() == 'href':
640 if attr_dict[key][:1]=='#':
641 attr_dict[key] = '#rst-%s' % attr_dict[key][1:]
642 else:
643
644
645 attr_dict['target'] = '_top'
646
647
648 if re.match(r'^h\d+$', tagname):
649 attributes['class'] = ' '.join([attributes.get('class',''),
650 'heading']).strip()
651
652 return HTMLTranslator.starttag(self, node, tagname, suffix,
653 **attributes)
654
656 if self._directory is None: return
657
658
659 graph = node.graph(self._docindex, self._context, self._linker)
660 if graph is None: return
661
662
663 image_url = '%s.gif' % graph.uid
664 image_file = os.path.join(self._directory, image_url)
665 self.body.append(graph.to_html(image_file, image_url))
666 raise SkipNode()
667
675
676
677 -def python_code_directive(name, arguments, options, content, lineno,
678 content_offset, block_text, state, state_machine):
679 """
680 A custom restructuredtext directive which can be used to display
681 syntax-highlighted Python code blocks. This directive takes no
682 arguments, and the body should contain only Python code. This
683 directive can be used instead of doctest blocks when it is
684 inconvenient to list prompts on each line, or when you would
685 prefer that the output not contain prompts (e.g., to make
686 copy/paste easier).
687 """
688 required_arguments = 0
689 optional_arguments = 0
690
691 text = '\n'.join(content)
692 node = docutils.nodes.doctest_block(text, text, codeblock=True)
693 return [ node ]
694
695 python_code_directive.arguments = (0, 0, 0)
696 python_code_directive.content = True
697
698 directives.register_directive('python', python_code_directive)
699
700 -def term_role(name, rawtext, text, lineno, inliner,
701 options={}, content=[]):
702
703 text = docutils.utils.unescape(text)
704 node = docutils.nodes.emphasis(rawtext, text, **options)
705 node.attributes['classes'].append('term')
706
707 return [node], []
708
709 roles.register_local_role('term', term_role)
710
711
712
713
714
715
717 """
718 A custom docutils node that should be rendered using Graphviz dot.
719 This node does not directly store the graph; instead, it stores a
720 pointer to a function that can be used to generate the graph.
721 This allows the graph to be built based on information that might
722 not be available yet at parse time. This graph generation
723 function has the following signature:
724
725 >>> def generate_graph(docindex, context, linker, *args):
726 ... 'generates and returns a new DotGraph'
727
728 Where C{docindex} is a docindex containing the documentation that
729 epydoc has built; C{context} is the C{APIDoc} whose docstring
730 contains this dotgraph node; C{linker} is a L{DocstringLinker}
731 that can be used to resolve crossreferences; and C{args} is any
732 extra arguments that are passed to the C{dotgraph} constructor.
733 """
734 - def __init__(self, generate_graph_func, *generate_graph_args):
735 docutils.nodes.image.__init__(self)
736 self.graph_func = generate_graph_func
737 self.args = generate_graph_args
738 - def graph(self, docindex, context, linker):
739 return self.graph_func(docindex, context, linker, *self.args)
740
742 """A directive option spec for the orientation of a graph."""
743 argument = argument.lower().strip()
744 if argument == 'right': return 'LR'
745 if argument == 'left': return 'RL'
746 if argument == 'down': return 'TB'
747 if argument == 'up': return 'BT'
748 raise ValueError('%r unknown; choose from left, right, up, down' %
749 argument)
750
751 -def digraph_directive(name, arguments, options, content, lineno,
752 content_offset, block_text, state, state_machine):
753 """
754 A custom restructuredtext directive which can be used to display
755 Graphviz dot graphs. This directive takes a single argument,
756 which is used as the graph's name. The contents of the directive
757 are used as the body of the graph. Any href attributes whose
758 value has the form <name> will be replaced by the URL of the object
759 with that name. Here's a simple example::
760
761 .. digraph:: example_digraph
762 a -> b -> c
763 c -> a [dir=\"none\"]
764 """
765 if arguments: title = arguments[0]
766 else: title = ''
767 return [ dotgraph(_construct_digraph, title, options.get('caption'),
768 '\n'.join(content)) ]
769 digraph_directive.arguments = (0, 1, True)
770 digraph_directive.options = {'caption': directives.unchanged}
771 digraph_directive.content = True
772 directives.register_directive('digraph', digraph_directive)
773
780
781 -def classtree_directive(name, arguments, options, content, lineno,
782 content_offset, block_text, state, state_machine):
783 """
784 A custom restructuredtext directive which can be used to
785 graphically display a class hierarchy. If one or more arguments
786 are given, then those classes and all their descendants will be
787 displayed. If no arguments are given, and the directive is in a
788 class's docstring, then that class and all its descendants will be
789 displayed. It is an error to use this directive with no arguments
790 in a non-class docstring.
791
792 Options:
793 - C{:dir:} -- Specifies the orientation of the graph. One of
794 C{down}, C{right} (default), C{left}, C{up}.
795 """
796 return [ dotgraph(_construct_classtree, arguments, options) ]
797 classtree_directive.arguments = (0, 1, True)
798 classtree_directive.options = {'dir': _dir_option}
799 classtree_directive.content = False
800 directives.register_directive('classtree', classtree_directive)
801
803 """Graph generator for L{classtree_directive}"""
804 if len(arguments) == 1:
805 bases = [docindex.find(name, context) for name in
806 arguments[0].replace(',',' ').split()]
807 bases = [d for d in bases if isinstance(d, ClassDoc)]
808 elif isinstance(context, ClassDoc):
809 bases = [context]
810 else:
811 log.warning("Could not construct class tree: you must "
812 "specify one or more base classes.")
813 return None
814
815 return class_tree_graph(bases, linker, context, **options)
816
817 -def packagetree_directive(name, arguments, options, content, lineno,
818 content_offset, block_text, state, state_machine):
819 """
820 A custom restructuredtext directive which can be used to
821 graphically display a package hierarchy. If one or more arguments
822 are given, then those packages and all their submodules will be
823 displayed. If no arguments are given, and the directive is in a
824 package's docstring, then that package and all its submodules will
825 be displayed. It is an error to use this directive with no
826 arguments in a non-package docstring.
827
828 Options:
829 - C{:dir:} -- Specifies the orientation of the graph. One of
830 C{down}, C{right} (default), C{left}, C{up}.
831 """
832 return [ dotgraph(_construct_packagetree, arguments, options) ]
833 packagetree_directive.arguments = (0, 1, True)
834 packagetree_directive.options = {
835 'dir': _dir_option,
836 'style': lambda a:directives.choice(a.lower(), ('uml', 'tree'))}
837 packagetree_directive.content = False
838 directives.register_directive('packagetree', packagetree_directive)
839
841 """Graph generator for L{packagetree_directive}"""
842 if len(arguments) == 1:
843 packages = [docindex.find(name, context) for name in
844 arguments[0].replace(',',' ').split()]
845 packages = [d for d in packages if isinstance(d, ModuleDoc)]
846 elif isinstance(context, ModuleDoc):
847 packages = [context]
848 else:
849 log.warning("Could not construct package tree: you must "
850 "specify one or more root packages.")
851 return None
852
853 return package_tree_graph(packages, linker, context, **options)
854
855 -def importgraph_directive(name, arguments, options, content, lineno,
856 content_offset, block_text, state, state_machine):
858 importgraph_directive.options = {'dir': _dir_option}
859 importgraph_directive.content = False
860 directives.register_directive('importgraph', importgraph_directive)
861
863 """Graph generator for L{importgraph_directive}"""
864 modules = [d for d in docindex.root if isinstance(d, ModuleDoc)]
865 return import_graph(modules, docindex, linker, context, **options)
866
867 -def callgraph_directive(name, arguments, options, content, lineno,
868 content_offset, block_text, state, state_machine):
870 callgraph_directive.arguments = (0, 1, True)
871 callgraph_directive.options = {'dir': _dir_option,
872 'add_callers': directives.flag,
873 'add_callees': directives.flag}
874 callgraph_directive.content = False
875 directives.register_directive('callgraph', callgraph_directive)
876
878 """Graph generator for L{callgraph_directive}"""
879 if len(arguments) == 1:
880 docs = [docindex.find(name, context) for name in
881 arguments[0].replace(',',' ').split()]
882 docs = [doc for doc in docs if doc is not None]
883 else:
884 docs = [context]
885 return call_graph(docs, docindex, linker, context, **options)
886