epydoc.docwriter.html

219 """ 220 A class that renders a python module's source code into HTML 221 pages. These HTML pages are intended to be provided along with 222 the API documentation for a module, in case a user wants to learn 223 more about a particular object by examining its source code. 224 Links are therefore generated from the API documentation to the 225 source code pages, and from the source code pages back into the 226 API documentation. 227 228 The HTML generated by C{PythonSourceColorizer} has several notable 229 features: 230 231 - CSS styles are used to color tokens according to their type. 232 (See L{CSS_CLASSES} for a list of the different token types 233 that are identified). 234 235 - Line numbers are included to the left of each line. 236 237 - The first line of each class and function definition includes 238 a link to the API source documentation for that object. 239 240 - The first line of each class and function definition includes 241 an anchor that can be used to link directly to that class or 242 function. 243 244 - If javascript is enabled, and the page is loaded using the 245 anchor for a class or function (i.e., if the url ends in 246 C{'#I{<name>}'}), then that class or function will automatically 247 be highlighted; and all other classes and function definition 248 blocks will be 'collapsed'. These collapsed blocks can be 249 expanded by clicking on them. 250 251 - Unicode input is supported (including automatic detection 252 of C{'coding:'} declarations). 253 254 """ 255 #: A look-up table that is used to determine which CSS class 256 #: should be used to colorize a given token. The following keys 257 #: may be used: 258 #: - Any token name (e.g., C{'STRING'}) 259 #: - Any operator token (e.g., C{'='} or C{'@'}). 260 #: - C{'KEYWORD'} -- Python keywords such as C{'for'} and C{'if'} 261 #: - C{'DEFNAME'} -- the name of a class or function at the top 262 #: of its definition statement. 263 #: - C{'BASECLASS'} -- names of base classes at the top of a class 264 #: definition statement. 265 #: - C{'PARAM'} -- function parameters 266 #: - C{'DOCSTRING'} -- docstrings 267 #: - C{'DECORATOR'} -- decorator names 268 #: If no CSS class can be found for a given token, then it won't 269 #: be marked with any CSS class. 270 CSS_CLASSES = { 271 'NUMBER': 'py-number', 272 'STRING': 'py-string', 273 'COMMENT': 'py-comment', 274 'NAME': 'py-name', 275 'KEYWORD': 'py-keyword', 276 'DEFNAME': 'py-def-name', 277 'BASECLASS': 'py-base-class', 278 'PARAM': 'py-param', 279 'DOCSTRING': 'py-docstring', 280 'DECORATOR': 'py-decorator', 281 'OP': 'py-op', 282 '@': 'py-decorator', 283 } 284 285 #: HTML code for the beginning of a collapsable function or class 286 #: definition block. The block contains two <div>...</div> 287 #: elements -- a collapsed version and an expanded version -- and 288 #: only one of these elements is visible at any given time. By 289 #: default, all definition blocks are expanded. 290 #: 291 #: This string should be interpolated with the following values:: 292 #: (name, indentation, name) 293 #: Where C{name} is the anchor name for the function or class; and 294 #: indentation is a string of whitespace used to indent the 295 #: ellipsis marker in the collapsed version. 296 START_DEF_BLOCK = ( 297 '<div id="%s-collapsed" style="display:none;" ' 298 'pad="%s" indent="%s"></div>' 299 '<div id="%s-expanded">') 300 301 #: HTML code for the end of a collapsable function or class 302 #: definition block. 303 END_DEF_BLOCK = '</div>' 304 305 #: A regular expression used to pick out the unicode encoding for 306 #: the source file. 307 UNICODE_CODING_RE = re.compile(r'.*?\n?.*?coding[:=]\s*([-\w.]+)') 308 309 #: A configuration constant, used to determine whether or not to add 310 #: collapsable <div> elements for definition blocks. 311 ADD_DEF_BLOCKS = True 312 313 #: A configuration constant, used to determine whether or not to 314 #: add line numbers. 315 ADD_LINE_NUMBERS = True 316 317 #: A configuration constant, used to determine whether or not to 318 #: add tooltips for linked names. 319 ADD_TOOLTIPS = True 320 321 #: If true, then try to guess which target is appropriate for 322 #: linked names; if false, then always open a div asking the 323 #: user which one they want. 324 GUESS_LINK_TARGETS = False 325

326 - def __init__(self, module_filename, module_name, 327 docindex=None, url_func=None, name_to_docs=None):

328 """ 329 Create a new HTML colorizer for the specified module. 330 331 @param module_filename: The name of the file containing the 332 module; its text will be loaded from this file. 333 @param module_name: The dotted name of the module; this will 334 be used to create links back into the API source 335 documentation. 336 """ 337 # Get the source version, if possible. 338 try: module_filename = py_src_filename(module_filename) 339 except: pass 340 341 #: The filename of the module we're colorizing. 342 self.module_filename = module_filename 343 344 #: The dotted name of the module we're colorizing. 345 self.module_name = module_name 346 347 #: A docindex, used to create href links from identifiers to 348 #: the API documentation for their values. 349 self.docindex = docindex 350 351 #: A mapping from short names to lists of ValueDoc, used to 352 #: decide which values an identifier might map to when creating 353 #: href links from identifiers to the API docs for their values. 354 self.name_to_docs = name_to_docs 355 356 #: A function that maps APIDoc -> URL, used to create href 357 #: links from identifiers to the API documentation for their 358 #: values. 359 self.url_func = url_func 360 361 #: The index in C{text} of the last character of the last 362 #: token we've processed. 363 self.pos = 0 364 365 #: A list that maps line numbers to character offsets in 366 #: C{text}. In particular, line C{M{i}} begins at character 367 #: C{line_offset[i]} in C{text}. Since line numbers begin at 368 #: 1, the first element of C{line_offsets} is C{None}. 369 self.line_offsets = [] 370 371 #: A list of C{(toktype, toktext)} for all tokens on the 372 #: logical line that we are currently processing. Once a 373 #: complete line of tokens has been collected in C{cur_line}, 374 #: it is sent to L{handle_line} for processing. 375 self.cur_line = [] 376 377 #: A list of the names of the class or functions that include 378 #: the current block. C{context} has one element for each 379 #: level of indentation; C{context[i]} is the name of the class 380 #: or function defined by the C{i}th level of indentation, or 381 #: C{None} if that level of indentation doesn't correspond to a 382 #: class or function definition. 383 self.context = [] 384 385 #: A list, corresponding one-to-one with L{self.context}, 386 #: indicating the type of each entry. Each element of 387 #: C{context_types} is one of: C{'func'}, C{'class'}, C{None}. 388 self.context_types = [] 389 390 #: A list of indentation strings for each of the current 391 #: block's indents. I.e., the current total indentation can 392 #: be found by taking C{''.join(self.indents)}. 393 self.indents = [] 394 395 #: The line number of the line we're currently processing. 396 self.lineno = 0 397 398 #: The name of the class or function whose definition started 399 #: on the previous logical line, or C{None} if the previous 400 #: logical line was not a class or function definition. 401 self.def_name = None 402 403 #: The type of the class or function whose definition started 404 #: on the previous logical line, or C{None} if the previous 405 #: logical line was not a class or function definition. 406 #: Can be C{'func'}, C{'class'}, C{None}. 407 self.def_type = None

408 409

410 - def find_line_offsets(self):

411 """ 412 Construct the L{line_offsets} table from C{self.text}. 413 """ 414 # line 0 doesn't exist; line 1 starts at char offset 0. 415 self.line_offsets = [None, 0] 416 # Find all newlines in `text`, and add an entry to 417 # line_offsets for each one. 418 pos = self.text.find('\n') 419 while pos != -1: 420 self.line_offsets.append(pos+1) 421 pos = self.text.find('\n', pos+1) 422 # Add a final entry, marking the end of the string. 423 self.line_offsets.append(len(self.text))

424

425 - def lineno_to_html(self):

426 template = '%%%ds' % self.linenum_size 427 n = template % self.lineno 428 return '<a name="L%s"></a><tt class="py-lineno">%s</tt>' \ 429 % (self.lineno, n)

430

431 - def colorize(self):

432 """ 433 Return an HTML string that renders the source code for the 434 module that was specified in the constructor. 435 """ 436 # Initialize all our state variables 437 self.pos = 0 438 self.cur_line = [] 439 self.context = [] 440 self.context_types = [] 441 self.indents = [] 442 self.lineno = 1 443 self.def_name = None 444 self.def_type = None 445 446 # Cache, used so we only need to list the target elements once 447 # for each variable. 448 self.doclink_targets_cache = {} 449 450 # Load the module's text. 451 self.text = open(self.module_filename).read() 452 self.text = self.text.expandtabs().rstrip()+'\n' 453 454 # Construct the line_offsets table. 455 self.find_line_offsets() 456 457 num_lines = self.text.count('\n')+1 458 self.linenum_size = len(`num_lines+1`) 459 460 # Call the tokenizer, and send tokens to our `tokeneater()` 461 # method. If anything goes wrong, then fall-back to using 462 # the input text as-is (with no colorization). 463 try: 464 output = StringIO() 465 self.out = output.write 466 tokenize.tokenize(StringIO(self.text).readline, self.tokeneater) 467 html = output.getvalue() 468 except tokenize.TokenError, ex: 469 html = self.text 470 471 # Check for a unicode encoding declaration. 472 m = self.UNICODE_CODING_RE.match(self.text) 473 if m: coding = m.group(1) 474 else: coding = 'iso-8859-1' 475 476 # Decode the html string into unicode, and then encode it back 477 # into ascii, replacing any non-ascii characters with xml 478 # character references. 479 try: 480 html = html.decode(coding).encode('ascii', 'xmlcharrefreplace') 481 except LookupError: 482 coding = 'iso-8859-1' 483 html = html.decode(coding).encode('ascii', 'xmlcharrefreplace') 484 485 # Call expandto. 486 html += PYSRC_EXPANDTO_JAVASCRIPT 487 488 return html

489

490 - def tokeneater(self, toktype, toktext, (srow,scol), (erow,ecol), line):

491 """ 492 A callback function used by C{tokenize.tokenize} to handle 493 each token in the module. C{tokeneater} collects tokens into 494 the C{self.cur_line} list until a complete logical line has 495 been formed; and then calls L{handle_line} to process that line. 496 """ 497 # If we encounter any errors, then just give up. 498 if toktype == token.ERRORTOKEN: 499 raise tokenize.TokenError, toktype 500 501 # Did we skip anything whitespace? If so, add a pseudotoken 502 # for it, with toktype=None. (Note -- this skipped string 503 # might also contain continuation slashes; but I won't bother 504 # to colorize them.) 505 startpos = self.line_offsets[srow] + scol 506 if startpos > self.pos: 507 skipped = self.text[self.pos:startpos] 508 self.cur_line.append( (None, skipped) ) 509 510 # Update our position. 511 self.pos = startpos + len(toktext) 512 513 # Update our current line. 514 self.cur_line.append( (toktype, toktext) ) 515 516 # When we reach the end of a line, process it. 517 if toktype == token.NEWLINE or toktype == token.ENDMARKER: 518 self.handle_line(self.cur_line) 519 self.cur_line = []

520 521 _next_uid = 0 522 523 # [xx] note -- this works with byte strings, not unicode strings! 524 # I may change it to use unicode eventually, but when I do it 525 # needs to be changed all at once.

526 - def handle_line(self, line):

527 """ 528 Render a single logical line from the module, and write the 529 generated HTML to C{self.out}. 530 531 @param line: A single logical line, encoded as a list of 532 C{(toktype,tokttext)} pairs corresponding to the tokens in 533 the line. 534 """ 535 # def_name is the name of the function or class defined by 536 # this line; or None if no funciton or class is defined. 537 def_name = None 538 539 # def_type is the type of the function or class defined by 540 # this line; or None if no funciton or class is defined. 541 def_type = None 542 543 # does this line start a class/func def? 544 starting_def_block = False 545 546 in_base_list = False 547 in_param_list = False 548 in_param_default = 0 549 at_module_top = (self.lineno == 1) 550 551 ended_def_blocks = 0 552 553 # The html output. 554 if self.ADD_LINE_NUMBERS: 555 s = self.lineno_to_html() 556 self.lineno += 1 557 else: 558 s = '' 559 s += ' <tt class="py-line">' 560 561 # Loop through each token, and colorize it appropriately. 562 for i, (toktype, toktext) in enumerate(line): 563 if type(s) is not str: 564 if type(s) is unicode: 565 log.error('While colorizing %s -- got unexpected ' 566 'unicode string' % self.module_name) 567 s = s.encode('ascii', 'xmlcharrefreplace') 568 else: 569 raise ValueError('Unexpected value for s -- %s' % 570 type(s).__name__) 571 572 # For each token, determine its css class and whether it 573 # should link to a url. 574 css_class = None 575 url = None 576 tooltip = None 577 onclick = uid = targets = None # these 3 are used together. 578 579 # Is this token the class name in a class definition? If 580 # so, then make it a link back into the API docs. 581 if i>=2 and line[i-2][1] == 'class': 582 in_base_list = True 583 css_class = self.CSS_CLASSES['DEFNAME'] 584 def_name = toktext 585 def_type = 'class' 586 if 'func' not in self.context_types: 587 cls_name = self.context_name(def_name) 588 url = self.name2url(cls_name) 589 s = self.mark_def(s, cls_name) 590 starting_def_block = True 591 592 # Is this token the function name in a function def? If 593 # so, then make it a link back into the API docs. 594 elif i>=2 and line[i-2][1] == 'def': 595 in_param_list = True 596 css_class = self.CSS_CLASSES['DEFNAME'] 597 def_name = toktext 598 def_type = 'func' 599 if 'func' not in self.context_types: 600 cls_name = self.context_name() 601 func_name = self.context_name(def_name) 602 url = self.name2url(cls_name, def_name) 603 s = self.mark_def(s, func_name) 604 starting_def_block = True 605 606 # For each indent, update the indents list (which we use 607 # to keep track of indentation strings) and the context 608 # list. If this indent is the start of a class or 609 # function def block, then self.def_name will be its name; 610 # otherwise, it will be None. 611 elif toktype == token.INDENT: 612 self.indents.append(toktext) 613 self.context.append(self.def_name) 614 self.context_types.append(self.def_type) 615 616 # When we dedent, pop the last elements off the indents 617 # list and the context list. If the last context element 618 # is a name, then we're ending a class or function def 619 # block; so write an end-div tag. 620 elif toktype == token.DEDENT: 621 self.indents.pop() 622 self.context_types.pop() 623 if self.context.pop(): 624 ended_def_blocks += 1 625 626 # If this token contains whitespace, then don't bother to 627 # give it a css tag. 628 elif toktype in (None, tokenize.NL, token.NEWLINE, 629 token.ENDMARKER): 630 css_class = None 631 632 # Check if the token is a keyword. 633 elif toktype == token.NAME and keyword.iskeyword(toktext): 634 css_class = self.CSS_CLASSES['KEYWORD'] 635 636 elif in_base_list and toktype == token.NAME: 637 css_class = self.CSS_CLASSES['BASECLASS'] 638 639 elif (in_param_list and toktype == token.NAME and 640 not in_param_default): 641 css_class = self.CSS_CLASSES['PARAM'] 642 643 # Class/function docstring. 644 elif (self.def_name and line[i-1][0] == token.INDENT and 645 self.is_docstring(line, i)): 646 css_class = self.CSS_CLASSES['DOCSTRING'] 647 648 # Module docstring. 649 elif at_module_top and self.is_docstring(line, i): 650 css_class = self.CSS_CLASSES['DOCSTRING'] 651 652 # check for decorators?? 653 elif (toktype == token.NAME and 654 ((i>0 and line[i-1][1]=='@') or 655 (i>1 and line[i-1][0]==None and line[i-2][1] == '@'))): 656 css_class = self.CSS_CLASSES['DECORATOR'] 657 658 # If it's a name, try to link it. 659 elif toktype == token.NAME: 660 css_class = self.CSS_CLASSES['NAME'] 661 # If we have a variable named `toktext` in the current 662 # context, then link to that. Note that if we're inside 663 # a function, then that function is our context, not 664 # the namespace that contains it. [xx] this isn't always 665 # the right thing to do. 666 if (self.GUESS_LINK_TARGETS and self.docindex is not None 667 and self.url_func is not None): 668 context = [n for n in self.context if n is not None] 669 container = self.docindex.get_vardoc( 670 DottedName(self.module_name, *context)) 671 if isinstance(container, NamespaceDoc): 672 doc = container.variables.get(toktext) 673 if doc is not None: 674 url = self.url_func(doc) 675 tooltip = str(doc.canonical_name) 676 # Otherwise, check the name_to_docs index to see what 677 # else this name might refer to. 678 if (url is None and self.name_to_docs is not None 679 and self.url_func is not None): 680 docs = self.name_to_docs.get(toktext) 681 if docs: 682 tooltip='\n'.join([str(d.canonical_name) 683 for d in docs]) 684 if len(docs) == 1 and self.GUESS_LINK_TARGETS: 685 url = self.url_func(docs[0]) 686 else: 687 uid, onclick, targets = self.doclink(toktext, docs) 688 689 # For all other tokens, look up the CSS class to use 690 # based on the token's type. 691 else: 692 if toktype == token.OP and toktext in self.CSS_CLASSES: 693 css_class = self.CSS_CLASSES[toktext] 694 elif token.tok_name[toktype] in self.CSS_CLASSES: 695 css_class = self.CSS_CLASSES[token.tok_name[toktype]] 696 else: 697 css_class = None 698 699 # update our status.. 700 if toktext == ':': 701 in_base_list = False 702 in_param_list = False 703 if toktext == '=' and in_param_list: 704 in_param_default = True 705 if in_param_default: 706 if toktext in ('(','[','{'): in_param_default += 1 707 if toktext in (')',']','}'): in_param_default -= 1 708 if toktext == ',' and in_param_default == 1: 709 in_param_default = 0 710 711 # Write this token, with appropriate colorization. 712 if tooltip and self.ADD_TOOLTIPS: 713 tooltip_html = ' title="%s"' % tooltip 714 else: tooltip_html = '' 715 if css_class: css_class_html = ' class="%s"' % css_class 716 else: css_class_html = '' 717 if onclick: 718 if targets: targets_html = ' targets="%s"' % targets 719 else: targets_html = '' 720 s += ('<tt id="%s"%s%s><a%s%s href="#" onclick="%s">' % 721 (uid, css_class_html, targets_html, tooltip_html, 722 css_class_html, onclick)) 723 elif url: 724 if isinstance(url, unicode): 725 url = url.encode('ascii', 'xmlcharrefreplace') 726 s += ('<a%s%s href="%s">' % 727 (tooltip_html, css_class_html, url)) 728 elif css_class_html or tooltip_html: 729 s += '<tt%s%s>' % (tooltip_html, css_class_html) 730 if i == len(line)-1: 731 s += ' </tt>' # Closes <tt class="py-line"> 732 s += cgi.escape(toktext) 733 else: 734 try: 735 s += self.add_line_numbers(cgi.escape(toktext), css_class) 736 except Exception, e: 737 print (toktext, css_class, toktext.encode('ascii')) 738 raise 739 740 if onclick: s += "</a></tt>" 741 elif url: s += '</a>' 742 elif css_class_html or tooltip_html: s += '</tt>' 743 744 if self.ADD_DEF_BLOCKS: 745 for i in range(ended_def_blocks): 746 self.out(self.END_DEF_BLOCK) 747 748 # Strip any empty <tt>s. 749 s = re.sub(r'<tt class="[\w+]"></tt>', '', s) 750 751 # Write the line. 752 self.out(s) 753 754 if def_name and starting_def_block: 755 self.out('</div>') 756 757 # Add div's if we're starting a def block. 758 if (self.ADD_DEF_BLOCKS and def_name and starting_def_block and 759 (line[-2][1] == ':')): 760 indentation = (''.join(self.indents)+' ').replace(' ', '+') 761 linenum_padding = '+'*self.linenum_size 762 name=self.context_name(def_name) 763 self.out(self.START_DEF_BLOCK % (name, linenum_padding, 764 indentation, name)) 765 766 self.def_name = def_name 767 self.def_type = def_type

768

769 - def context_name(self, extra=None):

770 pieces = [n for n in self.context if n is not None] 771 if extra is not None: pieces.append(extra) 772 return '.'.join(pieces)

773

774 - def doclink(self, name, docs):

775 uid = 'link-%s' % self._next_uid 776 self._next_uid += 1 777 context = [n for n in self.context if n is not None] 778 container = DottedName(self.module_name, *context) 779 #else: 780 # container = None 781 targets = ','.join(['%s=%s' % (str(self.doc_descr(d,container)), 782 str(self.url_func(d))) 783 for d in docs]) 784 785 if targets in self.doclink_targets_cache: 786 onclick = ("return doclink('%s', '%s', '%s');" % 787 (uid, name, self.doclink_targets_cache[targets])) 788 return uid, onclick, None 789 else: 790 self.doclink_targets_cache[targets] = uid 791 onclick = ("return doclink('%s', '%s', '%s');" % 792 (uid, name, uid)) 793 return uid, onclick, targets

794

795 - def doc_descr(self, doc, context):

796 name = str(doc.canonical_name) 797 descr = '%s %s' % (self.doc_kind(doc), name) 798 if isinstance(doc, RoutineDoc): 799 descr += '()' 800 return descr

801 802 # [XX] copied streight from html.py; this should be consolidated, 803 # probably into apidoc.

804 - def doc_kind(self, doc):

805 if isinstance(doc, ModuleDoc) and doc.is_package == True: 806 return 'Package' 807 elif (isinstance(doc, ModuleDoc) and 808 doc.canonical_name[0].startswith('script')): 809 return 'Script' 810 elif isinstance(doc, ModuleDoc): 811 return 'Module' 812 elif isinstance(doc, ClassDoc): 813 return 'Class' 814 elif isinstance(doc, ClassMethodDoc): 815 return 'Class Method' 816 elif isinstance(doc, StaticMethodDoc): 817 return 'Static Method' 818 elif isinstance(doc, RoutineDoc): 819 if (self.docindex is not None and 820 isinstance(self.docindex.container(doc), ClassDoc)): 821 return 'Method' 822 else: 823 return 'Function' 824 else: 825 return 'Variable'

826

827 - def mark_def(self, s, name):

828 replacement = ('<a name="%s"></a><div id="%s-def">\\1' 829 '<a class="py-toggle" href="#" id="%s-toggle" ' 830 'onclick="return toggle(\'%s\');">-</a>\\2' % 831 (name, name, name, name)) 832 return re.sub('(.*) (<tt class="py-line">.*)\Z', replacement, s)

833

834 - def is_docstring(self, line, i):

835 if line[i][0] != token.STRING: return False 836 for toktype, toktext in line[i:]: 837 if toktype not in (token.NEWLINE, tokenize.COMMENT, 838 tokenize.NL, token.STRING, None): 839 return False 840 return True

841

842 - def add_line_numbers(self, s, css_class):

843 result = '' 844 start = 0 845 end = s.find('\n')+1 846 while end: 847 result += s[start:end-1] 848 if css_class: result += '</tt>' 849 result += ' </tt>' # py-line 850 result += '\n' 851 if self.ADD_LINE_NUMBERS: 852 result += self.lineno_to_html() 853 result += ' <tt class="py-line">' 854 if css_class: result += '<tt class="%s">' % css_class 855 start = end 856 end = s.find('\n', end)+1 857 self.lineno += 1 858 result += s[start:] 859 return result

860

861 - def name2url(self, class_name, func_name=None):

862 if class_name: 863 class_name = '%s.%s' % (self.module_name, class_name) 864 if func_name: 865 return '%s-class.html#%s' % (class_name, func_name) 866 else: 867 return '%s-class.html' % class_name 868 else: 869 return '%s-module.html#%s' % (self.module_name, func_name)

Source Code for Module epydoc.docwriter.html_colorize