nltk.test.doctest

1 #!/usr/bin/env python2.4 2 # 3 # A Test Driver for Doctest 4 # Author: Edward Loper <[email protected]> 5 # 6 # Provided as-is; use at your own risk; no warranty; no promises; enjoy! 7 8 """ 9 A driver for testing interactive python examples in text files and 10 docstrings. This doctest driver performs three functions: 11 12 - checking: Runs the interactive examples, and reports any examples 13 whose actual output does not match their expected output. 14 15 - debugging: Runs the interactive examples, and enters the debugger 16 whenever an example's actual output does not match its expected 17 output. 18 19 - updating: Runs the interactive examples, and replaces the expected 20 output with the actual output whenever they don't match. This is 21 used to update the output for new or out-of-date examples. 22 23 A number of other flags can be given; call the driver with the 24 `--help` option for a complete list. 25 """ 26 27 import os, os.path, sys, unittest, pdb, bdb, re, tempfile, traceback 28 import textwrap 29 from doctest import * 30 from doctest import DocTestCase, DocTestRunner 31 from optparse import OptionParser, OptionGroup, Option 32 from StringIO import StringIO 33 import coverage 34 35 # Use local NLTK. 36 root_dir = os.path.abspath(os.path.join(sys.path[0], '..', '..')) 37 sys.path.insert(0, root_dir) 38 39 __version__ = '0.1' 40 41 ########################################################################### 42 # Monkey-Patch to fix Doctest 43 ########################################################################### 44 45 # The original version of this class has a bug that interferes with 46 # code coverage functions. See: http://tinyurl.com/2htvfx

47 -class _OutputRedirectingPdb(pdb.Pdb):

48 - def __init__(self, out):

49 self.__out = out 50 self.__debugger_used = False 51 pdb.Pdb.__init__(self)

52

53 - def set_trace(self):

54 self.__debugger_used = True 55 pdb.Pdb.set_trace(self)

56

57 - def set_continue(self):

58 # Calling set_continue unconditionally would break unit test coverage 59 # reporting, as Bdb.set_continue calls sys.settrace(None). 60 if self.__debugger_used: 61 pdb.Pdb.set_continue(self) 62

63 - def trace_dispatch(self, *args):

64 save_stdout = sys.stdout 65 sys.stdout = self.__out 66 pdb.Pdb.trace_dispatch(self, *args) 67 sys.stdout = save_stdout

68 69 # Do the actual monkey-patching. 70 import doctest 71 doctest._OutputRedirectingPdb = _OutputRedirectingPdb 72 73 ########################################################################### 74 # Utility Functions 75 ########################################################################### 76 # These are copied from doctest; I don't import them because they're 77 # private. See the versions in doctest for docstrings & comments. 78

79 -def _exception_traceback(exc_info):

80 excout = StringIO() 81 exc_type, exc_val, exc_tb = exc_info 82 traceback.print_exception(exc_type, exc_val, exc_tb, file=excout) 83 return excout.getvalue()

84

85 -class _SpoofOut(StringIO):

86 - def getvalue(self):

87 result = StringIO.getvalue(self) 88 if result and not result.endswith("\n"): 89 result += "\n" 90 if hasattr(self, "softspace"): 91 del self.softspace 92 return result

93

94 - def truncate(self, size=None):

95 StringIO.truncate(self, size) 96 if hasattr(self, "softspace"): 97 del self.softspace

98 99 ########################################################################### 100 # MyParser 101 ########################################################################### 102

103 -class MyDocTestParser(DocTestParser):

104 PYLISTING_RE = re.compile(r''' 105 (^\.\.[ ]*pylisting::[ ]?\S*\n # directive 106 (?:[ ]*\n| # blank line or 107 [ ]+.*\n)*) # indented line 108 ''', re.VERBOSE+re.MULTILINE) 109 110 PYLISTING_EX = re.compile(r''' 111 (?:^[^ ].*\n # non-blank line 112 (?:[ ]*\n | # blank line or 113 [ ]+.*\n)*) # indented line 114 ''', re.VERBOSE+re.MULTILINE) 115 116 DOCTEST_OPTION_RE = re.compile(r''' 117 ^[ ]*:\w+:.*\n # :option: 118 (.*\S.*)* # non-blank lines 119 ''', re.VERBOSE+re.MULTILINE) 120

121 - def parse(self, string, name='<string>'):

122 output = [] 123 lineno_offset = 0 124 125 for piecenum, piece in enumerate(self.PYLISTING_RE.split(string)): 126 for example in DocTestParser.parse(self, piece, name): 127 if isinstance(example, Example): 128 example.lineno += lineno_offset 129 output.append(example) 130 131 # If we're inside a pylisting, then convert any 132 # subpieces that are not marked by python prompts into 133 # examples with an expected output of ''. 134 elif piecenum%2 == 1 and example.strip(): 135 output.append(example[:example.find('\n')]) 136 # order matters here: 137 pysrc = example[example.find('\n'):] 138 pysrc = self.DOCTEST_OPTION_RE.sub('', pysrc) 139 pysrc = textwrap.dedent(pysrc) 140 141 #for ex in self.PYLISTING_EX.findall(pysrc): 142 for ex in split_pysrc_into_statements(pysrc): 143 source = ex.strip() 144 if not source: continue 145 want = '' 146 exc_msg = None 147 indent = 4 # close enough. 148 lineno = lineno_offset # Not quite right! 149 options = self._find_options(source, name, lineno) 150 output.append(Example(source, want, exc_msg, 151 lineno, indent, options)) 152 else: 153 output.append(example) 154 155 lineno_offset += piece.count('\n') 156 157 # For debugging: 158 #for ex in output: 159 # if isinstance(ex, Example): 160 # print '-'*70 161 # print ex.source 162 #output = [] 163 164 return output

165

166 - def get_examples(self, string, name='<string>'):

167 examples = [] 168 ignore = False 169 170 for x in self.parse(string, name): 171 if isinstance(x, Example): 172 if not ignore: 173 examples.append(x) 174 else: 175 #print '.. doctest-ignore:: %s' % x.source.strip()[:50] 176 pass 177 else: 178 if re.search(r'\.\.\s*doctest-ignore::?\s*$', x): 179 ignore = True 180 elif x.strip(): 181 ignore = False 182 return examples

183 184 ########################################################################### 185 # Update Runner 186 ########################################################################### 187

188 -class UpdateRunner(DocTestRunner):

189 """ 190 A subclass of `DocTestRunner` that checks the output of each 191 example, and replaces the expected output with the actual output 192 for any examples that fail. 193 194 `UpdateRunner` can be used: 195 - To automatically fill in the expected output for new examples. 196 - To correct examples whose output has become out-of-date. 197 198 However, care must be taken not to update an example's expected 199 output with an incorrect value. 200 """

201 - def __init__(self, verbose=False, mark_updates=False):

202 '''Construct a new update runner''' 203 self._mark_updates = mark_updates 204 DocTestRunner.__init__(self, verbose=verbose)

205

206 - def run(self, test, compileflags=None, out=None, clear_globs=True):

207 '''Run the update runner''' 208 self._new_want = {} 209 (f,t) = DocTestRunner.run(self, test, compileflags, out, clear_globs) 210 211 # Update the test's docstring, and the lineno's of the 212 # examples, by breaking it into lines and replacing the old 213 # expected outputs with the new expected outputs. 214 old_lines = test.docstring.split('\n') 215 new_lines = [] 216 lineno = 0 217 offset = 0 218 219 for example in test.examples: 220 # Copy the lines up through the start of the example's 221 # output from old_lines to new_lines. 222 got_start = example.lineno + example.source.count('\n') 223 new_lines += old_lines[lineno:got_start] 224 lineno = got_start 225 # Do a sanity check to make sure we're at the right lineno 226 # (In particular, check that the example's expected output 227 # appears in old_lines where we expect it to appear.) 228 if example.want: 229 assert (example.want.split('\n')[0] == 230 old_lines[lineno][example.indent:]), \ 231 'Line number mismatch at %d' % lineno 232 # Skip over the old expected output. 233 old_len = example.want.count('\n') 234 lineno += old_len 235 # Mark any changes we make. 236 if self._mark_updates and example in self._new_want: 237 new_lines.append(' '*example.indent + '... ' + 238 '# [!!] OUTPUT AUTOMATICALLY UPDATED [!!]') 239 # Add the new expected output. 240 new_want = self._new_want.get(example, example.want) 241 if new_want: 242 new_want = '\n'.join([' '*example.indent+l 243 for l in new_want[:-1].split('\n')]) 244 new_lines.append(new_want) 245 # Update the example's want & lieno fields 246 example.want = new_want 247 example.lineno += offset 248 offset += example.want.count('\n') - old_len 249 # Add any remaining lines 250 new_lines += old_lines[lineno:] 251 252 # Update the test's docstring. 253 test.docstring = '\n'.join(new_lines) 254 255 # Return failures & tries 256 return (f,t)

257

258 - def report_start(self, out, test, example):

259 pass

260

261 - def report_success(self, out, test, example, got):

262 pass

263

264 - def report_unexpected_exception(self, out, test, example, exc_info):

265 replacement = _exception_traceback(exc_info) 266 self._new_want[example] = replacement 267 if self._verbose: 268 self._report_replacement(out, test, example, replacement)

269

270 - def report_failure(self, out, test, example, got):

271 self._new_want[example] = got 272 if self._verbose: 273 self._report_replacement(out, test, example, got)

274

275 - def _report_replacement(self, out, test, example, replacement):

276 want = '\n'.join([' '+l for l in example.want.split('\n')[:-1]]) 277 repl = '\n'.join([' '+l for l in replacement.split('\n')[:-1]]) 278 if want and repl: 279 diff = 'Replacing:\n%s\nWith:\n%s\n' % (want, repl) 280 elif want: 281 diff = 'Removing:\n%s\n' % want 282 elif repl: 283 diff = 'Adding:\n%s\n' % repl 284 out(self._header(test, example) + diff)

285 286 DIVIDER = '-'*70

287 - def _header(self, test, example):

288 if test.filename is None: 289 tag = ("On line #%s of %s" % 290 (example.lineno+1, test.name)) 291 elif test.lineno is None: 292 tag = ("On line #%s of %s in %s" % 293 (example.lineno+1, test.name, test.filename)) 294 else: 295 lineno = test.lineno+example.lineno+1 296 tag = ("On line #%s of %s (%s)" % 297 (lineno, test.filename, test.name)) 298 source_lines = example.source.rstrip().split('\n') 299 return (self.DIVIDER + '\n' + tag + '\n' + 300 ' >>> %s\n' % source_lines[0] + 301 ''.join([' ... %s\n' % l for l in source_lines[1:]]))

302 303 ########################################################################### 304 # Debugger 305 ########################################################################### 306

307 -def _indent(s, indent=4):

308 return re.sub('(?m)^(?!$)', indent*' ', s)

309 310 import keyword, token, tokenize

311 -class Debugger:

312 # Just using this for reporting: 313 runner = DocTestRunner() 314

315 - def __init__(self, checker=None, set_trace=None):

316 if checker is None: 317 checker = OutputChecker() 318 self.checker = checker 319 if set_trace is None: 320 set_trace = pdb.Pdb().set_trace 321 self.set_trace = set_trace

322

323 - def _check_output(self, example):

324 want = example.want 325 optionflags = self._get_optionflags(example) 326 got = sys.stdout.getvalue() 327 sys.stdout.truncate(0) 328 if not self.checker.check_output(want, got, optionflags): 329 self.runner.report_failure(self.save_stdout.write, 330 self.test, example, got) 331 return False 332 else: 333 return True

334

335 - def _check_exception(self, example):

336 want_exc_msg = example.exc_msg 337 optionflags = self._get_optionflags(example) 338 exc_info = sys.exc_info() 339 got_exc_msg = traceback.format_exception_only(*exc_info[:2])[-1] 340 if not self.checker.check_output(want_exc_msg, got_exc_msg, 341 optionflags): 342 got = _exception_traceback(exc_info) 343 self.runner.report_failure(self.save_stdout.write, 344 self.test, example, got) 345 return False 346 else: 347 return True

348

349 - def _print_if_not_none(self, *args):

350 if args == (None,): 351 pass 352 elif len(args) == 1: 353 print `args[0]` 354 else: 355 print `args` # not quite right: >>> 1,

356

357 - def _comment_line(self, line):

358 "Return a commented form of the given line" 359 line = line.rstrip() 360 if line: 361 return '# '+line 362 else: 363 return '#'

364

365 - def _script_from_examples(self, s):

366 output = [] 367 examplenum = 0 368 for piece in MyDocTestParser().parse(s): 369 if isinstance(piece, Example): 370 self._script_from_example(piece, examplenum, output) 371 examplenum += 1 372 else: 373 # Add non-example text. 374 output += [self._comment_line(l) 375 for l in piece.split('\n')[:-1]] 376 # Combine the output, and return it. 377 return '\n'.join(output)

378 379 _CHK_OUT = 'if not CHECK_OUTPUT(__examples__[%d]): __set_trace__()' 380 _CHK_EXC = 'if not CHECK_EXCEPTION(__examples__[%d]): __set_trace__()' 381

382 - def _script_from_example(self, example, i, output):

383 source = self._simulate_compile_singlemode(example.source)[:-1] 384 385 if example.exc_msg is None: 386 output.append(source) 387 output.append(self._CHK_OUT % i) 388 else: 389 output.append('try:') 390 output.append(_indent(source)) 391 output.append(' '+self._CHK_OUT % i) 392 output.append('except:') 393 output.append(' '+self._CHK_EXC % i)

394

395 - def _simulate_compile_singlemode(self, s):

396 # Calculate line offsets 397 lines = [0, 0] 398 pos = 0 399 while 1: 400 pos = s.find('\n', pos)+1 401 if not pos: break 402 lines.append(pos) 403 lines.append(len(s)) 404 405 oldpos = 0 406 parenlevel = 0 407 deflevel = 0 408 output = [] 409 stmt = [] 410 411 text = StringIO(s) 412 tok_gen = tokenize.generate_tokens(text.readline) 413 for toktype, tok, (srow,scol), (erow,ecol), line in tok_gen: 414 newpos = lines[srow] + scol 415 stmt.append(s[oldpos:newpos]) 416 if tok != '': 417 stmt.append(tok) 418 oldpos = newpos + len(tok) 419 420 # Update the paren level. 421 if tok in '([{': 422 parenlevel += 1 423 if tok in '}])': 424 parenlevel -= 1 425 426 if tok in ('def', 'class') and deflevel == 0: 427 deflevel = 1 428 if deflevel and toktype == token.INDENT: 429 deflevel += 1 430 if deflevel and toktype == token.DEDENT: 431 deflevel -= 1 432 433 # Are we starting a statement? 434 if ((toktype in (token.NEWLINE, tokenize.NL, tokenize.COMMENT, 435 token.INDENT, token.ENDMARKER) or 436 tok==':') and parenlevel == 0): 437 if deflevel == 0 and self._is_expr(stmt[1:-2]): 438 output += stmt[0] 439 output.append('__print__((') 440 output += stmt[1:-2] 441 output.append('))') 442 output += stmt[-2:] 443 else: 444 output += stmt 445 stmt = [] 446 return ''.join(output)

447

448 - def _is_expr(self, stmt):

449 stmt = [t for t in stmt if t] 450 if not stmt: 451 return False 452 453 # An assignment signifies a non-exception, *unless* it 454 # appears inside of parens (eg, ``f(x=1)``.) 455 parenlevel = 0 456 for tok in stmt: 457 if tok in '([{': parenlevel += 1 458 if tok in '}])': parenlevel -= 1 459 if (parenlevel == 0 and 460 tok in ('=', '+=', '-=', '*=', '/=', '%=', '&=', '+=', 461 '^=', '<<=', '>>=', '**=', '//=')): 462 return False 463 464 # Any keywords *except* "not", "or", "and", "lambda", "in", "is" 465 # signifies a non-expression. 466 if stmt[0] in ("assert", "break", "class", "continue", "def", 467 "del", "elif", "else", "except", "exec", 468 "finally", "for", "from", "global", "if", 469 "import", "pass", "print", "raise", "return", 470 "try", "while", "yield"): 471 return False 472 return True

473

474 - def _get_optionflags(self, example):

475 optionflags = 0 476 for (flag, val) in example.options.items(): 477 if val: 478 optionflags |= flag 479 else: 480 optionflags &= ~flag 481 return optionflags

482

483 - def debug(self, test, pm=False):

484 self.test = test 485 486 # Save the old stdout 487 self.save_stdout = sys.stdout 488 489 # Convert the source docstring to a script. 490 script = self._script_from_examples(test.docstring) 491 492 # Create a debugger. 493 debugger = _OutputRedirectingPdb(sys.stdout) 494 495 # Patch pdb.set_trace to restore sys.stdout during interactive 496 # debugging (so it's not still redirected to self._fakeout). 497 save_set_trace = pdb.set_trace 498 pdb.set_trace = debugger.set_trace 499 500 # Write the script to a temporary file. Note that 501 # tempfile.NameTemporaryFile() cannot be used. As the docs 502 # say, a file so created cannot be opened by name a second 503 # time on modern Windows boxes, and execfile() needs to open 504 # it. 505 srcfilename = tempfile.mktemp(".py", "doctestdebug_") 506 f = open(srcfilename, 'w') 507 f.write(script) 508 f.close() 509 510 # Set up the globals 511 test.globs['CHECK_OUTPUT'] = self._check_output 512 test.globs['CHECK_EXCEPTION'] = self._check_exception 513 test.globs['__print__'] = self._print_if_not_none 514 test.globs['__set_trace__'] = debugger.set_trace 515 test.globs['__examples__'] = self.test.examples 516 try: 517 if pm is False: 518 debugger.run("execfile(%r)" % srcfilename, 519 test.globs, test.globs) 520 else: 521 try: 522 sys.stdout = _SpoofOut() 523 try: 524 execfile(srcfilename, test.globs) 525 except bdb.BdbQuit: 526 return 527 except: 528 sys.stdout = self.save_stdout 529 exc_info = sys.exc_info() 530 exc_msg = traceback.format_exception_only( 531 exc_info[0], exc_info[1])[-1] 532 self.save_stdout.write(self.runner.DIVIDER+'\n') 533 self.save_stdout.write('Unexpected exception:\n' + 534 _indent(exc_msg)) 535 raise 536 #self.post_mortem(debugger, exc_info[2]) 537 finally: 538 sys.stdout = self.save_stdout 539 finally: 540 sys.set_trace = save_set_trace 541 os.remove(srcfilename)

542

543 - def post_mortem(self, debugger, t):

544 debugger.reset() 545 while t.tb_next is not None: 546 t = t.tb_next 547 debugger.interaction(t.tb_frame, t)

548 549 ########################################################################### 550 # Helper functions 551 ########################################################################### 552 553 # Name can be: 554 # - The filename of a text file 555 # - The filename of a python file 556 # - The dotted name of a python module 557 558 # Return a list of test!

559 -def find(name):

560 # Check for test names 561 if ':' in name: 562 (name, testname) = name.split(':') 563 else: 564 testname = None 565 566 if os.path.exists(name): 567 filename = os.path.normpath(os.path.abspath(name)) 568 ext = os.path.splitext(filename)[-1] 569 if (ext[-3:] != '.py' and ext[-4:-1] != '.py'): 570 # It's a text file; return the filename. 571 if testname is not None: 572 raise ValueError("test names can't be specified " 573 "for text files") 574 s = open(filename).read() 575 test = MyDocTestParser().get_doctest(s, {}, name, filename, 0) 576 return [test] 577 else: 578 # It's a python file; import it. Make sure to set the 579 # path correctly. 580 basedir, modname = find_module_from_filename(filename) 581 orig_path = sys.path[:] 582 try: 583 sys.path.insert(0, basedir) 584 module = import_from_name(modname) 585 finally: 586 sys.path[:] = orig_path 587 else: 588 module = import_from_name(name) 589 590 # Find tests. 591 tests = DocTestFinder().find(module) 592 if testname is not None: 593 testname = '%s.%s' % (module.__name__, testname) 594 tests = [t for t in tests if t.name.startswith(testname)] 595 if len(tests) == 0: 596 raise ValueError("test not found") 597 return tests

598

599 -def import_from_name(name):

600 try: 601 return __import__(name, globals(), locals(), ['*']) 602 except Exception, e: 603 raise ValueError, str(e) 604 except: 605 raise ValueError, 'Error importing %r' % name

606

607 -def find_module_from_filename(filename):

608 """ 609 Given a filename, return a tuple `(basedir, module)`, where 610 `module` is the module's name, and `basedir` is the directory it 611 should be loaded from (this directory should be added to the 612 path to import it). Packages are handled correctly. 613 """ 614 (basedir, file) = os.path.split(filename) 615 (module_name, ext) = os.path.splitext(file) 616 617 # If it's a package, then import with the directory name (don't 618 # use __init__ as the module name). 619 if module_name == '__init__': 620 (basedir, module_name) = os.path.split(basedir) 621 622 # If it's contained inside a package, then find the base dir. 623 if (os.path.exists(os.path.join(basedir, '__init__.py')) or 624 os.path.exists(os.path.join(basedir, '__init__.pyc')) or 625 os.path.exists(os.path.join(basedir, '__init__.pyw'))): 626 package = [] 627 while os.path.exists(os.path.join(basedir, '__init__.py')): 628 (basedir,dir) = os.path.split(basedir) 629 if dir == '': break 630 package.append(dir) 631 package.reverse() 632 module_name = '.'.join(package+[module_name]) 633 634 return (basedir, module_name)

635

636 -def split_pysrc_into_statements(s):

637 parens = 0 # Number of parens deep we're nested? 638 quote = None # What type of string are we in (if any)? 639 statements = [] # List of statements we've found 640 continuation = False # Did last line end with a backslash? 641 for line in s.lstrip().split('\n'): 642 # Check indentation level. 643 indent = re.match(r'\s*', line).end() 644 645 # [DEBUG PRINTF] 646 #print '%4d %6r %6s %5s %r' % (parens, quote, continuation, 647 # indent, line[:40]) 648 649 # Add the line as a new statement or a continuation. 650 if (parens == 0 and quote is None and indent == 0 and 651 (not continuation)): 652 if line.strip(): 653 statements.append(line) 654 else: 655 statements[-1] += '\n'+line 656 657 # Scan the line, checking for quotes, parens, and comment 658 # markers (so we can decide when a line is a continuation). 659 line_has_comment = False 660 for c in re.findall(r'\\.|"""|\'\'\'|"|\'|$|$|\[|\]|\{|\}|\#', line): 661 if quote: 662 if c == quote: 663 quote = None 664 elif c in '([{': 665 parens += 1 666 elif c in ')]}': 667 parens -= 1 668 elif c == '#': 669 line_has_comment = True 670 break 671 elif c[0] != '\\': 672 quote = c 673 if not line_has_comment: 674 continuation = line.strip().endswith('\\') 675 676 return statements

677 678 ########################################################################### 679 # Basic Actions 680 ########################################################################### 681

682 -def run(names, optionflags, verbosity):

683 suite = unittest.TestSuite() 684 for name in names: 685 try: 686 for test in find(name): 687 suite.addTest(DocTestCase(test, optionflags)) 688 except ValueError, e: 689 print >>sys.stderr, ('%s: Error processing %s -- %s' % 690 (sys.argv[0], name, e)) 691 unittest.TextTestRunner(verbosity=verbosity).run(suite)

692

693 -def debug(names, optionflags, verbosity, pm=True):

694 debugger = Debugger() 695 for name in names: 696 try: 697 for test in find(name): 698 debugger.debug(test, pm) 699 except ValueError, e: 700 raise 701 print >>sys.stderr, ('%s: Error processing %s -- %s' % 702 (sys.argv[0], name, e))

703

704 -def update(names, optionflags, verbosity):

705 runner = UpdateRunner(verbose=True) 706 for name in names: 707 try: 708 # Make sure we're running on a text file. 709 tests = find(name) 710 if len(tests) != 1 or tests[0].lineno != 0: 711 raise ValueError('update can only be used with text files') 712 test = tests[0] 713 714 # Run the updater! 715 (failures, tries) = runner.run(test) 716 717 # Confirm the changes. 718 if failures == 0: 719 print 'No updates needed!' 720 else: 721 print '*'*70 722 print '%d examples updated.' % failures 723 print '-'*70 724 sys.stdout.write('Accept updates? [y/N] ') 725 sys.stdout.flush() 726 if sys.stdin.readline().lower().strip() in ('y', 'yes'): 727 # Make a backup of the original contents. 728 backup = test.filename+'.bak' 729 print 'Renaming %s -> %s' % (name, backup) 730 os.rename(test.filename, backup) 731 # Write the new contents. 732 print 'Writing updated version to %s' % test.filename 733 out = open(test.filename, 'w') 734 out.write(test.docstring) 735 out.close() 736 else: 737 print 'Updates rejected!' 738 except ValueError, e: 739 raise 740 print >>sys.stderr, ('%s: Error processing %s -- %s' % 741 (sys.argv[0], name, e))

742 743 ########################################################################### 744 # Main script 745 ########################################################################### 746 747 # Action options 748 CHECK_OPT = Option("--check", 749 action="store_const", dest="action", const="check", 750 default="check", 751 help="Verify the output of the doctest examples in the " 752 "given files.") 753 754 UPDATE_OPT = Option("--update", "-u", 755 action="store_const", dest="action", const="update", 756 help="Update the expected output for new or out-of-date " 757 "doctest examples in the given files. In " 758 "particular, find every example whose actual output " 759 "does not match its expected output; and replace its " 760 "expected output with its actual output. You will " 761 "be asked to verify the changes before they are " 762 "written back to the file; be sure to check them over " 763 "carefully, to ensure that you don't accidentally " 764 "create broken test cases.") 765 766 DEBUG_OPT = Option("--debug", 767 action="store_const", dest="action", const="debug", 768 help="Verify the output of the doctest examples in the " 769 "given files. If any example fails, then enter the " 770 "python debugger.") 771 772 # Reporting options 773 VERBOSE_OPT = Option("-v", "--verbose", 774 action="count", dest="verbosity", default=1, 775 help="Increase verbosity.") 776 777 QUIET_OPT = Option("-q", "--quiet", 778 action="store_const", dest="verbosity", const=0, 779 help="Decrease verbosity.") 780 781 UDIFF_OPT = Option("--udiff", '-d', 782 action="store_const", dest="udiff", const=1, default=0, 783 help="Display test failures using unified diffs.") 784 785 CDIFF_OPT = Option("--cdiff", 786 action="store_const", dest="cdiff", const=1, default=0, 787 help="Display test failures using context diffs.") 788 789 NDIFF_OPT = Option("--ndiff", 790 action="store_const", dest="ndiff", const=1, default=0, 791 help="Display test failures using ndiffs.") 792 793 COVERAGE_OPT = Option("--coverage", 794 action="store", dest="coverage", metavar='FILENAME', 795 help="Generate coverage information, and write it to the " 796 "given file.") 797 798 # Output Comparison options 799 ELLIPSIS_OPT = Option("--ellipsis", 800 action="store_const", dest="ellipsis", const=1, default=0, 801 help="Allow \"...\" to be used for ellipsis in the " 802 "expected output.") 803 NORMWS_OPT = Option("--normalize_whitespace", 804 action="store_const", dest="normws", const=1, default=0, 805 help="Ignore whitespace differences between " 806 "the expected output and the actual output.") 807

808 -def main():

809 # Create the option parser. 810 optparser = OptionParser(usage='%prog [options] NAME ...', 811 version="Edloper's Doctest Driver, " 812 "version %s" % __version__) 813 814 action_group = OptionGroup(optparser, 'Actions (default=check)') 815 action_group.add_options([CHECK_OPT, UPDATE_OPT, DEBUG_OPT]) 816 optparser.add_option_group(action_group) 817 818 reporting_group = OptionGroup(optparser, 'Reporting') 819 reporting_group.add_options([VERBOSE_OPT, QUIET_OPT, 820 UDIFF_OPT, CDIFF_OPT, NDIFF_OPT, 821 COVERAGE_OPT]) 822 optparser.add_option_group(reporting_group) 823 824 compare_group = OptionGroup(optparser, 'Output Comparison') 825 compare_group.add_options([ELLIPSIS_OPT, NORMWS_OPT]) 826 optparser.add_option_group(compare_group) 827 828 # Extract optionflags and the list of file names. 829 optionvals, names = optparser.parse_args() 830 if len(names) == 0: 831 optparser.error("No files specified") 832 optionflags = (optionvals.udiff * REPORT_UDIFF | 833 optionvals.cdiff * REPORT_CDIFF | 834 optionvals.ellipsis * ELLIPSIS | 835 optionvals.normws * NORMALIZE_WHITESPACE) 836 837 # Check coverage, if requested 838 if optionvals.coverage: 839 coverage.use_cache(True, cache_file=optionvals.coverage) 840 coverage.start() 841 842 # Perform the requested action. 843 if optionvals.action == 'check': 844 run(names, optionflags, optionvals.verbosity) 845 elif optionvals.action == 'update': 846 update(names, optionflags, optionvals.verbosity) 847 elif optionvals.action == 'debug': 848 debug(names, optionflags, optionvals.verbosity) 849 else: 850 optparser.error('INTERNAL ERROR: Bad action %s' % optionvals.action) 851 852 # Check coverage, if requested 853 if optionvals.coverage: 854 coverage.stop()

855 856 if __name__ == '__main__': main() 857

Source Code for Module nltk.test.doctest_driver