1 """pytest is a tool that eases test running and debugging.
2
3 To be able to use pytest, you should either write tests using
4 the logilab.common.testlib's framework or the unittest module of the
5 Python's standard library.
6
7 You can customize pytest's behaviour by defining a ``pytestconf.py`` file
8 somewhere in your test directory. In this file, you can add options or
9 change the way tests are run.
10
11 To add command line options, you must define a ``update_parser`` function in
12 your ``pytestconf.py`` file. The function must accept a single parameter
13 that will be the OptionParser's instance to customize.
14
15 If you wish to customize the tester, you'll have to define a class named
16 ``CustomPyTester``. This class should extend the default `PyTester` class
17 defined in the pytest module. Take a look at the `PyTester` and `DjangoTester`
18 classes for more information about what can be done.
19
20 For instance, if you wish to add a custom -l option to specify a loglevel, you
21 could define the following ``pytestconf.py`` file ::
22
23 import logging
24 from logilab.common.pytest import PyTester
25
26 def update_parser(parser):
27 parser.add_option('-l', '--loglevel', dest='loglevel', action='store',
28 choices=('debug', 'info', 'warning', 'error', 'critical'),
29 default='critical', help="the default log level possible choices are "
30 "('debug', 'info', 'warning', 'error', 'critical')")
31 return parser
32
33
34 class CustomPyTester(PyTester):
35 def __init__(self, cvg, options):
36 super(CustomPyTester, self).__init__(cvg, options)
37 loglevel = options.loglevel.upper()
38 logger = logging.getLogger('erudi')
39 logger.setLevel(logging.getLevelName(loglevel))
40
41
42 In your TestCase class you can then get the value of a specific option with
43 the ``optval`` method::
44
45 class MyTestCase(TestCase):
46 def test_foo(self):
47 loglevel = self.optval('loglevel')
48 # ...
49
50
51 You can also tag your tag your test for fine filtering
52
53 With those tag::
54
55 from logilab.common.testlib import tag, TestCase
56
57 class Exemple(TestCase):
58
59 @tag('rouge', 'carre')
60 def toto(self):
61 pass
62
63 @tag('carre', 'vert')
64 def tata(self):
65 pass
66
67 @tag('rouge')
68 def titi(test):
69 pass
70
71 you can filter the function with a simple python expression
72
73 * ``toto`` and ``titi`` match ``rouge``
74
75 * ``toto``, ``tata`` and ``titi``, match ``rouge or carre``
76
77 * ``tata`` and ``titi`` match``rouge ^ carre``
78
79 * ``titi`` match ``rouge and not carre``
80
81
82
83
84
85
86 :copyright: 2000-2008 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
87 :contact: http://www.logilab.fr/ -- mailto:contact@logilab.fr
88 :license: General Public License version 2 - http://www.gnu.org/licenses
89 """
90 __docformat__ = "restructuredtext en"
91
92 PYTEST_DOC = """%prog [OPTIONS] [testfile [testpattern]]
93
94 examples:
95
96 pytest path/to/mytests.py
97 pytest path/to/mytests.py TheseTests
98 pytest path/to/mytests.py TheseTests.test_thisone
99 pytest path/to/mytests.py -m '(not long and database) or regr'
100
101 pytest one (will run both test_thisone and test_thatone)
102 pytest path/to/mytests.py -s not (will skip test_notthisone)
103
104 pytest --coverage test_foo.py
105 (only if logilab.devtools is available)
106 """
107
108 import os, sys, re
109 import os.path as osp
110 from time import time, clock
111 import warnings
112
113 from logilab.common.fileutils import abspath_listdir
114 from logilab.common import testlib
115 import doctest
116 import unittest
117
118
119 import imp
120
121 import __builtin__
122
123
124 try:
125 import django
126 from logilab.common.modutils import modpath_from_file, load_module_from_modpath
127 DJANGO_FOUND = True
128 except ImportError:
129 DJANGO_FOUND = False
130
131 CONF_FILE = 'pytestconf.py'
132
133
134
135
137 nesting = 0
138
140 if not cls.nesting:
141 cls.tracefunc = staticmethod(getattr(sys, '__settrace__', sys.settrace))
142 cls.oldtracer = getattr(sys, '__tracer__', None)
143 sys.__notrace__ = True
144 cls.tracefunc(None)
145 cls.nesting += 1
146 pause_tracing = classmethod(pause_tracing)
147
149 cls.nesting -= 1
150 assert cls.nesting >= 0
151 if not cls.nesting:
152 cls.tracefunc(cls.oldtracer)
153 delattr(sys, '__notrace__')
154 resume_tracing = classmethod(resume_tracing)
155
156
157 pause_tracing = TraceController.pause_tracing
158 resume_tracing = TraceController.resume_tracing
159
160
162 if hasattr(func, 'uncovered'):
163 return func
164 func.uncovered = True
165 def not_covered(*args, **kwargs):
166 pause_tracing()
167 try:
168 return func(*args, **kwargs)
169 finally:
170 resume_tracing()
171 not_covered.uncovered = True
172 return not_covered
173
174
175
176
177
178
179 unittest.TestCase = testlib.TestCase
180 unittest.main = testlib.unittest_main
181 unittest._TextTestResult = testlib.SkipAwareTestResult
182 unittest.TextTestRunner = testlib.SkipAwareTextTestRunner
183 unittest.TestLoader = testlib.NonStrictTestLoader
184 unittest.TestProgram = testlib.SkipAwareTestProgram
185 if sys.version_info >= (2, 4):
186 doctest.DocTestCase.__bases__ = (testlib.TestCase,)
187 else:
188 unittest.FunctionTestCase.__bases__ = (testlib.TestCase,)
189
190
191
192 TESTFILE_RE = re.compile("^((unit)?test.*|smoketest)\.py$")
194 """returns True if `filename` seems to be a test file"""
195 return TESTFILE_RE.match(osp.basename(filename))
196
197 TESTDIR_RE = re.compile("^(unit)?tests?$")
199 """returns True if `filename` seems to be a test directory"""
200 return TESTDIR_RE.match(osp.basename(dirpath))
201
202
204 """loads a ``pytestconf.py`` file and update default parser
205 and / or tester.
206 """
207 namespace = {}
208 execfile(path, namespace)
209 if 'update_parser' in namespace:
210 namespace['update_parser'](parser)
211 return namespace.get('CustomPyTester', PyTester)
212
213
215 """try to find project's root and add it to sys.path"""
216 curdir = osp.abspath(projdir)
217 previousdir = curdir
218 testercls = PyTester
219 conf_file_path = osp.join(curdir, CONF_FILE)
220 if osp.isfile(conf_file_path):
221 testercls = load_pytest_conf(conf_file_path, parser)
222 while this_is_a_testdir(curdir) or \
223 osp.isfile(osp.join(curdir, '__init__.py')):
224 newdir = osp.normpath(osp.join(curdir, os.pardir))
225 if newdir == curdir:
226 break
227 previousdir = curdir
228 curdir = newdir
229 conf_file_path = osp.join(curdir, CONF_FILE)
230 if osp.isfile(conf_file_path):
231 testercls = load_pytest_conf(conf_file_path, parser)
232 return previousdir, testercls
233
234
236 """this class holds global test statistics"""
238 self.ran = 0
239 self.skipped = 0
240 self.failures = 0
241 self.errors = 0
242 self.ttime = 0
243 self.ctime = 0
244 self.modulescount = 0
245 self.errmodules = []
246
247 - def feed(self, filename, testresult, ttime, ctime):
248 """integrates new test information into internal statistics"""
249 ran = testresult.testsRun
250 self.ran += ran
251 self.skipped += len(getattr(testresult, 'skipped', ()))
252 self.failures += len(testresult.failures)
253 self.errors += len(testresult.errors)
254 self.ttime += ttime
255 self.ctime += ctime
256 self.modulescount += 1
257 if not testresult.wasSuccessful():
258 problems = len(testresult.failures) + len(testresult.errors)
259 self.errmodules.append((filename[:-3], problems, ran))
260
261
263 """called when the test module could not be imported by unittest
264 """
265 self.errors += 1
266 self.modulescount += 1
267 self.ran += 1
268 self.errmodules.append((filename[:-3], 1, 1))
269
271 self.modulescount += 1
272 self.ran += 1
273 self.errmodules.append((filename[:-3], 0, 0))
274
276 """this is just presentation stuff"""
277 line1 = ['Ran %s test cases in %.2fs (%.2fs CPU)'
278 % (self.ran, self.ttime, self.ctime)]
279 if self.errors:
280 line1.append('%s errors' % self.errors)
281 if self.failures:
282 line1.append('%s failures' % self.failures)
283 if self.skipped:
284 line1.append('%s skipped' % self.skipped)
285 modulesok = self.modulescount - len(self.errmodules)
286 if self.errors or self.failures:
287 line2 = '%s modules OK (%s failed)' % (modulesok,
288 len(self.errmodules))
289 descr = ', '.join(['%s [%s/%s]' % info for info in self.errmodules])
290 line3 = '\nfailures: %s' % descr
291 elif modulesok:
292 line2 = 'All %s modules OK' % modulesok
293 line3 = ''
294 else:
295 return ''
296 return '%s\n%s%s' % (', '.join(line1), line2, line3)
297
298
299
301 """remove all modules from cache that come from `testdir`
302
303 This is used to avoid strange side-effects when using the
304 testall() mode of pytest.
305 For instance, if we run pytest on this tree::
306
307 A/test/test_utils.py
308 B/test/test_utils.py
309
310 we **have** to clean sys.modules to make sure the correct test_utils
311 module is ran in B
312 """
313 for modname, mod in sys.modules.items():
314 if mod is None:
315 continue
316 if not hasattr(mod, '__file__'):
317
318 continue
319 modfile = mod.__file__
320
321
322 if not osp.isabs(modfile) or modfile.startswith(testdir):
323 del sys.modules[modname]
324
325
326
328 """encapsulates testrun logic"""
329
336
338 """prints the report and returns appropriate exitcode"""
339
340 print "*" * 79
341 print self.report
342
344
345 if self._errcode is not None:
346 return self._errcode
347 return self.report.failures + self.report.errors
348
351 errcode = property(get_errcode, set_errcode)
352
353 - def testall(self, exitfirst=False):
354 """walks through current working directory, finds something
355 which can be considered as a testdir and runs every test there
356 """
357 here = os.getcwd()
358 for dirname, dirs, _ in os.walk(here):
359 for skipped in ('CVS', '.svn', '.hg'):
360 if skipped in dirs:
361 dirs.remove(skipped)
362 basename = osp.basename(dirname)
363 if this_is_a_testdir(basename):
364 print "going into", dirname
365
366 self.testonedir(dirname, exitfirst)
367 dirs[:] = []
368 if self.report.ran == 0:
369 print "no test dir found testing here:", here
370
371
372
373 self.testonedir(here)
374
376 """finds each testfile in the `testdir` and runs it"""
377 for filename in abspath_listdir(testdir):
378 if this_is_a_testfile(filename):
379 if self.options.exitfirst and not self.options.restart:
380
381 try:
382 restartfile = open(testlib.FILE_RESTART, "w")
383 restartfile.close()
384 except Exception, e:
385 print >> sys.__stderr__, "Error while overwriting \
386 succeeded test file :", osp.join(os.getcwd(),testlib.FILE_RESTART)
387 raise e
388
389 prog = self.testfile(filename, batchmode=True)
390 if exitfirst and (prog is None or not prog.result.wasSuccessful()):
391 break
392 self.firstwrite = True
393
394 remove_local_modules_from_sys(testdir)
395
396
397 - def testfile(self, filename, batchmode=False):
398 """runs every test in `filename`
399
400 :param filename: an absolute path pointing to a unittest file
401 """
402 here = os.getcwd()
403 dirname = osp.dirname(filename)
404 if dirname:
405 os.chdir(dirname)
406
407 if self.options.exitfirst and not self.options.restart and self.firstwrite:
408 try:
409 restartfile = open(testlib.FILE_RESTART, "w")
410 restartfile.close()
411 except Exception, e:
412 print >> sys.__stderr__, "Error while overwriting \
413 succeeded test file :", osp.join(os.getcwd(),testlib.FILE_RESTART)
414 raise e
415 modname = osp.basename(filename)[:-3]
416 try:
417 print >> sys.stderr, (' %s ' % osp.basename(filename)).center(70, '=')
418 except TypeError:
419 print >> sys.stderr, (' %s ' % osp.basename(filename)).center(70)
420 try:
421 tstart, cstart = time(), clock()
422 try:
423 testprog = testlib.unittest_main(modname, batchmode=batchmode, cvg=self.cvg,
424 options=self.options, outstream=sys.stderr)
425 except KeyboardInterrupt:
426 raise
427 except SystemExit, exc:
428 self.errcode = exc.code
429 raise
430 except testlib.TestSkipped:
431 print "Module skipped:", filename
432 self.report.skip_module(filename)
433 return None
434 except Exception:
435 self.report.failed_to_test_module(filename)
436 print >> sys.stderr, 'unhandled exception occurred while testing', modname
437 import traceback
438 traceback.print_exc(file=sys.stderr)
439 return None
440
441 tend, cend = time(), clock()
442 ttime, ctime = (tend - tstart), (cend - cstart)
443 self.report.feed(filename, testprog.result, ttime, ctime)
444 return testprog
445 finally:
446 if dirname:
447 os.chdir(here)
448
449
450
452
454 """try to find project's setting and load it"""
455 curdir = osp.abspath(dirname)
456 previousdir = curdir
457 while not osp.isfile(osp.join(curdir, 'settings.py')) and \
458 osp.isfile(osp.join(curdir, '__init__.py')):
459 newdir = osp.normpath(osp.join(curdir, os.pardir))
460 if newdir == curdir:
461 raise AssertionError('could not find settings.py')
462 previousdir = curdir
463 curdir = newdir
464
465 settings = load_module_from_modpath(modpath_from_file(osp.join(curdir, 'settings.py')))
466 from django.core.management import setup_environ
467 setup_environ(settings)
468 settings.DEBUG = False
469 self.settings = settings
470
471 if curdir not in sys.path:
472 sys.path.insert(1, curdir)
473
475
476 from django.test.utils import setup_test_environment
477 from django.test.utils import create_test_db
478 setup_test_environment()
479 create_test_db(verbosity=0)
480 self.dbname = self.settings.TEST_DATABASE_NAME
481
482
484
485 from django.test.utils import teardown_test_environment
486 from django.test.utils import destroy_test_db
487 teardown_test_environment()
488 print 'destroying', self.dbname
489 destroy_test_db(self.dbname, verbosity=0)
490
491
492 - def testall(self, exitfirst=False):
493 """walks through current working directory, finds something
494 which can be considered as a testdir and runs every test there
495 """
496 for dirname, dirs, _ in os.walk(os.getcwd()):
497 for skipped in ('CVS', '.svn', '.hg'):
498 if skipped in dirs:
499 dirs.remove(skipped)
500 if 'tests.py' in files:
501 self.testonedir(dirname, exitfirst)
502 dirs[:] = []
503 else:
504 basename = osp.basename(dirname)
505 if basename in ('test', 'tests'):
506 print "going into", dirname
507
508 self.testonedir(dirname, exitfirst)
509 dirs[:] = []
510
511
513 """finds each testfile in the `testdir` and runs it"""
514
515
516 testfiles = [fpath for fpath in abspath_listdir(testdir)
517 if this_is_a_testfile(fpath)]
518 if len(testfiles) > 1:
519 try:
520 testfiles.remove(osp.join(testdir, 'tests.py'))
521 except ValueError:
522 pass
523 for filename in testfiles:
524
525 prog = self.testfile(filename, batchmode=True)
526 if exitfirst and (prog is None or not prog.result.wasSuccessful()):
527 break
528
529 remove_local_modules_from_sys(testdir)
530
531
532 - def testfile(self, filename, batchmode=False):
533 """runs every test in `filename`
534
535 :param filename: an absolute path pointing to a unittest file
536 """
537 here = os.getcwd()
538 dirname = osp.dirname(filename)
539 if dirname:
540 os.chdir(dirname)
541 self.load_django_settings(dirname)
542 modname = osp.basename(filename)[:-3]
543 print >>sys.stderr, (' %s ' % osp.basename(filename)).center(70, '=')
544 try:
545 try:
546 tstart, cstart = time(), clock()
547 self.before_testfile()
548 testprog = testlib.unittest_main(modname, batchmode=batchmode, cvg=self.cvg)
549 tend, cend = time(), clock()
550 ttime, ctime = (tend - tstart), (cend - cstart)
551 self.report.feed(filename, testprog.result, ttime, ctime)
552 return testprog
553 except SystemExit:
554 raise
555 except Exception, exc:
556 import traceback
557 traceback.print_exc()
558 self.report.failed_to_test_module(filename)
559 print 'unhandled exception occurred while testing', modname
560 print 'error: %s' % exc
561 return None
562 finally:
563 self.after_testfile()
564 if dirname:
565 os.chdir(here)
566
567
569 """creates the OptionParser instance
570 """
571 from optparse import OptionParser
572 parser = OptionParser(usage=PYTEST_DOC)
573
574 parser.newargs = []
575 def rebuild_cmdline(option, opt, value, parser):
576 """carry the option to unittest_main"""
577 parser.newargs.append(opt)
578
579 def rebuild_and_store(option, opt, value, parser):
580 """carry the option to unittest_main and store
581 the value on current parser
582 """
583 parser.newargs.append(opt)
584 setattr(parser.values, option.dest, True)
585
586 def capture_and_rebuild(option, opt, value, parser):
587 warnings.simplefilter('ignore', DeprecationWarning)
588 rebuild_cmdline(option, opt, value, parser)
589
590
591 parser.add_option('-t', dest='testdir', default=None,
592 help="directory where the tests will be found")
593 parser.add_option('-d', dest='dbc', default=False,
594 action="store_true", help="enable design-by-contract")
595
596 parser.add_option('-v', '--verbose', callback=rebuild_cmdline,
597 action="callback", help="Verbose output")
598 parser.add_option('-i', '--pdb', callback=rebuild_and_store,
599 dest="pdb", action="callback",
600 help="Enable test failure inspection (conflicts with --coverage)")
601 parser.add_option('-x', '--exitfirst', callback=rebuild_and_store,
602 dest="exitfirst", default=False,
603 action="callback", help="Exit on first failure "
604 "(only make sense when pytest run one test file)")
605 parser.add_option('-R', '--restart', callback=rebuild_and_store,
606 dest="restart", default=False,
607 action="callback",
608 help="Restart tests from where it failed (implies exitfirst) "
609 "(only make sense if tests previously ran with exitfirst only)")
610 parser.add_option('-c', '--capture', callback=capture_and_rebuild,
611 action="callback",
612 help="Captures and prints standard out/err only on errors "
613 "(only make sense when pytest run one test file)")
614 parser.add_option('--color', callback=rebuild_cmdline,
615 action="callback",
616 help="colorize tracebacks")
617 parser.add_option('-p', '--printonly',
618
619
620
621 action="store", dest="printonly", default=None,
622 help="Only prints lines matching specified pattern (implies capture) "
623 "(only make sense when pytest run one test file)")
624 parser.add_option('-s', '--skip',
625
626
627
628 action="store", dest="skipped", default=None,
629 help="test names matching this name will be skipped "
630 "to skip several patterns, use commas")
631 parser.add_option('-q', '--quiet', callback=rebuild_cmdline,
632 action="callback", help="Minimal output")
633 parser.add_option('-P', '--profile', default=None, dest='profile',
634 help="Profile execution and store data in the given file")
635 parser.add_option('-m', '--match', default=None, dest='tags_pattern',
636 help="only execute test whose tag match the current pattern")
637
638 try:
639 from logilab.devtools.lib.coverage import Coverage
640 parser.add_option('--coverage', dest="coverage", default=False,
641 action="store_true",
642 help="run tests with pycoverage (conflicts with --pdb)")
643 except ImportError:
644 pass
645
646 if DJANGO_FOUND:
647 parser.add_option('-J', '--django', dest='django', default=False,
648 action="store_true",
649 help='use pytest for django test cases')
650 return parser
651
652
654 """Parse the command line and return (options processed), (options to pass to
655 unittest_main()), (explicitfile or None).
656 """
657
658 options, args = parser.parse_args()
659 if options.pdb and getattr(options, 'coverage', False):
660 parser.error("'pdb' and 'coverage' options are exclusive")
661 filenames = [arg for arg in args if arg.endswith('.py')]
662 if filenames:
663 if len(filenames) > 1:
664 parser.error("only one filename is acceptable")
665 explicitfile = filenames[0]
666 args.remove(explicitfile)
667 else:
668 explicitfile = None
669
670 testlib.ENABLE_DBC = options.dbc
671 newargs = parser.newargs
672 if options.printonly:
673 newargs.extend(['--printonly', options.printonly])
674 if options.skipped:
675 newargs.extend(['--skip', options.skipped])
676
677 if options.restart:
678 options.exitfirst = True
679
680
681 newargs += args
682 return options, explicitfile
683
684
685
687 parser = make_parser()
688 rootdir, testercls = project_root(parser)
689 options, explicitfile = parseargs(parser)
690
691 sys.argv[1:] = parser.newargs
692 covermode = getattr(options, 'coverage', None)
693 cvg = None
694 if not '' in sys.path:
695 sys.path.insert(0, '')
696 if covermode:
697
698 from logilab.devtools.lib.coverage import Coverage
699 cvg = Coverage([rootdir])
700 cvg.erase()
701 cvg.start()
702 if DJANGO_FOUND and options.django:
703 tester = DjangoTester(cvg, options)
704 else:
705 tester = testercls(cvg, options)
706 if explicitfile:
707 cmd, args = tester.testfile, (explicitfile,)
708 elif options.testdir:
709 cmd, args = tester.testonedir, (options.testdir, options.exitfirst)
710 else:
711 cmd, args = tester.testall, (options.exitfirst,)
712 try:
713 try:
714 if options.profile:
715 import hotshot
716 prof = hotshot.Profile(options.profile)
717 prof.runcall(cmd, *args)
718 prof.close()
719 print 'profile data saved in', options.profile
720 else:
721 cmd(*args)
722 except SystemExit:
723 raise
724 except:
725 import traceback
726 traceback.print_exc()
727 finally:
728 if covermode:
729 cvg.stop()
730 cvg.save()
731 tester.show_report()
732 if covermode:
733 print 'coverage information stored, use it with pycoverage -ra'
734 sys.exit(tester.errcode)
735