00001 """
00002 XML Test Runner for PyUnit
00003 """
00004
00005
00006
00007
00008 __version__ = "0.1"
00009
00010 import os.path
00011 import re
00012 import sys
00013 import time
00014 import traceback
00015 import unittest
00016 from xml.sax.saxutils import escape
00017
00018 try:
00019 from StringIO import StringIO
00020 except ImportError:
00021 from io import StringIO
00022
00023
00024 class _TestInfo(object):
00025
00026 """Information about a particular test.
00027
00028 Used by _XMLTestResult.
00029
00030 """
00031
00032 def __init__(self, test, time):
00033 (self._class, self._method) = test.id().rsplit(".", 1)
00034 self._time = time
00035 self._error = None
00036 self._failure = None
00037
00038 @staticmethod
00039 def create_success(test, time):
00040 """Create a _TestInfo instance for a successful test."""
00041 return _TestInfo(test, time)
00042
00043 @staticmethod
00044 def create_failure(test, time, failure):
00045 """Create a _TestInfo instance for a failed test."""
00046 info = _TestInfo(test, time)
00047 info._failure = failure
00048 return info
00049
00050 @staticmethod
00051 def create_error(test, time, error):
00052 """Create a _TestInfo instance for an erroneous test."""
00053 info = _TestInfo(test, time)
00054 info._error = error
00055 return info
00056
00057 def print_report(self, stream):
00058 """Print information about this test case in XML format to the
00059 supplied stream.
00060
00061 """
00062 stream.write(' <testcase classname="%(class)s" name="%(method)s" time="%(time).4f">' % \
00063 {
00064 "class": self._class,
00065 "method": self._method,
00066 "time": self._time,
00067 })
00068 if self._failure is not None:
00069 self._print_error(stream, 'failure', self._failure)
00070 if self._error is not None:
00071 self._print_error(stream, 'error', self._error)
00072 stream.write('</testcase>\n')
00073
00074 def _print_error(self, stream, tagname, error):
00075 """Print information from a failure or error to the supplied stream."""
00076 text = escape(str(error[1]))
00077 stream.write('\n')
00078 stream.write(' <%s type="%s">%s\n' \
00079 % (tagname, _clsname(error[0]), text))
00080 tb_stream = StringIO()
00081 traceback.print_tb(error[2], None, tb_stream)
00082 stream.write(escape(tb_stream.getvalue()))
00083 stream.write(' </%s>\n' % tagname)
00084 stream.write(' ')
00085
00086
00087 def _clsname(cls):
00088 return cls.__module__ + "." + cls.__name__
00089
00090
00091 class _XMLTestResult(unittest.TestResult):
00092
00093 """A test result class that stores result as XML.
00094
00095 Used by XMLTestRunner.
00096
00097 """
00098
00099 def __init__(self, classname):
00100 unittest.TestResult.__init__(self)
00101 self._test_name = classname
00102 self._start_time = None
00103 self._tests = []
00104 self._error = None
00105 self._failure = None
00106
00107 def startTest(self, test):
00108 unittest.TestResult.startTest(self, test)
00109 self._error = None
00110 self._failure = None
00111 self._start_time = time.time()
00112
00113 def stopTest(self, test):
00114 time_taken = time.time() - self._start_time
00115 unittest.TestResult.stopTest(self, test)
00116 if self._error:
00117 info = _TestInfo.create_error(test, time_taken, self._error)
00118 elif self._failure:
00119 info = _TestInfo.create_failure(test, time_taken, self._failure)
00120 else:
00121 info = _TestInfo.create_success(test, time_taken)
00122 self._tests.append(info)
00123
00124 def addError(self, test, err):
00125 unittest.TestResult.addError(self, test, err)
00126 self._error = err
00127
00128 def addFailure(self, test, err):
00129 unittest.TestResult.addFailure(self, test, err)
00130 self._failure = err
00131
00132 def print_report(self, stream, time_taken, out, err):
00133 """Prints the XML report to the supplied stream.
00134
00135 The time the tests took to perform as well as the captured standard
00136 output and standard error streams must be passed in.a
00137
00138 """
00139 stream.write('<testsuite errors="%(e)d" failures="%(f)d" ' % \
00140 { "e": len(self.errors), "f": len(self.failures) })
00141 stream.write('name="%(n)s" tests="%(t)d" time="%(time).3f">\n' % \
00142 {
00143 "n": self._test_name,
00144 "t": self.testsRun,
00145 "time": time_taken,
00146 })
00147 for info in self._tests:
00148 info.print_report(stream)
00149 stream.write(' <system-out><![CDATA[%s]]></system-out>\n' % out)
00150 stream.write(' <system-err><![CDATA[%s]]></system-err>\n' % err)
00151 stream.write('</testsuite>\n')
00152
00153
00154 class XMLTestRunner(object):
00155
00156 """A test runner that stores results in XML format compatible with JUnit.
00157
00158 XMLTestRunner(stream=None) -> XML test runner
00159
00160 The XML file is written to the supplied stream. If stream is None, the
00161 results are stored in a file called TEST-<module>.<class>.xml in the
00162 current working directory (if not overridden with the path property),
00163 where <module> and <class> are the module and class name of the test class.
00164
00165 """
00166
00167 def __init__(self, stream=None):
00168 self._stream = stream
00169 self._path = "."
00170
00171 def run(self, test):
00172 """Run the given test case or test suite."""
00173 class_ = test.__class__
00174 classname = class_.__module__ + "." + class_.__name__
00175 if self._stream == None:
00176 filename = "TEST-%s.xml" % classname
00177 stream = file(os.path.join(self._path, filename), "w")
00178 stream.write('<?xml version="1.0" encoding="utf-8"?>\n')
00179 else:
00180 stream = self._stream
00181
00182 result = _XMLTestResult(classname)
00183 start_time = time.time()
00184
00185 try:
00186 self._orig_stdout = sys.stdout
00187 self._orig_stderr = sys.stderr
00188 sys.stdout = StringIO()
00189 sys.stderr = StringIO()
00190 test(result)
00191 try:
00192 out_s = sys.stdout.getvalue()
00193 except AttributeError:
00194 out_s = ""
00195 try:
00196 err_s = sys.stderr.getvalue()
00197 except AttributeError:
00198 err_s = ""
00199 finally:
00200 sys.stdout = self._orig_stdout
00201 sys.stderr = self._orig_stderr
00202
00203
00204 time_taken = time.time() - start_time
00205 result.print_report(stream, time_taken, out_s, err_s)
00206 if self._stream is None:
00207 stream.close()
00208
00209 return result
00210
00211 def _set_path(self, path):
00212 self._path = path
00213
00214 path = property(lambda self: self._path, _set_path, None,
00215 """The path where the XML files are stored.
00216
00217 This property is ignored when the XML file is written to a file
00218 stream.""")
00219
00220
00221 class _fake_std_streams(object):
00222
00223 def __enter__(self):
00224 self._orig_stdout = sys.stdout
00225 self._orig_stderr = sys.stderr
00226 sys.stdout = StringIO()
00227 sys.stderr = StringIO()
00228
00229 def __exit__(self, exc_type, exc_val, exc_tb):
00230 sys.stdout = self._orig_stdout
00231 sys.stderr = self._orig_stderr
00232
00233
00234 class XMLTestRunnerTest(unittest.TestCase):
00235
00236 def setUp(self):
00237 self._stream = StringIO()
00238
00239 def _try_test_run(self, test_class, expected):
00240
00241 """Run the test suite against the supplied test class and compare the
00242 XML result against the expected XML string. Fail if the expected
00243 string doesn't match the actual string. All time attributes in the
00244 expected string should have the value "0.000". All error and failure
00245 messages are reduced to "Foobar".
00246
00247 """
00248
00249 runner = XMLTestRunner(self._stream)
00250 runner.run(unittest.makeSuite(test_class))
00251
00252 got = self._stream.getvalue()
00253
00254
00255 got = re.sub(r'time="\d+\.\d+"', 'time="0.000"', got)
00256
00257
00258 got = re.sub(r'(?s)<failure (.*?)>.*?</failure>', r'<failure \1>Foobar</failure>', got)
00259 got = re.sub(r'(?s)<error (.*?)>.*?</error>', r'<error \1>Foobar</error>', got)
00260
00261 got = got.replace('type="builtins.', 'type="exceptions.')
00262
00263 self.assertEqual(expected, got)
00264
00265 def test_no_tests(self):
00266 """Regression test: Check whether a test run without any tests
00267 matches a previous run.
00268
00269 """
00270 class TestTest(unittest.TestCase):
00271 pass
00272 self._try_test_run(TestTest, """<testsuite errors="0" failures="0" name="unittest.TestSuite" tests="0" time="0.000">
00273 <system-out><![CDATA[]]></system-out>
00274 <system-err><![CDATA[]]></system-err>
00275 </testsuite>
00276 """)
00277
00278 def test_success(self):
00279 """Regression test: Check whether a test run with a successful test
00280 matches a previous run.
00281
00282 """
00283 class TestTest(unittest.TestCase):
00284 def test_foo(self):
00285 pass
00286 self._try_test_run(TestTest, """<testsuite errors="0" failures="0" name="unittest.TestSuite" tests="1" time="0.000">
00287 <testcase classname="__main__.TestTest" name="test_foo" time="0.000"></testcase>
00288 <system-out><![CDATA[]]></system-out>
00289 <system-err><![CDATA[]]></system-err>
00290 </testsuite>
00291 """)
00292
00293 def test_failure(self):
00294 """Regression test: Check whether a test run with a failing test
00295 matches a previous run.
00296
00297 """
00298 class TestTest(unittest.TestCase):
00299 def test_foo(self):
00300 self.assert_(False)
00301 self._try_test_run(TestTest, """<testsuite errors="0" failures="1" name="unittest.TestSuite" tests="1" time="0.000">
00302 <testcase classname="__main__.TestTest" name="test_foo" time="0.000">
00303 <failure type="exceptions.AssertionError">Foobar</failure>
00304 </testcase>
00305 <system-out><![CDATA[]]></system-out>
00306 <system-err><![CDATA[]]></system-err>
00307 </testsuite>
00308 """)
00309
00310 def test_error(self):
00311 """Regression test: Check whether a test run with a erroneous test
00312 matches a previous run.
00313
00314 """
00315 class TestTest(unittest.TestCase):
00316 def test_foo(self):
00317 raise IndexError()
00318 self._try_test_run(TestTest, """<testsuite errors="1" failures="0" name="unittest.TestSuite" tests="1" time="0.000">
00319 <testcase classname="__main__.TestTest" name="test_foo" time="0.000">
00320 <error type="exceptions.IndexError">Foobar</error>
00321 </testcase>
00322 <system-out><![CDATA[]]></system-out>
00323 <system-err><![CDATA[]]></system-err>
00324 </testsuite>
00325 """)
00326
00327 def test_stdout_capture(self):
00328 """Regression test: Check whether a test run with output to stdout
00329 matches a previous run.
00330
00331 """
00332 class TestTest(unittest.TestCase):
00333 def test_foo(self):
00334 sys.stdout.write("Test\n")
00335 self._try_test_run(TestTest, """<testsuite errors="0" failures="0" name="unittest.TestSuite" tests="1" time="0.000">
00336 <testcase classname="__main__.TestTest" name="test_foo" time="0.000"></testcase>
00337 <system-out><![CDATA[Test
00338 ]]></system-out>
00339 <system-err><![CDATA[]]></system-err>
00340 </testsuite>
00341 """)
00342
00343 def test_stderr_capture(self):
00344 """Regression test: Check whether a test run with output to stderr
00345 matches a previous run.
00346
00347 """
00348 class TestTest(unittest.TestCase):
00349 def test_foo(self):
00350 sys.stderr.write("Test\n")
00351 self._try_test_run(TestTest, """<testsuite errors="0" failures="0" name="unittest.TestSuite" tests="1" time="0.000">
00352 <testcase classname="__main__.TestTest" name="test_foo" time="0.000"></testcase>
00353 <system-out><![CDATA[]]></system-out>
00354 <system-err><![CDATA[Test
00355 ]]></system-err>
00356 </testsuite>
00357 """)
00358
00359 class NullStream(object):
00360 """A file-like object that discards everything written to it."""
00361 def write(self, buffer):
00362 pass
00363
00364 def test_unittests_changing_stdout(self):
00365 """Check whether the XMLTestRunner recovers gracefully from unit tests
00366 that change stdout, but don't change it back properly.
00367
00368 """
00369 class TestTest(unittest.TestCase):
00370 def test_foo(self):
00371 sys.stdout = XMLTestRunnerTest.NullStream()
00372
00373 runner = XMLTestRunner(self._stream)
00374 runner.run(unittest.makeSuite(TestTest))
00375
00376 def test_unittests_changing_stderr(self):
00377 """Check whether the XMLTestRunner recovers gracefully from unit tests
00378 that change stderr, but don't change it back properly.
00379
00380 """
00381 class TestTest(unittest.TestCase):
00382 def test_foo(self):
00383 sys.stderr = XMLTestRunnerTest.NullStream()
00384
00385 runner = XMLTestRunner(self._stream)
00386 runner.run(unittest.makeSuite(TestTest))
00387
00388
00389 if __name__ == "__main__":
00390 unittest.main()