samedi 25 avril 2015

Overriding Python Unit Test module for custom output?

Aim:

I want to rewrite Python's UnitTest module so that when I call it I get the following JSON output within the sdout stream:

{  
   "errors":0,
   "failures":1,
   "ran":3,
   "skipped":0,
   "successful":2,
   "test_data":[  
      {  
         "index":0,
         "result":1
      },
      {  
         "index":1,
         "result":1
      },
      {  
         "index":2,
         "result":-1
      }
   ]
}

Working code:

This is the working code I've written / modified to generate the test results in JSON format:

#!/usr/bin/python
import unittest
import sys, os
import json
from user_code import *

class MyTest(unittest.TestCase):

    currentResult = None # holds last result object passed to run method

    @classmethod
    def setResult(cls, amount, errors, failures, skipped):
        cls.amount, cls.errors, cls.failures, cls.skipped = \
            amount, errors, failures, skipped

    def tearDown(self):
        amount = self.currentResult.testsRun
        errors = self.currentResult.errors
        failures = self.currentResult.failures
        skipped = self.currentResult.skipped
        self.setResult(amount, errors, failures, skipped)

    @classmethod
    def tearDownClass(cls):
        print json.dumps(
            {
                'ran': cls.amount, 
                'errors': len(cls.errors),
                'failures': len(cls.failures),
                'succeeded': cls.amount - len(cls.errors) - len(cls.failures),
                'skipped': len(cls.skipped)
            }, 
            sort_keys=True, indent=4, separators=(',', ': ')
        )
        return

    def run(self, result=None):
        self.currentResult = result # remember result for use in tearDown
        unittest.TestCase.run(self, result) # call superclass run method

    # Tests are defined below
    def test_1(self):
        self.assertEqual(1, 1)

# Lets run our tests
if __name__ == '__main__':
    unittest.TextTestRunner( descriptions=0, verbosity = 0 )
    unittest.main(exit=False)

Problem / non working code:

Now I want to add the "test_data" attribute to my JSON array so I can get individual test details. I've looked through the UnitTest module source code and it seems this is possible by overriding the TestCase, TextTestResult and TextTestRunner classes. I've attempted this by hacking together the following code:

#!/usr/bin/python

import unittest
import sys, os
import json

class MyTestRunner(unittest.TextTestRunner):
    def _makeResult(self):
        return MyTestResult(self.stream, self.descriptions, self.verbosity)

class MyTestResult(unittest._TextTestResult):

    """
        Holder for test result information.
        Test results are automatically managed by the TestCase and TestSuite
        classes, and do not need to be explicitly manipulated by writers of tests.

        Each instance holds the total number of tests run, and collections of
        failures and errors that occurred among those test runs. The collections
        contain tuples of (testcase, exceptioninfo), where exceptioninfo is the
        formatted traceback of the error that occurred.
    """

    _previousTestClass = None
    _testRunEntered = False
    _moduleSetUpFailed = False

    def __init__(self, stream=None, descriptions=None, verbosity=None):
        print "foo" # not printing?

        self.failfast = False
        self.failures = []
        self.errors = []
        self.testsRun = 0
        self.skipped = []
        self.expectedFailures = []
        self.unexpectedSuccesses = []
        self.shouldStop = False
        self.buffer = False
        self._stdout_buffer = None
        self._stderr_buffer = None
        self._original_stdout = sys.stdout
        self._original_stderr = sys.stderr
        self._mirrorOutput = False

        # List containing all the run tests, their index and their result. This is the new line of code.
        self.tests_run = []

    ###
    ### New function added
    ###
    def getTestsReport(self):

        """Returns the run tests as a list of the form [test_description, test_index, result]"""
        return self.tests_run

    ###
    ### Modified the functions so that we add the test case to the tests run list.
    ### -1 means Failure. 0 means error. 1 means success. 
    ###
    def addError(self, test, err):

        """
            Called when an error has occurred. 'err' is a tuple of values as
            returned by sys.exc_info().
        """
        self.errors.append((test, self._exc_info_to_string(err, test)))
        self._mirrorOutput = True
        self.tests_run.append([test.shortDescription(), self.testsRun, 0])
        TestResult.addError(self, test, err)

    def addFailure(self, test, err):

        """
            Called when an error has occurred. 'err' is a tuple of values as
            returned by sys.exc_info().
        """
        self.failures.append((test, self._exc_info_to_string(err, test)))
        self._mirrorOutput = True
        self.tests_run.append([test.shortDescription(), self.testsRun, -1])
        TestResult.addFailure(self, test, err)

    def addSuccess(self, test):

        "Called when a test has completed successfully"
        self.tests_run.append([test.shortDescription(), self.testsRun, 1])
        TestResult.addSuccess(self, test)

class MyTest(unittest.TestCase):

    currentResult = None # holds last result object passed to run method
    results = [] # Holds all results so we can report back to the CCC backend

    def setResult(cls, amount, errors, failures, skipped):
        cls.amount, cls.errors, cls.failures, cls.skipped = amount, errors, failures, skipped

    @classmethod
    def tearDown(self):
        amount = self.currentResult.testsRun
        errors = self.currentResult.errors
        failures = self.currentResult.failures
        skipped = self.currentResult.skipped
        self.setResult(amount, errors, failures, skipped)

    @classmethod
    def tearDownClass(cls):
        print json.dumps(
            {
                'ran': cls.amount, 
                'errors': len(cls.errors),
                'failures': len(cls.failures),
                'succeeded': cls.amount - len(cls.errors) - len(cls.failures),
                'skipped': len(cls.skipped),
                'results': cls.results
            }, 
            sort_keys=False, indent=4, separators=(',', ': ')
        )
        return

    def run(self, result=None):
        self.currentResult = result # remember result for use in tearDown
        unittest.TestCase.run(self, result)


    # Tests are defined below.
    def test_something(self):
        self.assertEqual(1, 1)

if __name__ == '__main__':
    MyTestRunner( stream=None, descriptions=0, verbosity=0 )
    unittest.main(exit=False)

But when I run this code I get the following error:

======================================================================
ERROR: test_something (__main__.MyTest)
----------------------------------------------------------------------
Traceback (most recent call last):
  File "tests.py", line 100, in tearDown
    amount = self.currentResult.testsRun
AttributeError: 'NoneType' object has no attribute 'testsRun'

======================================================================
ERROR: tearDownClass (__main__.MyTest)
----------------------------------------------------------------------
Traceback (most recent call last):
  File "tests.py", line 110, in tearDownClass
    'ran': cls.amount,
AttributeError: type object 'MyTest' has no attribute 'amount'

----------------------------------------------------------------------
Ran 1 test in 0.000s

FAILED (errors=2)

How can I fix these errors? The attributes mentioned in the stack trace are defined, so what is going wrong? I've tried debugging this code for hours, but I'm not getting anywhere so I thought I'd ask here for a little bit of help.

1 commentaire: