# Copyright (c) 2016 CEF Python, see the Authors file. # All rights reserved. Licensed under BSD 3-clause license. # Project website: https://github.com/cztomczak/cefpython """Run unit tests. With no arguments all tests are run. Read notes below. Usage: _test_runner.py [--debug] [FILE | _TESTCASE] Options: --debug Enable debug info FILE Run tests from single file _TESTCASE Test cases matching pattern to run eg "file.TestCase". Calling with this argument is for internal use only. It has side effects, so don't use it. See comments. Notes: - Files starting with "_" are ignored - If test case name contains "IsolatedTest" word then this test case will be run using a new instance of Python interpreter. In such case instead of calling "unittest.main()" use this code: "import _runner; _runner.main(os.path.basename(__file__))". - Tested only with TestCase objects. TestSuite usage is untested. """ import unittest import os import platform import sys from os.path import dirname, realpath import re import subprocess # Command line args CUSTOM_CMDLINE_ARG = "" def main(file_arg=""): # type: (str) -> None """Main entry point.""" # Set working dir to script's current os.chdir(dirname(realpath(__file__))) # Script arguments testcase_arg = "" if len(sys.argv) > 1 and sys.argv[1].startswith("--"): # Will allow to pass custom args like --debug to isolated tests # (main_test.py for example). global CUSTOM_CMDLINE_ARG CUSTOM_CMDLINE_ARG = sys.argv[1] elif len(sys.argv) > 1: if ".py" in sys.argv[1]: file_arg = sys.argv[1] else: testcase_arg = sys.argv[1] # Run tests runner = TestRunner() if testcase_arg: runner.run_testcase(testcase_arg) elif file_arg: runner.run_file(file_arg) else: runner.run_all() class TestRunner(object): """Customized test runner.""" ran = 0 errors = 0 failures = 0 cefpython_version = "-unknown-" _suites = None # type: unittest.TestSuite _isolated_suites = None # type: unittest.TestSuite _import_errors = None # type: unittest.TestSuite def _reset_state(self): # type: () -> None """Reset TestRunner state before test discovery.""" self.ran = 0 self.errors = 0 self.failures = 0 self._suites = unittest.TestSuite() self._isolated_suites = unittest.TestSuite() self._import_errors = unittest.TestSuite() # ---- Public methods def run_testcase(self, testcase): # type: (str) -> None """Run single test case eg 'foo.BarTest'. This is needed to run single testcase that is marked as IsolatedTest.""" self._discover("[!_]*.py", testcase) assert not self._count_suites(self._isolated_suites) if not self._count_suites(self._suites): print("[_test_runner.py] ERROR: test case not found") sys.exit(1) # Import errors found during discovery are ignored self._run_suites(self._suites) self._exit() def run_file(self, filename): # type: (str) -> None """Run test cases from a specific file. This is needed so that you can use _runner.main() in isolated tests.""" self._discover(filename) self._run_discovered_suites() def run_all(self): # type: () -> None """Run all tests from current directory.""" self._discover("[!_]*.py") self._run_discovered_suites() # ---- Private methods def _run_discovered_suites(self): # type: () -> None """Run both normal and isolated suites.""" suites = self._merge_suites(self._import_errors, self._suites) self._run_suites(suites) self._run_suites_in_isolation(self._isolated_suites) self._print_summary() def _run_suites(self, suites): # type: (unittest.TestSuite) -> None """Run suites.""" if not self._count_suites(suites): return runner = unittest.TextTestRunner(verbosity=2, descriptions=True, buffer=False) # Update "ran" before running suites, because after ran # counting them doesn't work (Python 3 issue). self.ran += self._count_suites(suites) result = runner.run(suites) self.errors += len(result.errors) self.failures += len(result.failures) def _run_suites_in_isolation(self, suites): # type: (unittest.TestSuite) -> None """Run each suite using new instance of Python interpreter.""" if not self._count_suites(suites): return for suite in suites: # Find test case identifier testcase_id = "" for testcase in suite: testcase_id = testcase.id() break # Run test using new instance of Python interpreter try: output = subprocess.check_output( [sys.executable, "_test_runner.py", testcase_id, CUSTOM_CMDLINE_ARG], stderr=subprocess.STDOUT) exit_code = 0 except subprocess.CalledProcessError as exc: output = exc.output exit_code = exc.returncode if type(output) != str: output = output.decode("utf-8", errors="replace") # Fetch number of sub-tests ran from output match = re.search(r"^Ran (\d+) sub-tests in \w+", output, re.MULTILINE) if match: self.ran += int(match.group(1)) # Fetch CEF Python version from output match = re.search(r"^CEF Python (\d+\.\d+)", output, re.MULTILINE) if match: self.cefpython_version = match.group(1) # Write original output sys.stdout.write(output) # If tests failed parse output for errors/failures if exit_code: if output: lines = output.splitlines() lastline = lines[len(lines)-1] match = re.search(r"failures=(\d+)", lastline) if match: self.failures += int(match.group(1)) match = re.search(r"errors=(\d+)", lastline) if match: self.errors += int(match.group(1)) if not self.errors and not self.failures: self.errors += 1 elif output: # Test case still might have failed and unittest would not # detect this. For example when assertion fails # in ClientHandler in core_test.py . if "Traceback (most recent call last)" in output\ or "AssertionError" in output: self.errors += 1 # Update ran self.ran += self._count_suites(suites) def _count_suites(self, suites): # type: (unittest.TestSuite) -> int count = 0 for suite in suites: if isinstance(suite, unittest.TestSuite): for _ in suite: count += 1 return count def _merge_suites(self, suites1, suites2): # type: (unittest.TestSuite, unittest.TestSuite) -> unittest.TestSuite merged = unittest.TestSuite() for suite in suites1: merged.addTest(suite) for suite in suites2: merged.addTest(suite) return merged def _discover(self, pattern, testcase_name=""): # type: (str, str) -> None """Test discovery using glob pattern from arg or main().""" self._reset_state() loader = unittest.TestLoader() discovered_suite = loader.discover(start_dir=".", pattern=pattern) for level1_suite in discovered_suite: for level2_suite in level1_suite: if isinstance(level2_suite, unittest.TestSuite): for testcase_obj in level2_suite: if testcase_name: if re.match(re.escape(testcase_name), testcase_obj.id()): self._suites.addTest(level2_suite) break elif "IsolatedTest" in testcase_obj.id(): self._isolated_suites.addTest(level2_suite) break else: self._suites.addTest(level2_suite) break elif not testcase_name: # unittest.loader.ModuleImportFailure # Warning: If there is an import error in a file # containing a test case that is being run then there # won't be any error displayed about it when running # that single test case. However running a single test # case is for internal use only, to run test cases in # isolation with a new instance of Python interpreter. # Import errors will always be showed when running all # tests or tests from file. self._import_errors.addTest(level2_suite) def _print_summary(self): # type: () -> None """Print summary and exit.""" print("-"*70) print("[_test_runner.py] CEF Python {ver}" .format(ver=self.cefpython_version)) print("[_test_runner.py] Python {ver} {arch}" .format(ver=platform.python_version(), arch=platform.architecture()[0])) print("[_test_runner.py] Ran {ran} tests in total" .format(ran=self.ran)) if self.errors or self.failures: failed_str = "[_test_runner.py] FAILED (" if self.failures: failed_str += ("failures="+str(self.failures)) if self.errors: if self.failures: failed_str += ", " failed_str += ("errors="+str(self.errors)) failed_str += ")" print(failed_str) else: print("[_test_runner.py] OK all unit tests succeeded") self._exit() def _exit(self): # type: () -> None """Exit with appropriate exit code.""" if self.errors or self.failures: sys.exit(1) else: sys.exit(0) if __name__ == "__main__": main()