#!/usr/bin/python # Copyright (C) 2010 Google Inc. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above # copyright notice, this list of conditions and the following disclaimer # in the documentation and/or other materials provided with the # distribution. # * Neither the name of Google Inc. nor the names of its # contributors may be used to endorse or promote products derived from # this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Unit tests for printing.py.""" import optparse import unittest import logging from webkitpy.common import array_stream from webkitpy.common.system import logtesting from webkitpy.layout_tests import port from webkitpy.layout_tests.layout_package import printing from webkitpy.layout_tests.layout_package import result_summary from webkitpy.layout_tests.layout_package import test_expectations from webkitpy.layout_tests.layout_package import test_failures from webkitpy.layout_tests.layout_package import test_results from webkitpy.layout_tests.layout_package import test_runner def get_options(args): print_options = printing.print_options() option_parser = optparse.OptionParser(option_list=print_options) return option_parser.parse_args(args) class TestUtilityFunctions(unittest.TestCase): def test_configure_logging(self): options, args = get_options([]) stream = array_stream.ArrayStream() handler = printing._configure_logging(stream, options.verbose) logging.info("this should be logged") self.assertFalse(stream.empty()) stream.reset() logging.debug("this should not be logged") self.assertTrue(stream.empty()) printing._restore_logging(handler) stream.reset() options, args = get_options(['--verbose']) handler = printing._configure_logging(stream, options.verbose) logging.debug("this should be logged") self.assertFalse(stream.empty()) printing._restore_logging(handler) def test_print_options(self): options, args = get_options([]) self.assertTrue(options is not None) def test_parse_print_options(self): def test_switches(args, expected_switches_str, verbose=False, child_processes=1, is_fully_parallel=False): options, args = get_options(args) if expected_switches_str: expected_switches = set(expected_switches_str.split(',')) else: expected_switches = set() switches = printing.parse_print_options(options.print_options, verbose, child_processes, is_fully_parallel) self.assertEqual(expected_switches, switches) # test that we default to the default set of switches test_switches([], printing.PRINT_DEFAULT) # test that verbose defaults to everything test_switches([], printing.PRINT_EVERYTHING, verbose=True) # test that --print default does what it's supposed to test_switches(['--print', 'default'], printing.PRINT_DEFAULT) # test that --print nothing does what it's supposed to test_switches(['--print', 'nothing'], None) # test that --print everything does what it's supposed to test_switches(['--print', 'everything'], printing.PRINT_EVERYTHING) # this tests that '--print X' overrides '--verbose' test_switches(['--print', 'actual'], 'actual', verbose=True) class Testprinter(unittest.TestCase): def get_printer(self, args=None, single_threaded=False, is_fully_parallel=False): args = args or [] printing_options = printing.print_options() option_parser = optparse.OptionParser(option_list=printing_options) options, args = option_parser.parse_args(args) self._port = port.get('test', options) nproc = 2 if single_threaded: nproc = 1 regular_output = array_stream.ArrayStream() buildbot_output = array_stream.ArrayStream() printer = printing.Printer(self._port, options, regular_output, buildbot_output, single_threaded, is_fully_parallel) return printer, regular_output, buildbot_output def get_result(self, test, result_type=test_expectations.PASS, run_time=0): failures = [] if result_type == test_expectations.TIMEOUT: failures = [test_failures.FailureTimeout()] elif result_type == test_expectations.CRASH: failures = [test_failures.FailureCrash()] path = self._port._filesystem.join(self._port.layout_tests_dir(), test) return test_results.TestResult(path, failures=failures, test_run_time=run_time) def get_result_summary(self, tests, expectations_str): test_paths = [self._port._filesystem.join(self._port.layout_tests_dir(), test) for test in tests] expectations = test_expectations.TestExpectations( self._port, test_paths, expectations_str, self._port.test_configuration(), is_lint_mode=False) rs = result_summary.ResultSummary(expectations, test_paths) return test_paths, rs, expectations def test_help_printer(self): # Here and below we'll call the "regular" printer err and the # buildbot printer out; this corresponds to how things run on the # bots with stderr and stdout. printer, err, out = self.get_printer() # This routine should print something to stdout. testing what it is # is kind of pointless. printer.help_printing() self.assertFalse(err.empty()) self.assertTrue(out.empty()) def do_switch_tests(self, method_name, switch, to_buildbot, message='hello', exp_err=None, exp_bot=None): def do_helper(method_name, switch, message, exp_err, exp_bot): printer, err, bot = self.get_printer(['--print', switch]) getattr(printer, method_name)(message) self.assertEqual(err.get(), exp_err) self.assertEqual(bot.get(), exp_bot) if to_buildbot: if exp_err is None: exp_err = [] if exp_bot is None: exp_bot = [message + "\n"] else: if exp_err is None: exp_err = [message + "\n"] if exp_bot is None: exp_bot = [] do_helper(method_name, 'nothing', 'hello', [], []) do_helper(method_name, switch, 'hello', exp_err, exp_bot) do_helper(method_name, 'everything', 'hello', exp_err, exp_bot) def test_configure_and_cleanup(self): # This test verifies that calling cleanup repeatedly and deleting # the object is safe. printer, err, out = self.get_printer(['--print', 'everything']) printer.cleanup() printer.cleanup() printer = None def test_print_actual(self): # Actual results need to be logged to the buildbot's stream. self.do_switch_tests('print_actual', 'actual', to_buildbot=True) def test_print_actual_buildbot(self): # FIXME: Test that the format of the actual results matches what the # buildbot is expecting. pass def test_print_config(self): self.do_switch_tests('print_config', 'config', to_buildbot=False) def test_print_expected(self): self.do_switch_tests('print_expected', 'expected', to_buildbot=False) def test_print_timing(self): self.do_switch_tests('print_timing', 'timing', to_buildbot=False) def test_print_update(self): # Note that there shouldn't be a carriage return here; updates() # are meant to be overwritten. self.do_switch_tests('print_update', 'updates', to_buildbot=False, message='hello', exp_err=['hello']) def test_print_one_line_summary(self): printer, err, out = self.get_printer(['--print', 'nothing']) printer.print_one_line_summary(1, 1, 0) self.assertTrue(err.empty()) printer, err, out = self.get_printer(['--print', 'one-line-summary']) printer.print_one_line_summary(1, 1, 0) self.assertEquals(err.get(), ["All 1 tests ran as expected.\n", "\n"]) printer, err, out = self.get_printer(['--print', 'everything']) printer.print_one_line_summary(1, 1, 0) self.assertEquals(err.get(), ["All 1 tests ran as expected.\n", "\n"]) err.reset() printer.print_one_line_summary(2, 1, 1) self.assertEquals(err.get(), ["1 test ran as expected, 1 didn't:\n", "\n"]) err.reset() printer.print_one_line_summary(3, 2, 1) self.assertEquals(err.get(), ["2 tests ran as expected, 1 didn't:\n", "\n"]) err.reset() printer.print_one_line_summary(3, 2, 0) self.assertEquals(err.get(), ['\n', "2 tests ran as expected (1 didn't run).\n", '\n']) def test_print_test_result(self): # Note here that we don't use meaningful exp_str and got_str values; # the actual contents of the string are treated opaquely by # print_test_result() when tracing, and usually we don't want # to test what exactly is printed, just that something # was printed (or that nothing was printed). # # FIXME: this is actually some goofy layering; it would be nice # we could refactor it so that the args weren't redundant. Maybe # the TestResult should contain what was expected, and the # strings could be derived from the TestResult? printer, err, out = self.get_printer(['--print', 'nothing']) result = self.get_result('passes/image.html') printer.print_test_result(result, expected=False, exp_str='', got_str='') self.assertTrue(err.empty()) printer, err, out = self.get_printer(['--print', 'unexpected']) printer.print_test_result(result, expected=True, exp_str='', got_str='') self.assertTrue(err.empty()) printer.print_test_result(result, expected=False, exp_str='', got_str='') self.assertEquals(err.get(), [' passes/image.html -> unexpected pass\n']) printer, err, out = self.get_printer(['--print', 'everything']) printer.print_test_result(result, expected=True, exp_str='', got_str='') self.assertTrue(err.empty()) printer.print_test_result(result, expected=False, exp_str='', got_str='') self.assertEquals(err.get(), [' passes/image.html -> unexpected pass\n']) printer, err, out = self.get_printer(['--print', 'nothing']) printer.print_test_result(result, expected=False, exp_str='', got_str='') self.assertTrue(err.empty()) printer, err, out = self.get_printer(['--print', 'trace-unexpected']) printer.print_test_result(result, expected=True, exp_str='', got_str='') self.assertTrue(err.empty()) printer, err, out = self.get_printer(['--print', 'trace-unexpected']) printer.print_test_result(result, expected=False, exp_str='', got_str='') self.assertFalse(err.empty()) printer, err, out = self.get_printer(['--print', 'trace-unexpected']) result = self.get_result("passes/text.html") printer.print_test_result(result, expected=False, exp_str='', got_str='') self.assertFalse(err.empty()) err.reset() printer.print_test_result(result, expected=False, exp_str='', got_str='') self.assertFalse(err.empty()) printer, err, out = self.get_printer(['--print', 'trace-everything']) result = self.get_result('passes/image.html') printer.print_test_result(result, expected=True, exp_str='', got_str='') result = self.get_result('failures/expected/missing_text.html') printer.print_test_result(result, expected=True, exp_str='', got_str='') result = self.get_result('failures/expected/missing_check.html') printer.print_test_result(result, expected=True, exp_str='', got_str='') result = self.get_result('failures/expected/missing_image.html') printer.print_test_result(result, expected=True, exp_str='', got_str='') self.assertFalse(err.empty()) err.reset() printer.print_test_result(result, expected=False, exp_str='', got_str='') def test_print_progress(self): expectations = '' # test that we print nothing printer, err, out = self.get_printer(['--print', 'nothing']) tests = ['passes/text.html', 'failures/expected/timeout.html', 'failures/expected/crash.html'] paths, rs, exp = self.get_result_summary(tests, expectations) printer.print_progress(rs, False, paths) self.assertTrue(out.empty()) self.assertTrue(err.empty()) printer.print_progress(rs, True, paths) self.assertTrue(out.empty()) self.assertTrue(err.empty()) # test regular functionality printer, err, out = self.get_printer(['--print', 'one-line-progress']) printer.print_progress(rs, False, paths) self.assertTrue(out.empty()) self.assertFalse(err.empty()) err.reset() out.reset() printer.print_progress(rs, True, paths) self.assertFalse(err.empty()) self.assertTrue(out.empty()) def test_print_progress__detailed(self): tests = ['passes/text.html', 'failures/expected/timeout.html', 'failures/expected/crash.html'] expectations = 'BUGX : failures/expected/timeout.html = TIMEOUT' # first, test that it is disabled properly # should still print one-line-progress printer, err, out = self.get_printer( ['--print', 'detailed-progress'], single_threaded=False) paths, rs, exp = self.get_result_summary(tests, expectations) printer.print_progress(rs, False, paths) self.assertFalse(err.empty()) self.assertTrue(out.empty()) # now test the enabled paths printer, err, out = self.get_printer( ['--print', 'detailed-progress'], single_threaded=True) paths, rs, exp = self.get_result_summary(tests, expectations) printer.print_progress(rs, False, paths) self.assertFalse(err.empty()) self.assertTrue(out.empty()) err.reset() out.reset() printer.print_progress(rs, True, paths) self.assertFalse(err.empty()) self.assertTrue(out.empty()) rs.add(self.get_result('passes/text.html', test_expectations.TIMEOUT), False) rs.add(self.get_result('failures/expected/timeout.html'), True) rs.add(self.get_result('failures/expected/crash.html', test_expectations.CRASH), True) err.reset() out.reset() printer.print_progress(rs, False, paths) self.assertFalse(err.empty()) self.assertTrue(out.empty()) # We only clear the meter when retrying w/ detailed-progress. err.reset() out.reset() printer.print_progress(rs, True, paths) self.assertFalse(err.empty()) self.assertTrue(out.empty()) printer, err, out = self.get_printer( ['--print', 'detailed-progress,unexpected'], single_threaded=True) paths, rs, exp = self.get_result_summary(tests, expectations) printer.print_progress(rs, False, paths) self.assertFalse(err.empty()) self.assertTrue(out.empty()) err.reset() out.reset() printer.print_progress(rs, True, paths) self.assertFalse(err.empty()) self.assertTrue(out.empty()) rs.add(self.get_result('passes/text.html', test_expectations.TIMEOUT), False) rs.add(self.get_result('failures/expected/timeout.html'), True) rs.add(self.get_result('failures/expected/crash.html', test_expectations.CRASH), True) err.reset() out.reset() printer.print_progress(rs, False, paths) self.assertFalse(err.empty()) self.assertTrue(out.empty()) # We only clear the meter when retrying w/ detailed-progress. err.reset() out.reset() printer.print_progress(rs, True, paths) self.assertFalse(err.empty()) self.assertTrue(out.empty()) def test_write_nothing(self): printer, err, out = self.get_printer(['--print', 'nothing']) printer.write("foo") self.assertTrue(err.empty()) def test_write_misc(self): printer, err, out = self.get_printer(['--print', 'misc']) printer.write("foo") self.assertFalse(err.empty()) err.reset() printer.write("foo", "config") self.assertTrue(err.empty()) def test_write_everything(self): printer, err, out = self.get_printer(['--print', 'everything']) printer.write("foo") self.assertFalse(err.empty()) err.reset() printer.write("foo", "config") self.assertFalse(err.empty()) def test_write_verbose(self): printer, err, out = self.get_printer(['--verbose']) printer.write("foo") self.assertTrue(not err.empty() and "foo" in err.get()[0]) self.assertTrue(out.empty()) def test_print_unexpected_results(self): # This routine is the only one that prints stuff that the bots # care about. # # FIXME: there's some weird layering going on here. It seems # like we shouldn't be both using an expectations string and # having to specify whether or not the result was expected. # This whole set of tests should probably be rewritten. # # FIXME: Plus, the fact that we're having to call into # run_webkit_tests is clearly a layering inversion. def get_unexpected_results(expected, passing, flaky): """Return an unexpected results summary matching the input description. There are a lot of different combinations of test results that can be tested; this routine produces various combinations based on the values of the input flags. Args expected: whether the tests ran as expected passing: whether the tests should all pass flaky: whether the tests should be flaky (if False, they produce the same results on both runs; if True, they all pass on the second run). """ paths, rs, exp = self.get_result_summary(tests, expectations) if expected: rs.add(self.get_result('passes/text.html', test_expectations.PASS), expected) rs.add(self.get_result('failures/expected/timeout.html', test_expectations.TIMEOUT), expected) rs.add(self.get_result('failures/expected/crash.html', test_expectations.CRASH), expected) elif passing: rs.add(self.get_result('passes/text.html'), expected) rs.add(self.get_result('failures/expected/timeout.html'), expected) rs.add(self.get_result('failures/expected/crash.html'), expected) else: rs.add(self.get_result('passes/text.html', test_expectations.TIMEOUT), expected) rs.add(self.get_result('failures/expected/timeout.html', test_expectations.CRASH), expected) rs.add(self.get_result('failures/expected/crash.html', test_expectations.TIMEOUT), expected) retry = rs if flaky: paths, retry, exp = self.get_result_summary(tests, expectations) retry.add(self.get_result('passes/text.html'), True) retry.add(self.get_result('failures/expected/timeout.html'), True) retry.add(self.get_result('failures/expected/crash.html'), True) unexpected_results = test_runner.summarize_results(self._port, exp, rs, retry, test_timings={}, only_unexpected=True) return unexpected_results tests = ['passes/text.html', 'failures/expected/timeout.html', 'failures/expected/crash.html'] expectations = '' printer, err, out = self.get_printer(['--print', 'nothing']) ur = get_unexpected_results(expected=False, passing=False, flaky=False) printer.print_unexpected_results(ur) self.assertTrue(err.empty()) self.assertTrue(out.empty()) printer, err, out = self.get_printer(['--print', 'unexpected-results']) # test everything running as expected ur = get_unexpected_results(expected=True, passing=False, flaky=False) printer.print_unexpected_results(ur) self.assertTrue(err.empty()) self.assertTrue(out.empty()) # test failures err.reset() out.reset() ur = get_unexpected_results(expected=False, passing=False, flaky=False) printer.print_unexpected_results(ur) self.assertTrue(err.empty()) self.assertFalse(out.empty()) # test unexpected flaky results err.reset() out.reset() ur = get_unexpected_results(expected=False, passing=True, flaky=False) printer.print_unexpected_results(ur) self.assertTrue(err.empty()) self.assertFalse(out.empty()) # test unexpected passes err.reset() out.reset() ur = get_unexpected_results(expected=False, passing=False, flaky=True) printer.print_unexpected_results(ur) self.assertTrue(err.empty()) self.assertFalse(out.empty()) err.reset() out.reset() printer, err, out = self.get_printer(['--print', 'everything']) ur = get_unexpected_results(expected=False, passing=False, flaky=False) printer.print_unexpected_results(ur) self.assertTrue(err.empty()) self.assertFalse(out.empty()) expectations = """ BUGX : failures/expected/crash.html = CRASH BUGX : failures/expected/timeout.html = TIMEOUT """ err.reset() out.reset() ur = get_unexpected_results(expected=False, passing=False, flaky=False) printer.print_unexpected_results(ur) self.assertTrue(err.empty()) self.assertFalse(out.empty()) err.reset() out.reset() ur = get_unexpected_results(expected=False, passing=True, flaky=False) printer.print_unexpected_results(ur) self.assertTrue(err.empty()) self.assertFalse(out.empty()) # Test handling of --verbose as well. err.reset() out.reset() printer, err, out = self.get_printer(['--verbose']) ur = get_unexpected_results(expected=False, passing=False, flaky=False) printer.print_unexpected_results(ur) self.assertTrue(err.empty()) self.assertFalse(out.empty()) def test_print_unexpected_results_buildbot(self): # FIXME: Test that print_unexpected_results() produces the printer the # buildbot is expecting. pass if __name__ == '__main__': unittest.main()