1#!/usr/bin/env python3 2# 3# Copyright 2016 - The Android Open Source Project 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16 17import fnmatch 18import functools 19import importlib 20import logging 21import os 22import traceback 23from concurrent.futures import ThreadPoolExecutor 24 25from acts import asserts 26from acts import error 27from acts import keys 28from acts import logger 29from acts import records 30from acts import signals 31from acts import tracelogger 32from acts import utils 33from acts.event import event_bus 34from acts.event import subscription_bundle 35from acts.event.decorators import subscribe_static 36from acts.event.event import TestCaseBeginEvent 37from acts.event.event import TestCaseEndEvent 38from acts.event.event import TestClassBeginEvent 39from acts.event.event import TestClassEndEvent 40from acts.event.subscription_bundle import SubscriptionBundle 41 42from mobly.base_test import BaseTestClass as MoblyBaseTest 43from mobly.records import ExceptionRecord 44 45# Macro strings for test result reporting 46TEST_CASE_TOKEN = "[Test Case]" 47RESULT_LINE_TEMPLATE = TEST_CASE_TOKEN + " %s %s" 48 49 50@subscribe_static(TestCaseBeginEvent) 51def _logcat_log_test_begin(event): 52 """Ensures that logcat is running. Write a logcat line indicating test case 53 begin.""" 54 test_instance = event.test_class 55 try: 56 for ad in getattr(test_instance, 'android_devices', []): 57 if not ad.is_adb_logcat_on: 58 ad.start_adb_logcat() 59 # Write test start token to adb log if android device is attached. 60 if not ad.skip_sl4a: 61 ad.droid.logV("%s BEGIN %s" % 62 (TEST_CASE_TOKEN, event.test_case_name)) 63 64 except error.ActsError as e: 65 test_instance.results.error.append( 66 ExceptionRecord(e, 'Logcat for test begin: %s' % 67 event.test_case_name)) 68 test_instance.log.error('BaseTest setup_test error: %s' % e.message) 69 except Exception as e: 70 test_instance.log.warning( 71 'Unable to send BEGIN log command to all devices.') 72 test_instance.log.warning('Error: %s' % e) 73 74 75@subscribe_static(TestCaseEndEvent) 76def _logcat_log_test_end(event): 77 """Write a logcat line indicating test case end.""" 78 test_instance = event.test_class 79 try: 80 # Write test end token to adb log if android device is attached. 81 for ad in getattr(test_instance, 'android_devices', []): 82 if not ad.skip_sl4a: 83 ad.droid.logV("%s END %s" % 84 (TEST_CASE_TOKEN, event.test_case_name)) 85 86 except error.ActsError as e: 87 test_instance.results.error.append( 88 ExceptionRecord(e, 89 'Logcat for test end: %s' % event.test_case_name)) 90 test_instance.log.error('BaseTest teardown_test error: %s' % e.message) 91 except Exception as e: 92 test_instance.log.warning( 93 'Unable to send END log command to all devices.') 94 test_instance.log.warning('Error: %s' % e) 95 96 97@subscribe_static(TestCaseBeginEvent) 98def _syslog_log_test_begin(event): 99 """This adds a BEGIN log message with the test name to the syslog of any 100 Fuchsia device""" 101 test_instance = event.test_class 102 try: 103 for fd in getattr(test_instance, 'fuchsia_devices', []): 104 if not fd.skip_sl4f: 105 fd.logging_lib.logI("%s BEGIN %s" % 106 (TEST_CASE_TOKEN, event.test_case_name)) 107 108 except Exception as e: 109 test_instance.log.warning( 110 'Unable to send BEGIN log command to all devices.') 111 test_instance.log.warning('Error: %s' % e) 112 113 114@subscribe_static(TestCaseEndEvent) 115def _syslog_log_test_end(event): 116 """This adds a END log message with the test name to the syslog of any 117 Fuchsia device""" 118 test_instance = event.test_class 119 try: 120 for fd in getattr(test_instance, 'fuchsia_devices', []): 121 if not fd.skip_sl4f: 122 fd.logging_lib.logI("%s END %s" % 123 (TEST_CASE_TOKEN, event.test_case_name)) 124 125 except Exception as e: 126 test_instance.log.warning( 127 'Unable to send END log command to all devices.') 128 test_instance.log.warning('Error: %s' % e) 129 130 131event_bus.register_subscription(_logcat_log_test_begin.subscription) 132event_bus.register_subscription(_logcat_log_test_end.subscription) 133event_bus.register_subscription(_syslog_log_test_begin.subscription) 134event_bus.register_subscription(_syslog_log_test_end.subscription) 135 136 137class Error(Exception): 138 """Raised for exceptions that occured in BaseTestClass.""" 139 140 141class BaseTestClass(MoblyBaseTest): 142 """Base class for all test classes to inherit from. Inherits some 143 functionality from Mobly's base test class. 144 145 This class gets all the controller objects from test_runner and executes 146 the test cases requested within itself. 147 148 Most attributes of this class are set at runtime based on the configuration 149 provided. 150 151 Attributes: 152 tests: A list of strings, each representing a test case name. 153 TAG: A string used to refer to a test class. Default is the test class 154 name. 155 log: A logger object used for logging. 156 results: A records.TestResult object for aggregating test results from 157 the execution of test cases. 158 controller_configs: A dict of controller configs provided by the user 159 via the testbed config. 160 consecutive_failures: Tracks the number of consecutive test case 161 failures within this class. 162 consecutive_failure_limit: Number of consecutive test failures to allow 163 before blocking remaining tests in the same 164 test class. 165 size_limit_reached: True if the size of the log directory has reached 166 its limit. 167 current_test_name: A string that's the name of the test case currently 168 being executed. If no test is executing, this should 169 be None. 170 """ 171 172 TAG = None 173 174 def __init__(self, configs): 175 """Initializes a BaseTestClass given a TestRunConfig, which provides 176 all of the config information for this test class. 177 178 Args: 179 configs: A config_parser.TestRunConfig object. 180 """ 181 super().__init__(configs) 182 183 self.__handle_file_user_params() 184 185 self.class_subscriptions = SubscriptionBundle() 186 self.class_subscriptions.register() 187 self.all_subscriptions = [self.class_subscriptions] 188 189 self.current_test_name = None 190 self.log = tracelogger.TraceLogger(logging.getLogger()) 191 # TODO: remove after converging log path definitions with mobly 192 self.log_path = configs.log_path 193 194 self.consecutive_failures = 0 195 self.consecutive_failure_limit = self.user_params.get( 196 'consecutive_failure_limit', -1) 197 self.size_limit_reached = False 198 self.retryable_exceptions = signals.TestFailure 199 200 def _import_builtin_controllers(self): 201 """Import built-in controller modules. 202 203 Go through the testbed configs, find any built-in controller configs 204 and import the corresponding controller module from acts.controllers 205 package. 206 207 Returns: 208 A list of controller modules. 209 """ 210 builtin_controllers = [] 211 for ctrl_name in keys.Config.builtin_controller_names.value: 212 if ctrl_name in self.controller_configs: 213 module_name = keys.get_module_name(ctrl_name) 214 module = importlib.import_module("acts.controllers.%s" % 215 module_name) 216 builtin_controllers.append(module) 217 return builtin_controllers 218 219 def __handle_file_user_params(self): 220 """For backwards compatibility, moves all contents of the "files" dict 221 into the root level of user_params. 222 223 This allows existing tests to run with the new Mobly-style format 224 without needing to make changes. 225 """ 226 for key, value in self.user_params.items(): 227 if key.endswith('files') and isinstance(value, dict): 228 new_user_params = dict(value) 229 new_user_params.update(self.user_params) 230 self.user_params = new_user_params 231 break 232 233 @staticmethod 234 def get_module_reference_name(a_module): 235 """Returns the module's reference name. 236 237 This is largely for backwards compatibility with log parsing. If the 238 module defines ACTS_CONTROLLER_REFERENCE_NAME, it will return that 239 value, or the module's submodule name. 240 241 Args: 242 a_module: Any module. Ideally, a controller module. 243 Returns: 244 A string corresponding to the module's name. 245 """ 246 if hasattr(a_module, 'ACTS_CONTROLLER_REFERENCE_NAME'): 247 return a_module.ACTS_CONTROLLER_REFERENCE_NAME 248 else: 249 return a_module.__name__.split('.')[-1] 250 251 def register_controller(self, 252 controller_module, 253 required=True, 254 builtin=False): 255 """Registers an ACTS controller module for a test class. Invokes Mobly's 256 implementation of register_controller. 257 258 An ACTS controller module is a Python lib that can be used to control 259 a device, service, or equipment. To be ACTS compatible, a controller 260 module needs to have the following members: 261 262 def create(configs): 263 [Required] Creates controller objects from configurations. 264 Args: 265 configs: A list of serialized data like string/dict. Each 266 element of the list is a configuration for a 267 controller object. 268 Returns: 269 A list of objects. 270 271 def destroy(objects): 272 [Required] Destroys controller objects created by the create 273 function. Each controller object shall be properly cleaned up 274 and all the resources held should be released, e.g. memory 275 allocation, sockets, file handlers etc. 276 Args: 277 A list of controller objects created by the create function. 278 279 def get_info(objects): 280 [Optional] Gets info from the controller objects used in a test 281 run. The info will be included in test_result_summary.json under 282 the key "ControllerInfo". Such information could include unique 283 ID, version, or anything that could be useful for describing the 284 test bed and debugging. 285 Args: 286 objects: A list of controller objects created by the create 287 function. 288 Returns: 289 A list of json serializable objects, each represents the 290 info of a controller object. The order of the info object 291 should follow that of the input objects. 292 Registering a controller module declares a test class's dependency the 293 controller. If the module config exists and the module matches the 294 controller interface, controller objects will be instantiated with 295 corresponding configs. The module should be imported first. 296 297 Args: 298 controller_module: A module that follows the controller module 299 interface. 300 required: A bool. If True, failing to register the specified 301 controller module raises exceptions. If False, returns None upon 302 failures. 303 builtin: Specifies that the module is a builtin controller module in 304 ACTS. If true, adds itself to test attributes. 305 Returns: 306 A list of controller objects instantiated from controller_module, or 307 None. 308 309 Raises: 310 When required is True, ControllerError is raised if no corresponding 311 config can be found. 312 Regardless of the value of "required", ControllerError is raised if 313 the controller module has already been registered or any other error 314 occurred in the registration process. 315 """ 316 module_ref_name = self.get_module_reference_name(controller_module) 317 module_config_name = controller_module.MOBLY_CONTROLLER_CONFIG_NAME 318 319 # Get controller objects from Mobly's register_controller 320 controllers = self._controller_manager.register_controller( 321 controller_module, required=required) 322 if not controllers: 323 return None 324 325 # Log controller information 326 # Implementation of "get_info" is optional for a controller module. 327 if hasattr(controller_module, "get_info"): 328 controller_info = controller_module.get_info(controllers) 329 self.log.info("Controller %s: %s", module_config_name, 330 controller_info) 331 332 if builtin: 333 setattr(self, module_ref_name, controllers) 334 return controllers 335 336 def _setup_class(self): 337 """Proxy function to guarantee the base implementation of setup_class 338 is called. 339 """ 340 event_bus.post(TestClassBeginEvent(self)) 341 # Import and register the built-in controller modules specified 342 # in testbed config. 343 for module in self._import_builtin_controllers(): 344 self.register_controller(module, builtin=True) 345 return self.setup_class() 346 347 def _teardown_class(self): 348 """Proxy function to guarantee the base implementation of teardown_class 349 is called. 350 """ 351 super()._teardown_class() 352 event_bus.post(TestClassEndEvent(self, self.results)) 353 354 def _setup_test(self, test_name): 355 """Proxy function to guarantee the base implementation of setup_test is 356 called. 357 """ 358 self.current_test_name = test_name 359 360 # Skip the test if the consecutive test case failure limit is reached. 361 if self.consecutive_failures == self.consecutive_failure_limit: 362 raise signals.TestError('Consecutive test failure') 363 364 return self.setup_test() 365 366 def setup_test(self): 367 """Setup function that will be called every time before executing each 368 test case in the test class. 369 370 To signal setup failure, return False or raise an exception. If 371 exceptions were raised, the stack trace would appear in log, but the 372 exceptions would not propagate to upper levels. 373 374 Implementation is optional. 375 """ 376 return True 377 378 def _teardown_test(self, test_name): 379 """Proxy function to guarantee the base implementation of teardown_test 380 is called. 381 """ 382 self.log.debug('Tearing down test %s' % test_name) 383 self.teardown_test() 384 385 def _on_fail(self, record): 386 """Proxy function to guarantee the base implementation of on_fail is 387 called. 388 389 Args: 390 record: The records.TestResultRecord object for the failed test 391 case. 392 """ 393 self.consecutive_failures += 1 394 if record.details: 395 self.log.error(record.details) 396 self.log.info(RESULT_LINE_TEMPLATE, record.test_name, record.result) 397 self.on_fail(record.test_name, record.begin_time) 398 399 def on_fail(self, test_name, begin_time): 400 """A function that is executed upon a test case failure. 401 402 User implementation is optional. 403 404 Args: 405 test_name: Name of the test that triggered this function. 406 begin_time: Logline format timestamp taken when the test started. 407 """ 408 def _on_pass(self, record): 409 """Proxy function to guarantee the base implementation of on_pass is 410 called. 411 412 Args: 413 record: The records.TestResultRecord object for the passed test 414 case. 415 """ 416 self.consecutive_failures = 0 417 msg = record.details 418 if msg: 419 self.log.info(msg) 420 self.log.info(RESULT_LINE_TEMPLATE, record.test_name, record.result) 421 self.on_pass(record.test_name, record.begin_time) 422 423 def on_pass(self, test_name, begin_time): 424 """A function that is executed upon a test case passing. 425 426 Implementation is optional. 427 428 Args: 429 test_name: Name of the test that triggered this function. 430 begin_time: Logline format timestamp taken when the test started. 431 """ 432 def _on_skip(self, record): 433 """Proxy function to guarantee the base implementation of on_skip is 434 called. 435 436 Args: 437 record: The records.TestResultRecord object for the skipped test 438 case. 439 """ 440 self.log.info(RESULT_LINE_TEMPLATE, record.test_name, record.result) 441 self.log.info("Reason to skip: %s", record.details) 442 self.on_skip(record.test_name, record.begin_time) 443 444 def on_skip(self, test_name, begin_time): 445 """A function that is executed upon a test case being skipped. 446 447 Implementation is optional. 448 449 Args: 450 test_name: Name of the test that triggered this function. 451 begin_time: Logline format timestamp taken when the test started. 452 """ 453 def _on_exception(self, record): 454 """Proxy function to guarantee the base implementation of on_exception 455 is called. 456 457 Args: 458 record: The records.TestResultRecord object for the failed test 459 case. 460 """ 461 self.log.exception(record.details) 462 self.on_exception(record.test_name, record.begin_time) 463 464 def on_exception(self, test_name, begin_time): 465 """A function that is executed upon an unhandled exception from a test 466 case. 467 468 Implementation is optional. 469 470 Args: 471 test_name: Name of the test that triggered this function. 472 begin_time: Logline format timestamp taken when the test started. 473 """ 474 def on_retry(self): 475 """Function to run before retrying a test through get_func_with_retry. 476 477 This function runs when a test is automatically retried. The function 478 can be used to modify internal test parameters, for example, to retry 479 a test with slightly different input variables. 480 """ 481 def _exec_procedure_func(self, func, tr_record): 482 """Executes a procedure function like on_pass, on_fail etc. 483 484 This function will alternate the 'Result' of the test's record if 485 exceptions happened when executing the procedure function. 486 487 This will let signals.TestAbortAll through so abort_all works in all 488 procedure functions. 489 490 Args: 491 func: The procedure function to be executed. 492 tr_record: The TestResultRecord object associated with the test 493 case executed. 494 """ 495 try: 496 func(tr_record) 497 except signals.TestAbortAll: 498 raise 499 except Exception as e: 500 self.log.exception("Exception happened when executing %s for %s.", 501 func.__name__, self.current_test_name) 502 tr_record.add_error(func.__name__, e) 503 504 def exec_one_testcase(self, test_name, test_func): 505 """Executes one test case and update test results. 506 507 Executes one test case, create a records.TestResultRecord object with 508 the execution information, and add the record to the test class's test 509 results. 510 511 Args: 512 test_name: Name of the test. 513 test_func: The test function. 514 """ 515 class_name = self.__class__.__name__ 516 tr_record = records.TestResultRecord(test_name, class_name) 517 tr_record.test_begin() 518 self.begin_time = int(tr_record.begin_time) 519 self.log_begin_time = tr_record.log_begin_time 520 self.test_name = tr_record.test_name 521 event_bus.post(TestCaseBeginEvent(self, self.test_name)) 522 self.log.info("%s %s", TEST_CASE_TOKEN, test_name) 523 524 # Enable test retry if specified in the ACTS config 525 retry_tests = self.user_params.get('retry_tests', []) 526 full_test_name = '%s.%s' % (class_name, self.test_name) 527 if any(name in retry_tests for name in [class_name, full_test_name]): 528 test_func = self.get_func_with_retry(test_func) 529 530 verdict = None 531 test_signal = None 532 try: 533 try: 534 ret = self._setup_test(self.test_name) 535 asserts.assert_true(ret is not False, 536 "Setup for %s failed." % test_name) 537 verdict = test_func() 538 finally: 539 try: 540 self._teardown_test(self.test_name) 541 except signals.TestAbortAll: 542 raise 543 except Exception as e: 544 self.log.error(traceback.format_exc()) 545 tr_record.add_error("teardown_test", e) 546 self._exec_procedure_func(self._on_exception, tr_record) 547 except (signals.TestFailure, AssertionError) as e: 548 test_signal = e 549 if self.user_params.get( 550 keys.Config.key_test_failure_tracebacks.value, False): 551 self.log.exception(e) 552 tr_record.test_fail(e) 553 self._exec_procedure_func(self._on_fail, tr_record) 554 except signals.TestSkip as e: 555 # Test skipped. 556 test_signal = e 557 tr_record.test_skip(e) 558 self._exec_procedure_func(self._on_skip, tr_record) 559 except (signals.TestAbortClass, signals.TestAbortAll) as e: 560 # Abort signals, pass along. 561 test_signal = e 562 tr_record.test_fail(e) 563 self._exec_procedure_func(self._on_fail, tr_record) 564 raise e 565 except signals.TestPass as e: 566 # Explicit test pass. 567 test_signal = e 568 tr_record.test_pass(e) 569 self._exec_procedure_func(self._on_pass, tr_record) 570 except Exception as e: 571 test_signal = e 572 self.log.error(traceback.format_exc()) 573 # Exception happened during test. 574 tr_record.test_error(e) 575 self._exec_procedure_func(self._on_exception, tr_record) 576 self._exec_procedure_func(self._on_fail, tr_record) 577 else: 578 if verdict or (verdict is None): 579 # Test passed. 580 tr_record.test_pass() 581 self._exec_procedure_func(self._on_pass, tr_record) 582 return 583 tr_record.test_fail() 584 self._exec_procedure_func(self._on_fail, tr_record) 585 finally: 586 self.results.add_record(tr_record) 587 self.summary_writer.dump(tr_record.to_dict(), 588 records.TestSummaryEntryType.RECORD) 589 self.current_test_name = None 590 event_bus.post(TestCaseEndEvent(self, self.test_name, test_signal)) 591 592 def get_func_with_retry(self, func, attempts=2): 593 """Returns a wrapped test method that re-runs after failure. Return test 594 result upon success. If attempt limit reached, collect all failure 595 messages and raise a TestFailure signal. 596 597 Params: 598 func: The test method 599 attempts: Number of attempts to run test 600 601 Returns: result of the test method 602 """ 603 exceptions = self.retryable_exceptions 604 605 def wrapper(*args, **kwargs): 606 error_msgs = [] 607 extras = {} 608 retry = False 609 for i in range(attempts): 610 try: 611 if retry: 612 self.teardown_test() 613 self.setup_test() 614 self.on_retry() 615 return func(*args, **kwargs) 616 except exceptions as e: 617 retry = True 618 msg = 'Failure on attempt %d: %s' % (i + 1, e.details) 619 self.log.warning(msg) 620 error_msgs.append(msg) 621 if e.extras: 622 extras['Attempt %d' % (i + 1)] = e.extras 623 raise signals.TestFailure('\n'.join(error_msgs), extras) 624 625 return wrapper 626 627 def run_generated_testcases(self, 628 test_func, 629 settings, 630 args=None, 631 kwargs=None, 632 tag="", 633 name_func=None, 634 format_args=False): 635 """Deprecated. Please use setup_generated_tests and generate_tests. 636 637 Generated test cases are not written down as functions, but as a list 638 of parameter sets. This way we reduce code repetition and improve 639 test case scalability. 640 641 Args: 642 test_func: The common logic shared by all these generated test 643 cases. This function should take at least one argument, 644 which is a parameter set. 645 settings: A list of strings representing parameter sets. These are 646 usually json strings that get loaded in the test_func. 647 args: Iterable of additional position args to be passed to 648 test_func. 649 kwargs: Dict of additional keyword args to be passed to test_func 650 tag: Name of this group of generated test cases. Ignored if 651 name_func is provided and operates properly. 652 name_func: A function that takes a test setting and generates a 653 proper test name. The test name should be shorter than 654 utils.MAX_FILENAME_LEN. Names over the limit will be 655 truncated. 656 format_args: If True, args will be appended as the first argument 657 in the args list passed to test_func. 658 659 Returns: 660 A list of settings that did not pass. 661 """ 662 args = args or () 663 kwargs = kwargs or {} 664 failed_settings = [] 665 666 for setting in settings: 667 test_name = "{} {}".format(tag, setting) 668 669 if name_func: 670 try: 671 test_name = name_func(setting, *args, **kwargs) 672 except: 673 self.log.exception(("Failed to get test name from " 674 "test_func. Fall back to default %s"), 675 test_name) 676 677 self.results.requested.append(test_name) 678 679 if len(test_name) > utils.MAX_FILENAME_LEN: 680 test_name = test_name[:utils.MAX_FILENAME_LEN] 681 682 previous_success_cnt = len(self.results.passed) 683 684 if format_args: 685 self.exec_one_testcase( 686 test_name, 687 functools.partial(test_func, *(args + (setting, )), 688 **kwargs)) 689 else: 690 self.exec_one_testcase( 691 test_name, 692 functools.partial(test_func, *((setting, ) + args), 693 **kwargs)) 694 695 if len(self.results.passed) - previous_success_cnt != 1: 696 failed_settings.append(setting) 697 698 return failed_settings 699 700 def _exec_func(self, func, *args): 701 """Executes a function with exception safeguard. 702 703 This will let signals.TestAbortAll through so abort_all works in all 704 procedure functions. 705 706 Args: 707 func: Function to be executed. 708 args: Arguments to be passed to the function. 709 710 Returns: 711 Whatever the function returns, or False if unhandled exception 712 occured. 713 """ 714 try: 715 return func(*args) 716 except signals.TestAbortAll: 717 raise 718 except: 719 self.log.exception("Exception happened when executing %s in %s.", 720 func.__name__, self.TAG) 721 return False 722 723 def _block_all_test_cases(self, tests, reason='Failed class setup'): 724 """ 725 Block all passed in test cases. 726 Args: 727 tests: The tests to block. 728 reason: Message describing the reason that the tests are blocked. 729 Default is 'Failed class setup' 730 """ 731 for test_name, test_func in tests: 732 signal = signals.TestError(reason) 733 record = records.TestResultRecord(test_name, self.TAG) 734 record.test_begin() 735 if hasattr(test_func, 'gather'): 736 signal.extras = test_func.gather() 737 record.test_error(signal) 738 self.results.add_record(record) 739 self.summary_writer.dump(record.to_dict(), 740 records.TestSummaryEntryType.RECORD) 741 self._on_skip(record) 742 743 def run(self, test_names=None): 744 """Runs test cases within a test class by the order they appear in the 745 execution list. 746 747 One of these test cases lists will be executed, shown here in priority 748 order: 749 1. The test_names list, which is passed from cmd line. 750 2. The self.tests list defined in test class. Invalid names are 751 ignored. 752 3. All function that matches test case naming convention in the test 753 class. 754 755 Args: 756 test_names: A list of string that are test case names/patterns 757 requested in cmd line. 758 759 Returns: 760 The test results object of this class. 761 """ 762 # Executes pre-setup procedures, like generating test methods. 763 if not self._setup_generated_tests(): 764 return self.results 765 766 self.register_test_class_event_subscriptions() 767 self.log.info("==========> %s <==========", self.TAG) 768 # Devise the actual test cases to run in the test class. 769 if self.tests: 770 # Specified by run list in class. 771 valid_tests = list(self.tests) 772 else: 773 # No test case specified by user, gather the run list automatically. 774 valid_tests = self.get_existing_test_names() 775 if test_names: 776 # Match test cases with any of the user-specified patterns 777 matches = [] 778 for test_name in test_names: 779 for valid_test in valid_tests: 780 if (fnmatch.fnmatch(valid_test, test_name) 781 and valid_test not in matches): 782 matches.append(valid_test) 783 else: 784 matches = valid_tests 785 self.results.requested = matches 786 self.summary_writer.dump(self.results.requested_test_names_dict(), 787 records.TestSummaryEntryType.TEST_NAME_LIST) 788 tests = self._get_test_methods(matches) 789 790 # Setup for the class. 791 setup_fail = False 792 try: 793 if self._setup_class() is False: 794 self.log.error("Failed to setup %s.", self.TAG) 795 self._block_all_test_cases(tests) 796 setup_fail = True 797 except signals.TestAbortClass: 798 self.log.exception('Test class %s aborted' % self.TAG) 799 setup_fail = True 800 except Exception as e: 801 self.log.exception("Failed to setup %s.", self.TAG) 802 self._block_all_test_cases(tests) 803 setup_fail = True 804 if setup_fail: 805 self._exec_func(self._teardown_class) 806 self.log.info("Summary for test class %s: %s", self.TAG, 807 self.results.summary_str()) 808 return self.results 809 810 # Run tests in order. 811 test_case_iterations = self.user_params.get( 812 keys.Config.key_test_case_iterations.value, 1) 813 if any([substr in self.__class__.__name__ for substr in 814 ['Preflight', 'Postflight']]): 815 test_case_iterations = 1 816 try: 817 for test_name, test_func in tests: 818 for _ in range(test_case_iterations): 819 self.exec_one_testcase(test_name, test_func) 820 return self.results 821 except signals.TestAbortClass: 822 self.log.exception('Test class %s aborted' % self.TAG) 823 return self.results 824 except signals.TestAbortAll as e: 825 # Piggy-back test results on this exception object so we don't lose 826 # results from this test class. 827 setattr(e, "results", self.results) 828 raise e 829 finally: 830 self._exec_func(self._teardown_class) 831 self.log.info("Summary for test class %s: %s", self.TAG, 832 self.results.summary_str()) 833 834 def _ad_take_bugreport(self, ad, test_name, begin_time): 835 for i in range(3): 836 try: 837 ad.take_bug_report(test_name, begin_time) 838 return True 839 except Exception as e: 840 ad.log.error("bugreport attempt %s error: %s", i + 1, e) 841 842 def _ad_take_extra_logs(self, ad, test_name, begin_time): 843 result = True 844 if getattr(ad, "qxdm_log", False): 845 # Gather qxdm log modified 3 minutes earlier than test start time 846 if begin_time: 847 qxdm_begin_time = begin_time - 1000 * 60 * 3 848 else: 849 qxdm_begin_time = None 850 try: 851 ad.get_qxdm_logs(test_name, qxdm_begin_time) 852 except Exception as e: 853 ad.log.error("Failed to get QXDM log for %s with error %s", 854 test_name, e) 855 result = False 856 857 try: 858 ad.check_crash_report(test_name, begin_time, log_crash_report=True) 859 except Exception as e: 860 ad.log.error("Failed to check crash report for %s with error %s", 861 test_name, e) 862 result = False 863 return result 864 865 def _skip_bug_report(self, test_name): 866 """A function to check whether we should skip creating a bug report. 867 868 Args: 869 test_name: The test case name 870 871 Returns: True if bug report is to be skipped. 872 """ 873 if "no_bug_report_on_fail" in self.user_params: 874 return True 875 876 # If the current test class or test case is found in the set of 877 # problematic tests, we skip bugreport and other failure artifact 878 # creation. 879 class_name = self.__class__.__name__ 880 quiet_tests = self.user_params.get('quiet_tests', []) 881 if class_name in quiet_tests: 882 self.log.info( 883 "Skipping bug report, as directed for this test class.") 884 return True 885 full_test_name = '%s.%s' % (class_name, test_name) 886 if full_test_name in quiet_tests: 887 self.log.info( 888 "Skipping bug report, as directed for this test case.") 889 return True 890 891 # Once we hit a certain log path size, it's not going to get smaller. 892 # We cache the result so we don't have to keep doing directory walks. 893 if self.size_limit_reached: 894 return True 895 try: 896 max_log_size = int( 897 self.user_params.get("soft_output_size_limit") or "invalid") 898 log_path = getattr(logging, "log_path", None) 899 if log_path: 900 curr_log_size = utils.get_directory_size(log_path) 901 if curr_log_size > max_log_size: 902 self.log.info( 903 "Skipping bug report, as we've reached the size limit." 904 ) 905 self.size_limit_reached = True 906 return True 907 except ValueError: 908 pass 909 return False 910 911 def _take_bug_report(self, test_name, begin_time): 912 if self._skip_bug_report(test_name): 913 return 914 915 executor = ThreadPoolExecutor(max_workers=10) 916 for ad in getattr(self, 'android_devices', []): 917 executor.submit(self._ad_take_bugreport, ad, test_name, begin_time) 918 executor.submit(self._ad_take_extra_logs, ad, test_name, 919 begin_time) 920 executor.shutdown() 921 922 def _reboot_device(self, ad): 923 ad.log.info("Rebooting device.") 924 ad = ad.reboot() 925 926 def _cleanup_logger_sessions(self): 927 for (mylogger, session) in self.logger_sessions: 928 self.log.info("Resetting a diagnostic session %s, %s", mylogger, 929 session) 930 mylogger.reset() 931 self.logger_sessions = [] 932 933 def _pull_diag_logs(self, test_name, begin_time): 934 for (mylogger, session) in self.logger_sessions: 935 self.log.info("Pulling diagnostic session %s", mylogger) 936 mylogger.stop(session) 937 diag_path = os.path.join( 938 self.log_path, logger.epoch_to_log_line_timestamp(begin_time)) 939 os.makedirs(diag_path, exist_ok=True) 940 mylogger.pull(session, diag_path) 941 942 def register_test_class_event_subscriptions(self): 943 self.class_subscriptions = subscription_bundle.create_from_instance( 944 self) 945 self.class_subscriptions.register() 946 947 def unregister_test_class_event_subscriptions(self): 948 for package in self.all_subscriptions: 949 package.unregister() 950