1#!/usr/bin/env python 2# 3# Copyright 2018, The Android Open Source Project 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16 17""" 18Atest Argument Parser class for atest. 19""" 20 21# pylint: disable=line-too-long 22 23import argparse 24import pydoc 25 26import constants 27 28# Constants used for AtestArgParser and EPILOG_TEMPLATE 29HELP_DESC = ('A command line tool that allows users to build, install, and run ' 30 'Android tests locally, greatly speeding test re-runs without ' 31 'requiring knowledge of Trade Federation test harness command line' 32 ' options.') 33 34# Constants used for arg help message(sorted in alphabetic) 35ACLOUD_CREATE = 'Create AVD(s) via acloud command.' 36ALL_ABI = 'Set to run tests for all abis.' 37BUILD = 'Run a build.' 38CLEAR_CACHE = 'Wipe out the test_infos cache of the test.' 39COLLECT_TESTS_ONLY = ('Collect a list test cases of the instrumentation tests ' 40 'without testing them in real.') 41DISABLE_TEARDOWN = 'Disable test teardown and cleanup.' 42DRY_RUN = 'Dry run atest without building, installing and running tests in real.' 43ENABLE_FILE_PATTERNS = 'Enable FILE_PATTERNS in TEST_MAPPING.' 44HISTORY = ('Show test results in chronological order(with specified number or ' 45 'all by default).') 46HOST = ('Run the test completely on the host without a device. ' 47 '(Note: running a host test that requires a device without ' 48 '--host will fail.)') 49INCLUDE_SUBDIRS = 'Search TEST_MAPPING files in subdirs as well.' 50INFO = 'Show module information.' 51INSTALL = 'Install an APK.' 52INSTANT = ('Run the instant_app version of the module if the module supports it. ' 53 'Note: Nothing\'s going to run if it\'s not an Instant App test and ' 54 '"--instant" is passed.') 55ITERATION = 'Loop-run tests until the max iteration is reached. (10 by default)' 56LATEST_RESULT = 'Print latest test result.' 57LIST_MODULES = 'List testable modules for the given suite.' 58NO_METRICS = 'Do not send metrics.' 59REBUILD_MODULE_INFO = ('Forces a rebuild of the module-info.json file. ' 60 'This may be necessary following a repo sync or ' 61 'when writing a new test.') 62RERUN_UNTIL_FAILURE = ('Rerun all tests until a failure occurs or the max ' 63 'iteration is reached. (10 by default)') 64RETRY_ANY_FAILURE = ('Rerun failed tests until passed or the max iteration ' 65 'is reached. (10 by default)') 66SERIAL = 'The device to run the test on.' 67START_AVD = 'Automatically create an AVD and run tests on the virtual device.' 68TEST = ('Run the tests. WARNING: Many test configs force cleanup of device ' 69 'after test run. In this case, "-d" must be used in previous test run to ' 70 'disable cleanup for "-t" to work. Otherwise, device will need to be ' 71 'setup again with "-i".') 72TEST_MAPPING = 'Run tests defined in TEST_MAPPING files.' 73TF_TEMPLATE = ('Add extra tradefed template for ATest suite, ' 74 'e.g. atest <test> --tf-template <template_key>=<template_path>') 75TF_DEBUG = 'Enable tradefed debug mode with a specify port. Default value is 10888.' 76SHARDING = 'Option to specify sharding count. The default value is 2' 77UPDATE_CMD_MAPPING = ('Update the test command of input tests. Warning: result ' 78 'will be saved under tools/tradefederation/core/atest/test_data.') 79USER_TYPE = 'Run test with specific user type, e.g. atest <test> --user-type secondary_user' 80VERBOSE = 'Display DEBUG level logging.' 81VERIFY_CMD_MAPPING = 'Verify the test command of input tests.' 82VERSION = 'Display version string.' 83WAIT_FOR_DEBUGGER = 'Wait for debugger prior to execution (Instrumentation tests only).' 84FLAKES_INFO = 'Test result with flakes info.' 85 86def _positive_int(value): 87 """Verify value by whether or not a positive integer. 88 89 Args: 90 value: A string of a command-line argument. 91 92 Returns: 93 int of value, if it is an positive integer. 94 Otherwise, raise argparse.ArgumentTypeError. 95 """ 96 err_msg = "invalid positive int value: '%s'" % value 97 try: 98 converted_value = int(value) 99 if converted_value < 1: 100 raise argparse.ArgumentTypeError(err_msg) 101 return converted_value 102 except ValueError: 103 raise argparse.ArgumentTypeError(err_msg) 104 105 106class AtestArgParser(argparse.ArgumentParser): 107 """Atest wrapper of ArgumentParser.""" 108 109 def __init__(self): 110 """Initialise an ArgumentParser instance.""" 111 super(AtestArgParser, self).__init__( 112 description=HELP_DESC, add_help=False) 113 114 def add_atest_args(self): 115 """A function that does ArgumentParser.add_argument()""" 116 self.add_argument('tests', nargs='*', help='Tests to build and/or run.') 117 # Options that to do with testing. 118 self.add_argument('-a', '--all-abi', action='store_true', help=ALL_ABI) 119 self.add_argument('-b', '--build', action='append_const', dest='steps', 120 const=constants.BUILD_STEP, help=BUILD) 121 self.add_argument('-d', '--disable-teardown', action='store_true', 122 help=DISABLE_TEARDOWN) 123 self.add_argument('--host', action='store_true', help=HOST) 124 self.add_argument('-i', '--install', action='append_const', 125 dest='steps', const=constants.INSTALL_STEP, 126 help=INSTALL) 127 self.add_argument('-m', constants.REBUILD_MODULE_INFO_FLAG, 128 action='store_true', help=REBUILD_MODULE_INFO) 129 self.add_argument('--sharding', nargs='?', const=2, 130 type=_positive_int, default=0, 131 help=SHARDING) 132 self.add_argument('-t', '--test', action='append_const', dest='steps', 133 const=constants.TEST_STEP, help=TEST) 134 self.add_argument('-w', '--wait-for-debugger', action='store_true', 135 help=WAIT_FOR_DEBUGGER) 136 137 # Options related to Test Mapping 138 self.add_argument('-p', '--test-mapping', action='store_true', 139 help=TEST_MAPPING) 140 self.add_argument('--include-subdirs', action='store_true', 141 help=INCLUDE_SUBDIRS) 142 # TODO(146980564): Remove enable-file-patterns when support 143 # file-patterns in TEST_MAPPING by default. 144 self.add_argument('--enable-file-patterns', action='store_true', 145 help=ENABLE_FILE_PATTERNS) 146 147 # Options for information queries and dry-runs: 148 # A group of options for dry-runs. They are mutually exclusive 149 # in a command line. 150 group = self.add_mutually_exclusive_group() 151 group.add_argument('--collect-tests-only', action='store_true', 152 help=COLLECT_TESTS_ONLY) 153 group.add_argument('--dry-run', action='store_true', help=DRY_RUN) 154 self.add_argument('-h', '--help', action='store_true', 155 help='Print this help message.') 156 self.add_argument('--info', action='store_true', help=INFO) 157 self.add_argument('-L', '--list-modules', help=LIST_MODULES) 158 self.add_argument('-v', '--verbose', action='store_true', help=VERBOSE) 159 self.add_argument('-V', '--version', action='store_true', help=VERSION) 160 161 # Options that to do with acloud/AVDs. 162 agroup = self.add_mutually_exclusive_group() 163 agroup.add_argument('--acloud-create', nargs=argparse.REMAINDER, type=str, 164 help=ACLOUD_CREATE) 165 agroup.add_argument('--start-avd', action='store_true', 166 help=START_AVD) 167 agroup.add_argument('-s', '--serial', help=SERIAL) 168 169 # Options that to query flakes info in test result 170 self.add_argument('--flakes-info', action='store_true', 171 help=FLAKES_INFO) 172 173 # Obsolete options that will be removed soon. 174 self.add_argument('--generate-baseline', nargs='?', 175 type=int, const=5, default=0, 176 help='Generate baseline metrics, run 5 iterations by' 177 'default. Provide an int argument to specify ' 178 '# iterations.') 179 self.add_argument('--generate-new-metrics', nargs='?', 180 type=int, const=5, default=0, 181 help='Generate new metrics, run 5 iterations by ' 182 'default. Provide an int argument to specify ' 183 '# iterations.') 184 self.add_argument('--detect-regression', nargs='*', 185 help='Run regression detection algorithm. Supply ' 186 'path to baseline and/or new metrics folders.') 187 188 # Options related to module parameterization 189 self.add_argument('--instant', action='store_true', help=INSTANT) 190 self.add_argument('--user-type', help=USER_TYPE) 191 192 # Option for dry-run command mapping result and cleaning cache. 193 self.add_argument('-c', '--clear-cache', action='store_true', 194 help=CLEAR_CACHE) 195 self.add_argument('-u', '--update-cmd-mapping', action='store_true', 196 help=UPDATE_CMD_MAPPING) 197 self.add_argument('-y', '--verify-cmd-mapping', action='store_true', 198 help=VERIFY_CMD_MAPPING) 199 # Options for Tradefed debug mode. 200 self.add_argument('-D', '--tf-debug', nargs='?', const=10888, 201 type=_positive_int, default=0, 202 help=TF_DEBUG) 203 # Options for Tradefed customization related. 204 self.add_argument('--tf-template', action='append', 205 help=TF_TEMPLATE) 206 207 # A group of options for rerun strategy. They are mutually exclusive 208 # in a command line. 209 group = self.add_mutually_exclusive_group() 210 # Option for rerun tests for the specified number iterations. 211 group.add_argument('--iterations', nargs='?', 212 type=_positive_int, const=10, default=0, 213 metavar='MAX_ITERATIONS', help=ITERATION) 214 group.add_argument('--rerun-until-failure', nargs='?', 215 type=_positive_int, const=10, default=0, 216 metavar='MAX_ITERATIONS', help=RERUN_UNTIL_FAILURE) 217 group.add_argument('--retry-any-failure', nargs='?', 218 type=_positive_int, const=10, default=0, 219 metavar='MAX_ITERATIONS', help=RETRY_ANY_FAILURE) 220 221 # A group of options for history. They are mutually exclusive 222 # in a command line. 223 history_group = self.add_mutually_exclusive_group() 224 # History related options. 225 history_group.add_argument('--latest-result', action='store_true', 226 help=LATEST_RESULT) 227 history_group.add_argument('--history', nargs='?', const='99999', 228 help=HISTORY) 229 230 # Options for disabling collecting data for metrics. 231 self.add_argument(constants.NO_METRICS_ARG, action='store_true', 232 help=NO_METRICS) 233 234 # This arg actually doesn't consume anything, it's primarily used for 235 # the help description and creating custom_args in the NameSpace object. 236 self.add_argument('--', dest='custom_args', nargs='*', 237 help='Specify custom args for the test runners. ' 238 'Everything after -- will be consumed as ' 239 'custom args.') 240 241 def get_args(self): 242 """This method is to get args from actions and return optional args. 243 244 Returns: 245 A list of optional arguments. 246 """ 247 argument_list = [] 248 # The output of _get_optional_actions(): [['-t', '--test'], [--info]] 249 # return an argument list: ['-t', '--test', '--info'] 250 for arg in self._get_optional_actions(): 251 argument_list.extend(arg.option_strings) 252 return argument_list 253 254 255def print_epilog_text(): 256 """Pagination print EPILOG_TEXT. 257 258 Returns: 259 STDOUT from pydoc.pager(). 260 """ 261 epilog_text = EPILOG_TEMPLATE.format(ACLOUD_CREATE=ACLOUD_CREATE, 262 ALL_ABI=ALL_ABI, 263 BUILD=BUILD, 264 CLEAR_CACHE=CLEAR_CACHE, 265 COLLECT_TESTS_ONLY=COLLECT_TESTS_ONLY, 266 DISABLE_TEARDOWN=DISABLE_TEARDOWN, 267 DRY_RUN=DRY_RUN, 268 ENABLE_FILE_PATTERNS=ENABLE_FILE_PATTERNS, 269 HELP_DESC=HELP_DESC, 270 HISTORY=HISTORY, 271 HOST=HOST, 272 INCLUDE_SUBDIRS=INCLUDE_SUBDIRS, 273 INFO=INFO, 274 INSTALL=INSTALL, 275 INSTANT=INSTANT, 276 ITERATION=ITERATION, 277 LATEST_RESULT=LATEST_RESULT, 278 LIST_MODULES=LIST_MODULES, 279 NO_METRICS=NO_METRICS, 280 REBUILD_MODULE_INFO=REBUILD_MODULE_INFO, 281 RERUN_UNTIL_FAILURE=RERUN_UNTIL_FAILURE, 282 RETRY_ANY_FAILURE=RETRY_ANY_FAILURE, 283 SERIAL=SERIAL, 284 SHARDING=SHARDING, 285 START_AVD=START_AVD, 286 TEST=TEST, 287 TEST_MAPPING=TEST_MAPPING, 288 TF_DEBUG=TF_DEBUG, 289 TF_TEMPLATE=TF_TEMPLATE, 290 USER_TYPE=USER_TYPE, 291 UPDATE_CMD_MAPPING=UPDATE_CMD_MAPPING, 292 VERBOSE=VERBOSE, 293 VERSION=VERSION, 294 VERIFY_CMD_MAPPING=VERIFY_CMD_MAPPING, 295 WAIT_FOR_DEBUGGER=WAIT_FOR_DEBUGGER, 296 FLAKES_INFO=FLAKES_INFO) 297 return pydoc.pager(epilog_text) 298 299 300EPILOG_TEMPLATE = r'''ATEST(1) ASuite/ATest 301 302NAME 303 atest - {HELP_DESC} 304 305 306SYNOPSIS 307 atest [OPTION]... [TEST_TARGET]... -- [CUSTOM_ARGS]... 308 309 310OPTIONS 311 Below arguments are catagorised by features and purposes. Arguments marked with default will apply even the user does not pass it explicitly. 312 313 [ Testing ] 314 -a, --all-abi 315 {ALL_ABI} 316 317 If only need to run tests for a specific abi, please use: 318 atest <test> -- --abi arm64-v8a # ARM 64-bit 319 atest <test> -- --abi armeabi-v7a # ARM 32-bit 320 321 -b, --build: 322 {BUILD} (default) 323 324 -d, --disable-teardown 325 {DISABLE_TEARDOWN} 326 327 -D --tf-debug 328 {TF_DEBUG} 329 330 --history 331 {HISTORY} 332 333 --host 334 {HOST} 335 336 -i, --install 337 {INSTALL} (default) 338 339 -m, --rebuild-module-info 340 {REBUILD_MODULE_INFO} (default) 341 342 -s, --serial 343 {SERIAL} 344 345 --sharding 346 {SHARDING} 347 348 -t, --test 349 {TEST} (default) 350 351 --tf-template 352 {TF_TEMPLATE} 353 354 -w, --wait-for-debugger 355 {WAIT_FOR_DEBUGGER} 356 357 358 [ Test Mapping ] 359 -p, --test-mapping 360 {TEST_MAPPING} 361 362 --include-subdirs 363 {INCLUDE_SUBDIRS} 364 365 --enable-file-patterns 366 {ENABLE_FILE_PATTERNS} 367 368 369 [ Information/Queries ] 370 --collect-tests-only 371 {COLLECT_TESTS_ONLY} 372 373 --info 374 {INFO} 375 376 -L, --list-modules 377 {LIST_MODULES} 378 379 --latest-result 380 {LATEST_RESULT} 381 382 -v, --verbose 383 {VERBOSE} 384 385 -V, --version 386 {VERSION} 387 388 389 [ Dry-Run and Caching ] 390 --dry-run 391 {DRY_RUN} 392 393 -c, --clear-cache 394 {CLEAR_CACHE} 395 396 -u, --update-cmd-mapping 397 {UPDATE_CMD_MAPPING} 398 399 -y, --verify-cmd-mapping 400 {VERIFY_CMD_MAPPING} 401 402 403 [ Module Parameterization ] 404 --instant 405 {INSTANT} 406 407 --user-type 408 {USER_TYPE} 409 410 411 [ Iteration Testing ] 412 --iterations 413 {ITERATION} 414 415 --rerun-until-failure 416 {RERUN_UNTIL_FAILURE} 417 418 --retry-any-failure 419 {RETRY_ANY_FAILURE} 420 421 422 [ Testing With AVDs ] 423 --start-avd 424 {START_AVD} 425 426 --acloud-create 427 {ACLOUD_CREATE} 428 429 430 [ Testing With Flakes Info ] 431 --flakes-info 432 {FLAKES_INFO} 433 434 435 [ Metrics ] 436 --no-metrics 437 {NO_METRICS} 438 439 440EXAMPLES 441 - - - - - - - - - 442 IDENTIFYING TESTS 443 - - - - - - - - - 444 445 The positional argument <tests> should be a reference to one or more of the tests you'd like to run. Multiple tests can be run in one command by separating test references with spaces. 446 447 Usage template: atest <reference_to_test_1> <reference_to_test_2> 448 449 A <reference_to_test> can be satisfied by the test's MODULE NAME, MODULE:CLASS, CLASS NAME, TF INTEGRATION TEST, FILE PATH or PACKAGE NAME. Explanations and examples of each follow. 450 451 452 < MODULE NAME > 453 454 Identifying a test by its module name will run the entire module. Input the name as it appears in the LOCAL_MODULE or LOCAL_PACKAGE_NAME variables in that test's Android.mk or Android.bp file. 455 456 Note: Use < TF INTEGRATION TEST > to run non-module tests integrated directly into TradeFed. 457 458 Examples: 459 atest FrameworksServicesTests 460 atest CtsJankDeviceTestCases 461 462 463 < MODULE:CLASS > 464 465 Identifying a test by its class name will run just the tests in that class and not the whole module. MODULE:CLASS is the preferred way to run a single class. MODULE is the same as described above. CLASS is the name of the test class in the .java file. It can either be the fully qualified class name or just the basic name. 466 467 Examples: 468 atest FrameworksServicesTests:ScreenDecorWindowTests 469 atest FrameworksServicesTests:com.android.server.wm.ScreenDecorWindowTests 470 atest CtsJankDeviceTestCases:CtsDeviceJankUi 471 472 473 < CLASS NAME > 474 475 A single class can also be run by referencing the class name without the module name. 476 477 Examples: 478 atest ScreenDecorWindowTests 479 atest CtsDeviceJankUi 480 481 However, this will take more time than the equivalent MODULE:CLASS reference, so we suggest using a MODULE:CLASS reference whenever possible. Examples below are ordered by performance from the fastest to the slowest: 482 483 Examples: 484 atest FrameworksServicesTests:com.android.server.wm.ScreenDecorWindowTests 485 atest FrameworksServicesTests:ScreenDecorWindowTests 486 atest ScreenDecorWindowTests 487 488 < TF INTEGRATION TEST > 489 490 To run tests that are integrated directly into TradeFed (non-modules), input the name as it appears in the output of the "tradefed.sh list configs" cmd. 491 492 Examples: 493 atest example/reboot 494 atest native-benchmark 495 496 497 < FILE PATH > 498 499 Both module-based tests and integration-based tests can be run by inputting the path to their test file or dir as appropriate. A single class can also be run by inputting the path to the class's java file. 500 501 Both relative and absolute paths are supported. 502 503 Example - 2 ways to run the `CtsJankDeviceTestCases` module via path: 504 1. run module from android <repo root>: 505 atest cts/tests/jank/jank 506 507 2. from <android root>/cts/tests/jank: 508 atest . 509 510 Example - run a specific class within CtsJankDeviceTestCases module from <android repo> root via path: 511 atest cts/tests/jank/src/android/jank/cts/ui/CtsDeviceJankUi.java 512 513 Example - run an integration test from <android repo> root via path: 514 atest tools/tradefederation/contrib/res/config/example/reboot.xml 515 516 517 < PACKAGE NAME > 518 519 Atest supports searching tests from package name as well. 520 521 Examples: 522 atest com.android.server.wm 523 atest android.jank.cts 524 525 526 - - - - - - - - - - - - - - - - - - - - - - - - - - 527 SPECIFYING INDIVIDUAL STEPS: BUILD, INSTALL OR RUN 528 - - - - - - - - - - - - - - - - - - - - - - - - - - 529 530 The -b, -i and -t options allow you to specify which steps you want to run. If none of those options are given, then all steps are run. If any of these options are provided then only the listed steps are run. 531 532 Note: -i alone is not currently support and can only be included with -t. 533 Both -b and -t can be run alone. 534 535 Examples: 536 atest -b <test> (just build targets) 537 atest -t <test> (run tests only) 538 atest -it <test> (install apk and run tests) 539 atest -bt <test> (build targets, run tests, but skip installing apk) 540 541 542 Atest now has the ability to force a test to skip its cleanup/teardown step. Many tests, e.g. CTS, cleanup the device after the test is run, so trying to rerun your test with -t will fail without having the --disable-teardown parameter. Use -d before -t to skip the test clean up step and test iteratively. 543 544 atest -d <test> (disable installing apk and cleanning up device) 545 atest -t <test> 546 547 Note that -t disables both setup/install and teardown/cleanup of the device. So you can continue to rerun your test with just 548 549 atest -t <test> 550 551 as many times as you want. 552 553 554 - - - - - - - - - - - - - 555 RUNNING SPECIFIC METHODS 556 - - - - - - - - - - - - - 557 558 It is possible to run only specific methods within a test class. To run only specific methods, identify the class in any of the ways supported for identifying a class (MODULE:CLASS, FILE PATH, etc) and then append the name of the method or method using the following template: 559 560 <reference_to_class>#<method1> 561 562 Multiple methods can be specified with commas: 563 564 <reference_to_class>#<method1>,<method2>,<method3>... 565 566 Examples: 567 atest com.android.server.wm.ScreenDecorWindowTests#testMultipleDecors 568 569 atest FrameworksServicesTests:ScreenDecorWindowTests#testFlagChange,testRemoval 570 571 572 - - - - - - - - - - - - - 573 RUNNING MULTIPLE CLASSES 574 - - - - - - - - - - - - - 575 576 To run multiple classes, deliminate them with spaces just like you would when running multiple tests. Atest will handle building and running classes in the most efficient way possible, so specifying a subset of classes in a module will improve performance over running the whole module. 577 578 579 Examples: 580 - two classes in same module: 581 atest FrameworksServicesTests:ScreenDecorWindowTests FrameworksServicesTests:DimmerTests 582 583 - two classes, different modules: 584 atest FrameworksServicesTests:ScreenDecorWindowTests CtsJankDeviceTestCases:CtsDeviceJankUi 585 586 587 - - - - - - - - - - - 588 RUNNING NATIVE TESTS 589 - - - - - - - - - - - 590 591 Atest can run native test. 592 593 Example: 594 - Input tests: 595 atest -a libinput_tests inputflinger_tests 596 597 Use -a|--all-abi to run the tests for all available device architectures, which in this example is armeabi-v7a (ARM 32-bit) and arm64-v8a (ARM 64-bit). 598 599 To select a specific native test to run, use colon (:) to specify the test name and hashtag (#) to further specify an individual method. For example, for the following test definition: 600 601 TEST_F(InputDispatcherTest, InjectInputEvent_ValidatesKeyEvents) 602 603 You can run the entire test using: 604 605 atest inputflinger_tests:InputDispatcherTest 606 607 or an individual test method using: 608 609 atest inputflinger_tests:InputDispatcherTest#InjectInputEvent_ValidatesKeyEvents 610 611 612 - - - - - - - - - - - - - - 613 RUNNING TESTS IN ITERATION 614 - - - - - - - - - - - - - - 615 616 To run tests in iterations, simply pass --iterations argument. No matter pass or fail, atest won't stop testing until the max iteration is reached. 617 618 Example: 619 atest <test> --iterations # 10 iterations(by default). 620 atest <test> --iterations 5 # run <test> 5 times. 621 622 Two approaches that assist users to detect flaky tests: 623 624 1) Run all tests until a failure occurs or the max iteration is reached. 625 626 Example: 627 - 10 iterations(by default). 628 atest <test> --rerun-until-failure 629 - stop when failed or reached the 20th run. 630 atest <test> --rerun-until-failure 20 631 632 2) Run failed tests until passed or the max iteration is reached. 633 634 Example: 635 - 10 iterations(by default). 636 atest <test> --retry-any-failure 637 - stop when passed or reached the 20th run. 638 atest <test> --retry-any-failure 20 639 640 641 - - - - - - - - - - - - 642 RUNNING TESTS ON AVD(s) 643 - - - - - - - - - - - - 644 645 Atest is able to run tests with the newly created AVD. Atest can build and 'acloud create' simultanously, and run tests after the AVD has been created successfully. 646 647 Examples: 648 - Start an AVD before running tests on that newly created device. 649 650 acloud create && atest <test> 651 652 can be simplified by: 653 654 atest <test> --start-avd 655 656 - Start AVD(s) by specifing 'acloud create' arguments and run tests on that newly created device. 657 658 atest <test> --acloud-create "--build-id 6509363 --build-target aosp_cf_x86_phone-userdebug --branch aosp_master" 659 660 To know detail about the argument, please run 'acloud create --help'. 661 662 [WARNING] 663 * --acloud-create must be the LAST optional argument: the remainder args will be consumed as its positional args. 664 * --acloud-create/--start-avd do not delete newly created AVDs. The users will be deleting them manually. 665 666 667 - - - - - - - - - - - - - - - - 668 REGRESSION DETECTION (obsolete) 669 - - - - - - - - - - - - - - - - 670 671 ********************** Warning ********************** 672 Please STOP using arguments below -- they are obsolete and will be removed in a near future: 673 --detect-regression 674 --generate-baseline 675 --generate-new-metrics 676 677 Please check RUNNING TESTS IN ITERATION out for alternatives. 678 ****************************************************** 679 680 Generate pre-patch or post-patch metrics without running regression detection: 681 682 Example: 683 atest <test> --generate-baseline <optional iter> 684 atest <test> --generate-new-metrics <optional iter> 685 686 Local regression detection can be run in three options: 687 688 1) Provide a folder containing baseline (pre-patch) metrics (generated previously). Atest will run the tests n (default 5) iterations, generate a new set of post-patch metrics, and compare those against existing metrics. 689 690 Example: 691 atest <test> --detect-regression </path/to/baseline> --generate-new-metrics <optional iter> 692 693 2) Provide a folder containing post-patch metrics (generated previously). Atest will run the tests n (default 5) iterations, generate a new set of pre-patch metrics, and compare those against those provided. Note: the developer needs to revert the device/tests to pre-patch state to generate baseline metrics. 694 695 Example: 696 atest <test> --detect-regression </path/to/new> --generate-baseline <optional iter> 697 698 3) Provide 2 folders containing both pre-patch and post-patch metrics. Atest will run no tests but the regression detection algorithm. 699 700 Example: 701 atest --detect-regression </path/to/baseline> </path/to/new> 702 703 704 - - - - - - - - - - - - 705 TESTS IN TEST MAPPING 706 - - - - - - - - - - - - 707 708 Atest can run tests in TEST_MAPPING files: 709 710 1) Run presubmit tests in TEST_MAPPING files in current and parent 711 directories. You can also specify a target directory. 712 713 Example: 714 atest (run presubmit tests in TEST_MAPPING files in current and parent directories) 715 atest --test-mapping </path/to/project> 716 (run presubmit tests in TEST_MAPPING files in </path/to/project> and its parent directories) 717 718 2) Run a specified test group in TEST_MAPPING files. 719 720 Example: 721 atest :postsubmit 722 (run postsubmit tests in TEST_MAPPING files in current and parent directories) 723 atest :all 724 (Run tests from all groups in TEST_MAPPING files) 725 atest --test-mapping </path/to/project>:postsubmit 726 (run postsubmit tests in TEST_MAPPING files in </path/to/project> and its parent directories) 727 728 3) Run tests in TEST_MAPPING files including sub directories 729 730 By default, atest will only search for tests in TEST_MAPPING files in current (or given directory) and its parent directories. If you want to run tests in TEST_MAPPING files in the sub-directories, you can use option --include-subdirs to force atest to include those tests too. 731 732 Example: 733 atest --include-subdirs [optional </path/to/project>:<test_group_name>] 734 (run presubmit tests in TEST_MAPPING files in current, sub and parent directories) 735 A path can be provided optionally if you want to search for tests in a given directory, with optional test group name. By default, the test group is presubmit. 736 737 738 - - - - - - - - - - - - - - 739 ADDITIONAL ARGS TO TRADEFED 740 - - - - - - - - - - - - - - 741 742 When trying to pass custom arguments for the test runners, everything after '--' 743 will be consumed as custom args. 744 745 Example: 746 atest -v <test> -- <custom_args1> <custom_args2> 747 748 749 2020-06-04 750''' 751