1 /* 2 * Copyright (C) 2017 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package com.android.tradefed.testtype.junit4; 18 19 import static org.junit.Assert.assertTrue; 20 21 import com.android.annotations.VisibleForTesting; 22 import com.android.ddmlib.Log.LogLevel; 23 import com.android.ddmlib.testrunner.RemoteAndroidTestRunner; 24 import com.android.ddmlib.testrunner.TestResult.TestStatus; 25 import com.android.tradefed.build.IBuildInfo; 26 import com.android.tradefed.device.DeviceNotAvailableException; 27 import com.android.tradefed.device.ITestDevice; 28 import com.android.tradefed.invoker.IInvocationContext; 29 import com.android.tradefed.invoker.TestInformation; 30 import com.android.tradefed.log.LogUtil.CLog; 31 import com.android.tradefed.result.CollectingTestListener; 32 import com.android.tradefed.result.ITestLifeCycleReceiver; 33 import com.android.tradefed.result.TestDescription; 34 import com.android.tradefed.result.TestResult; 35 import com.android.tradefed.result.TestRunResult; 36 import com.android.tradefed.result.ddmlib.DefaultRemoteAndroidTestRunner; 37 import com.android.tradefed.targetprep.BuildError; 38 import com.android.tradefed.targetprep.TargetSetupError; 39 import com.android.tradefed.targetprep.suite.SuiteApkInstaller; 40 import com.android.tradefed.testtype.IAbi; 41 import com.android.tradefed.testtype.IAbiReceiver; 42 import com.android.tradefed.testtype.ITestInformationReceiver; 43 import com.android.tradefed.util.ListInstrumentationParser; 44 import com.android.tradefed.util.ListInstrumentationParser.InstrumentationTarget; 45 46 import org.junit.After; 47 import org.junit.Assume; 48 49 import java.util.ArrayList; 50 import java.util.HashMap; 51 import java.util.LinkedHashMap; 52 import java.util.List; 53 import java.util.Map; 54 import java.util.concurrent.TimeUnit; 55 import java.util.stream.Collectors; 56 57 /** 58 * Base test class for running host JUnit4 style tests. This class provides support to install, run 59 * and clean up instrumentation tests from the host side. This class is multi-devices compatible. 60 * Should be the single source of truth to run instrumentation tests from host side in order to 61 * avoid duplicated utility and base class. 62 */ 63 public abstract class BaseHostJUnit4Test implements IAbiReceiver, ITestInformationReceiver { 64 65 static final long DEFAULT_TEST_TIMEOUT_MS = 10 * 60 * 1000L; 66 private static final long DEFAULT_MAX_TIMEOUT_TO_OUTPUT_MS = 10 * 60 * 1000L; // 10min 67 private static final Map<String, String> DEFAULT_INSTRUMENTATION_ARGS = new HashMap<>(); 68 69 private IAbi mAbi; 70 private TestInformation mTestInfo; 71 private Map<SuiteApkInstaller, ITestDevice> mInstallers = new LinkedHashMap<>(); 72 private TestRunResult mLatestInstruRes; 73 getDevice()74 public final ITestDevice getDevice() { 75 return mTestInfo.getDevice(); 76 } 77 getBuild()78 public final IBuildInfo getBuild() { 79 return mTestInfo.getBuildInfo(); 80 } 81 82 @Override setTestInformation(TestInformation testInformation)83 public final void setTestInformation(TestInformation testInformation) { 84 mTestInfo = testInformation; 85 } 86 87 @Override getTestInformation()88 public TestInformation getTestInformation() { 89 return mTestInfo; 90 } 91 92 @Override setAbi(IAbi abi)93 public final void setAbi(IAbi abi) { 94 mAbi = abi; 95 } 96 97 @Override getAbi()98 public final IAbi getAbi() { 99 return mAbi; 100 } 101 getInvocationContext()102 public final IInvocationContext getInvocationContext() { 103 return mTestInfo.getContext(); 104 } 105 getListDevices()106 public final List<ITestDevice> getListDevices() { 107 return mTestInfo.getContext().getDevices(); 108 } 109 110 /** 111 * Automatic tear down for all the apk installed. This will uninstall all the apk from the 112 * device they where installed on. 113 */ 114 @After autoTearDown()115 public final void autoTearDown() throws DeviceNotAvailableException { 116 mLatestInstruRes = null; 117 for (SuiteApkInstaller installer : mInstallers.keySet()) { 118 installer.tearDown(mTestInfo, null); 119 } 120 mInstallers.clear(); 121 } 122 123 // ------------------------- Utility APIs provided for tests ------------------------- 124 125 /** 126 * Install an apk given its name on the device. Apk will be auto-cleaned. 127 * 128 * @param apkFileName The name of the apk file. 129 * @param options extra options given to the install command 130 */ installPackage(String apkFileName, String... options)131 public final void installPackage(String apkFileName, String... options) 132 throws DeviceNotAvailableException, TargetSetupError { 133 installPackage(getDevice(), apkFileName, options); 134 } 135 136 /** 137 * Install an apk given its name on a given device. Apk will be auto-cleaned. 138 * 139 * @param device the {@link ITestDevice} on which to install the apk. 140 * @param apkFileName The name of the apk file. 141 * @param options extra options given to the install command 142 */ installPackage(ITestDevice device, String apkFileName, String... options)143 public final void installPackage(ITestDevice device, String apkFileName, String... options) 144 throws DeviceNotAvailableException, TargetSetupError { 145 SuiteApkInstaller installer = createSuiteApkInstaller(); 146 // Force the apk clean up 147 installer.setCleanApk(true); 148 // Store the preparer for cleanup 149 mInstallers.put(installer, device); 150 installer.addTestFileName(apkFileName); 151 installer.setAbi(getAbi()); 152 for (String option : options) { 153 installer.addInstallArg(option); 154 } 155 try { 156 installer.setUp(mTestInfo); 157 } catch (BuildError e) { 158 // For some reason we forgot the BuildError part of the interface so it's hard to add 159 // it now 160 throw new TargetSetupError( 161 e.getMessage(), e, device.getDeviceDescriptor(), e.getErrorId()); 162 } 163 } 164 165 /** 166 * Install an apk given its name for a specific user. 167 * 168 * @param apkFileName The name of the apk file. 169 * @param grantPermission whether to pass the grant permission flag when installing the apk. 170 * @param userId the user id of the user where to install the apk. 171 * @param options extra options given to the install command 172 */ installPackageAsUser( String apkFileName, boolean grantPermission, int userId, String... options)173 public final void installPackageAsUser( 174 String apkFileName, boolean grantPermission, int userId, String... options) 175 throws DeviceNotAvailableException, TargetSetupError { 176 installPackageAsUser(getDevice(), apkFileName, grantPermission, userId, options); 177 } 178 179 /** 180 * Install an apk given its name for a specific user on a given device. 181 * 182 * @param device the {@link ITestDevice} on which to install the apk. 183 * @param apkFileName The name of the apk file. 184 * @param grantPermission whether to pass the grant permission flag when installing the apk. 185 * @param userId the user id of the user where to install the apk. 186 * @param options extra options given to the install command 187 */ installPackageAsUser( ITestDevice device, String apkFileName, boolean grantPermission, int userId, String... options)188 public final void installPackageAsUser( 189 ITestDevice device, 190 String apkFileName, 191 boolean grantPermission, 192 int userId, 193 String... options) 194 throws DeviceNotAvailableException, TargetSetupError { 195 SuiteApkInstaller installer = createSuiteApkInstaller(); 196 // Force the apk clean up 197 installer.setCleanApk(true); 198 // Store the preparer for cleanup 199 mInstallers.put(installer, device); 200 installer.addTestFileName(apkFileName); 201 installer.setUserId(userId); 202 installer.setShouldGrantPermission(grantPermission); 203 installer.setAbi(getAbi()); 204 for (String option : options) { 205 installer.addInstallArg(option); 206 } 207 try { 208 installer.setUp(mTestInfo); 209 } catch (BuildError e) { 210 // For some reason we forgot the BuildError part of the interface so it's hard to add 211 // it now 212 throw new TargetSetupError( 213 e.getMessage(), e, device.getDeviceDescriptor(), e.getErrorId()); 214 } 215 } 216 217 /** 218 * Method to run an installed instrumentation package. Use {@link #getLastDeviceRunResults()} 219 * right after to get the details of results. 220 * 221 * @param pkgName the name of the package to run. 222 * @param testClassName the name of the test class to run. 223 * @return True if it succeed without failure. False otherwise. 224 */ runDeviceTests(String pkgName, String testClassName)225 public final boolean runDeviceTests(String pkgName, String testClassName) 226 throws DeviceNotAvailableException { 227 return runDeviceTests(getDevice(), pkgName, testClassName, null); 228 } 229 230 /** 231 * Method to run an installed instrumentation package. Use {@link #getLastDeviceRunResults()} 232 * right after to get the details of results. 233 * 234 * @param pkgName the name of the package to run. 235 * @param testClassName the name of the test class to run. 236 * @param testTimeoutMs the timeout in millisecond to be applied to each test case. 237 * @return True if it succeed without failure. False otherwise. 238 */ runDeviceTests(String pkgName, String testClassName, Long testTimeoutMs)239 public final boolean runDeviceTests(String pkgName, String testClassName, Long testTimeoutMs) 240 throws DeviceNotAvailableException { 241 return runDeviceTests( 242 getDevice(), 243 null, 244 pkgName, 245 testClassName, 246 null, 247 null, 248 testTimeoutMs, 249 DEFAULT_MAX_TIMEOUT_TO_OUTPUT_MS, 250 0L, 251 true, 252 false, 253 DEFAULT_INSTRUMENTATION_ARGS); 254 } 255 256 /** 257 * Method to run an installed instrumentation package. Use {@link #getLastDeviceRunResults()} 258 * right after to get the details of results. 259 * 260 * @param pkgName the name of the package to run. 261 * @param testClassName the name of the test class to run. 262 * @param userId the id of the user to run the test against. can be null. 263 * @param testTimeoutMs the timeout in millisecond to be applied to each test case. 264 * @return True if it succeed without failure. False otherwise. 265 */ runDeviceTests( String pkgName, String testClassName, Integer userId, Long testTimeoutMs)266 public final boolean runDeviceTests( 267 String pkgName, String testClassName, Integer userId, Long testTimeoutMs) 268 throws DeviceNotAvailableException { 269 return runDeviceTests( 270 getDevice(), 271 null, 272 pkgName, 273 testClassName, 274 null, 275 userId, 276 testTimeoutMs, 277 DEFAULT_MAX_TIMEOUT_TO_OUTPUT_MS, 278 0L, 279 true, 280 false, 281 DEFAULT_INSTRUMENTATION_ARGS); 282 } 283 284 /** 285 * Method to run an installed instrumentation package. Use {@link #getLastDeviceRunResults()} 286 * right after to get the details of results. 287 * 288 * @param pkgName the name of the package to run. 289 * @param testClassName the name of the test class to run. 290 * @param testMethodName the name of the test method in the class to be run. 291 * @return True if it succeed without failure. False otherwise. 292 */ runDeviceTests(String pkgName, String testClassName, String testMethodName)293 public final boolean runDeviceTests(String pkgName, String testClassName, String testMethodName) 294 throws DeviceNotAvailableException { 295 return runDeviceTests(getDevice(), pkgName, testClassName, testMethodName); 296 } 297 298 /** 299 * Method to run an installed instrumentation package. Use {@link #getLastDeviceRunResults()} 300 * right after to get the details of results. 301 * 302 * @param runner the instrumentation runner to be used. 303 * @param pkgName the name of the package to run. 304 * @param testClassName the name of the test class to run. 305 * @param testMethodName the name of the test method in the class to be run. 306 * @return True if it succeed without failure. False otherwise. 307 */ runDeviceTests( String runner, String pkgName, String testClassName, String testMethodName)308 public final boolean runDeviceTests( 309 String runner, String pkgName, String testClassName, String testMethodName) 310 throws DeviceNotAvailableException { 311 return runDeviceTests( 312 getDevice(), 313 runner, 314 pkgName, 315 testClassName, 316 testMethodName, 317 null, 318 DEFAULT_TEST_TIMEOUT_MS, 319 DEFAULT_MAX_TIMEOUT_TO_OUTPUT_MS, 320 0L, 321 true, 322 false, 323 DEFAULT_INSTRUMENTATION_ARGS); 324 } 325 326 /** 327 * Method to run an installed instrumentation package. Use {@link #getLastDeviceRunResults()} 328 * right after to get the details of results. 329 * 330 * @param device the device agaisnt which to run the instrumentation. 331 * @param pkgName the name of the package to run. 332 * @param testClassName the name of the test class to run. 333 * @param testMethodName the name of the test method in the class to be run. 334 * @return True if it succeed without failure. False otherwise. 335 */ runDeviceTests( ITestDevice device, String pkgName, String testClassName, String testMethodName)336 public final boolean runDeviceTests( 337 ITestDevice device, String pkgName, String testClassName, String testMethodName) 338 throws DeviceNotAvailableException { 339 return runDeviceTests( 340 device, 341 null, 342 pkgName, 343 testClassName, 344 testMethodName, 345 null, 346 DEFAULT_TEST_TIMEOUT_MS, 347 DEFAULT_MAX_TIMEOUT_TO_OUTPUT_MS, 348 0L, 349 true, 350 false, 351 DEFAULT_INSTRUMENTATION_ARGS); 352 } 353 354 /** 355 * Method to run an installed instrumentation package. Use {@link #getLastDeviceRunResults()} 356 * right after to get the details of results. 357 * 358 * @param device the device agaisnt which to run the instrumentation. 359 * @param pkgName the name of the package to run. 360 * @param testClassName the name of the test class to run. 361 * @param testMethodName the name of the test method in the class to be run. 362 * @param testTimeoutMs the timeout in millisecond to be applied to each test case. 363 * @return True if it succeed without failure. False otherwise. 364 */ runDeviceTests( ITestDevice device, String pkgName, String testClassName, String testMethodName, Long testTimeoutMs)365 public final boolean runDeviceTests( 366 ITestDevice device, 367 String pkgName, 368 String testClassName, 369 String testMethodName, 370 Long testTimeoutMs) 371 throws DeviceNotAvailableException { 372 return runDeviceTests( 373 device, 374 null, 375 pkgName, 376 testClassName, 377 testMethodName, 378 null, 379 testTimeoutMs, 380 DEFAULT_MAX_TIMEOUT_TO_OUTPUT_MS, 381 0L, 382 true, 383 false, 384 DEFAULT_INSTRUMENTATION_ARGS); 385 } 386 387 /** 388 * Method to run an installed instrumentation package. Use {@link #getLastDeviceRunResults()} 389 * right after to get the details of results. 390 * 391 * @param device the device agaisnt which to run the instrumentation. 392 * @param pkgName the name of the package to run. 393 * @param testClassName the name of the test class to run. 394 * @param testMethodName the name of the test method in the class to be run. 395 * @param userId the id of the user to run the test against. can be null. 396 * @param testTimeoutMs the timeout in millisecond to be applied to each test case. 397 * @return True if it succeed without failure. False otherwise. 398 */ runDeviceTests( ITestDevice device, String pkgName, String testClassName, String testMethodName, Integer userId, Long testTimeoutMs)399 public final boolean runDeviceTests( 400 ITestDevice device, 401 String pkgName, 402 String testClassName, 403 String testMethodName, 404 Integer userId, 405 Long testTimeoutMs) 406 throws DeviceNotAvailableException { 407 return runDeviceTests( 408 device, 409 null, 410 pkgName, 411 testClassName, 412 testMethodName, 413 userId, 414 testTimeoutMs, 415 DEFAULT_MAX_TIMEOUT_TO_OUTPUT_MS, 416 0L, 417 true, 418 false, 419 DEFAULT_INSTRUMENTATION_ARGS); 420 } 421 422 /** 423 * Method to run an installed instrumentation package. Use {@link #getLastDeviceRunResults()} 424 * right after to get the details of results. 425 * 426 * @param device the device agaisnt which to run the instrumentation. 427 * @param pkgName the name of the package to run. 428 * @param testClassName the name of the test class to run. 429 * @param testMethodName the name of the test method in the class to be run. 430 * @param testTimeoutMs the timeout in millisecond to be applied to each test case. 431 * @param maxTimeToOutputMs the max timeout the test has to start outputting something. 432 * @param maxInstrumentationTimeoutMs the max timeout the full instrumentation has to complete. 433 * @return True if it succeed without failure. False otherwise. 434 */ runDeviceTests( ITestDevice device, String pkgName, String testClassName, String testMethodName, Long testTimeoutMs, Long maxTimeToOutputMs, Long maxInstrumentationTimeoutMs)435 public final boolean runDeviceTests( 436 ITestDevice device, 437 String pkgName, 438 String testClassName, 439 String testMethodName, 440 Long testTimeoutMs, 441 Long maxTimeToOutputMs, 442 Long maxInstrumentationTimeoutMs) 443 throws DeviceNotAvailableException { 444 return runDeviceTests( 445 device, 446 null, 447 pkgName, 448 testClassName, 449 testMethodName, 450 null, 451 testTimeoutMs, 452 maxTimeToOutputMs, 453 maxInstrumentationTimeoutMs, 454 true, 455 false, 456 DEFAULT_INSTRUMENTATION_ARGS); 457 } 458 459 /** 460 * Runs the instrumentation base on the information in {@link DeviceTestRunOptions}. 461 * 462 * @param options the {@link DeviceTestRunOptions} driving the instrumentation setup. 463 * @return True if it succeeded without failure. False otherwise. 464 * @throws DeviceNotAvailableException 465 */ runDeviceTests(DeviceTestRunOptions options)466 public final boolean runDeviceTests(DeviceTestRunOptions options) 467 throws DeviceNotAvailableException { 468 return runDeviceTests( 469 options.getDevice() == null ? getDevice() : options.getDevice(), 470 options.getRunner(), 471 options.getPackageName(), 472 options.getTestClassName(), 473 options.getTestMethodName(), 474 options.getUserId(), 475 options.getTestTimeoutMs(), 476 options.getMaxTimeToOutputMs(), 477 options.getMaxInstrumentationTimeoutMs(), 478 options.shouldCheckResults(), 479 options.isHiddenApiCheckDisabled(), 480 options.isTestApiCheckDisabled(), 481 options.isIsolatedStorageDisabled(), 482 options.getInstrumentationArgs(), 483 options.getExtraListeners()); 484 } 485 486 /** 487 * Method to run an installed instrumentation package. Use {@link #getLastDeviceRunResults()} 488 * right after to get the details of results. 489 * 490 * @param device the device agaisnt which to run the instrumentation. 491 * @param pkgName the name of the package to run. 492 * @param testClassName the name of the test class to run. 493 * @param testMethodName the name of the test method in the class to be run. 494 * @param userId the id of the user to run the test against. can be null. 495 * @param testTimeoutMs the timeout in millisecond to be applied to each test case. 496 * @param maxTimeToOutputMs the max timeout the test has to start outputting something. 497 * @param maxInstrumentationTimeoutMs the max timeout the full instrumentation has to complete. 498 * @param checkResults whether or not the results are checked for crashes. 499 * @param isHiddenApiCheckDisabled whether or not we should disable the hidden api check. 500 * @param instrumentationArgs arguments to pass to the instrumentation. 501 * @return True if it succeeded without failure. False otherwise. 502 */ runDeviceTests( ITestDevice device, String runner, String pkgName, String testClassName, String testMethodName, Integer userId, Long testTimeoutMs, Long maxTimeToOutputMs, Long maxInstrumentationTimeoutMs, boolean checkResults, boolean isHiddenApiCheckDisabled, Map<String, String> instrumentationArgs)503 public final boolean runDeviceTests( 504 ITestDevice device, 505 String runner, 506 String pkgName, 507 String testClassName, 508 String testMethodName, 509 Integer userId, 510 Long testTimeoutMs, 511 Long maxTimeToOutputMs, 512 Long maxInstrumentationTimeoutMs, 513 boolean checkResults, 514 boolean isHiddenApiCheckDisabled, 515 Map<String, String> instrumentationArgs) 516 throws DeviceNotAvailableException { 517 return runDeviceTests( 518 device, 519 runner, 520 pkgName, 521 testClassName, 522 testMethodName, 523 userId, 524 testTimeoutMs, 525 maxTimeToOutputMs, 526 maxInstrumentationTimeoutMs, 527 checkResults, 528 isHiddenApiCheckDisabled, 529 true, 530 false, 531 instrumentationArgs, 532 new ArrayList<>()); 533 } 534 535 /** 536 * Method to run an installed instrumentation package. Use {@link #getLastDeviceRunResults()} 537 * right after to get the details of results. 538 * 539 * @param device the device agaisnt which to run the instrumentation. 540 * @param pkgName the name of the package to run. 541 * @param testClassName the name of the test class to run. 542 * @param testMethodName the name of the test method in the class to be run. 543 * @param userId the id of the user to run the test against. can be null. 544 * @param testTimeoutMs the timeout in millisecond to be applied to each test case. 545 * @param maxTimeToOutputMs the max timeout the test has to start outputting something. 546 * @param maxInstrumentationTimeoutMs the max timeout the full instrumentation has to complete. 547 * @param checkResults whether or not the results are checked for crashes. 548 * @param isHiddenApiCheckDisabled whether or not we should disable the hidden api check. 549 * @param isTestApiCheckDisabled whether or not we should disable the test api check. 550 * @param isIsolatedStorageDisabled whether or not we should disable isolated storage. 551 * @param instrumentationArgs arguments to pass to the instrumentation. 552 * @return True if it succeeded without failure. False otherwise. 553 */ runDeviceTests( ITestDevice device, String runner, String pkgName, String testClassName, String testMethodName, Integer userId, Long testTimeoutMs, Long maxTimeToOutputMs, Long maxInstrumentationTimeoutMs, boolean checkResults, boolean isHiddenApiCheckDisabled, boolean isTestApiCheckDisabled, boolean isIsolatedStorageDisabled, Map<String, String> instrumentationArgs, List<ITestLifeCycleReceiver> extraListeners)554 public final boolean runDeviceTests( 555 ITestDevice device, 556 String runner, 557 String pkgName, 558 String testClassName, 559 String testMethodName, 560 Integer userId, 561 Long testTimeoutMs, 562 Long maxTimeToOutputMs, 563 Long maxInstrumentationTimeoutMs, 564 boolean checkResults, 565 boolean isHiddenApiCheckDisabled, 566 boolean isTestApiCheckDisabled, 567 boolean isIsolatedStorageDisabled, 568 Map<String, String> instrumentationArgs, 569 List<ITestLifeCycleReceiver> extraListeners) 570 throws DeviceNotAvailableException { 571 return runDeviceTests( 572 device, 573 runner, 574 pkgName, 575 testClassName, 576 testMethodName, 577 userId, 578 testTimeoutMs, 579 maxTimeToOutputMs, 580 maxInstrumentationTimeoutMs, 581 checkResults, 582 isHiddenApiCheckDisabled, 583 isTestApiCheckDisabled, 584 isIsolatedStorageDisabled, 585 false, // leave window animations enabled for existing invocations 586 instrumentationArgs, 587 extraListeners); 588 } 589 590 /** 591 * Method to run an installed instrumentation package. Use {@link #getLastDeviceRunResults()} 592 * right after to get the details of results. 593 * 594 * @param device the device agaisnt which to run the instrumentation. 595 * @param pkgName the name of the package to run. 596 * @param testClassName the name of the test class to run. 597 * @param testMethodName the name of the test method in the class to be run. 598 * @param userId the id of the user to run the test against. can be null. 599 * @param testTimeoutMs the timeout in millisecond to be applied to each test case. 600 * @param maxTimeToOutputMs the max timeout the test has to start outputting something. 601 * @param maxInstrumentationTimeoutMs the max timeout the full instrumentation has to complete. 602 * @param checkResults whether or not the results are checked for crashes. 603 * @param isHiddenApiCheckDisabled whether or not we should disable the hidden api check. 604 * @param isTestApiCheckDisabled whether or not we should disable the test api check. 605 * @param isIsolatedStorageDisabled whether or not we should disable isolated storage. 606 * @param isWindowAnimationDisabled whether or not we should disable window animation. 607 * @param instrumentationArgs arguments to pass to the instrumentation. 608 * @return True if it succeeded without failure. False otherwise. 609 */ runDeviceTests( ITestDevice device, String runner, String pkgName, String testClassName, String testMethodName, Integer userId, Long testTimeoutMs, Long maxTimeToOutputMs, Long maxInstrumentationTimeoutMs, boolean checkResults, boolean isHiddenApiCheckDisabled, boolean isTestApiCheckDisabled, boolean isIsolatedStorageDisabled, boolean isWindowAnimationDisabled, Map<String, String> instrumentationArgs, List<ITestLifeCycleReceiver> extraListeners)610 public final boolean runDeviceTests( 611 ITestDevice device, 612 String runner, 613 String pkgName, 614 String testClassName, 615 String testMethodName, 616 Integer userId, 617 Long testTimeoutMs, 618 Long maxTimeToOutputMs, 619 Long maxInstrumentationTimeoutMs, 620 boolean checkResults, 621 boolean isHiddenApiCheckDisabled, 622 boolean isTestApiCheckDisabled, 623 boolean isIsolatedStorageDisabled, 624 boolean isWindowAnimationDisabled, 625 Map<String, String> instrumentationArgs, 626 List<ITestLifeCycleReceiver> extraListeners) 627 throws DeviceNotAvailableException { 628 TestRunResult runResult = 629 doRunTests( 630 device, 631 runner, 632 pkgName, 633 testClassName, 634 testMethodName, 635 userId, 636 testTimeoutMs, 637 maxTimeToOutputMs, 638 maxInstrumentationTimeoutMs, 639 isHiddenApiCheckDisabled, 640 isTestApiCheckDisabled, 641 isIsolatedStorageDisabled, 642 isWindowAnimationDisabled, 643 instrumentationArgs, 644 extraListeners); 645 mLatestInstruRes = runResult; 646 printTestResult(runResult); 647 if (checkResults) { 648 if (runResult.isRunFailure()) { 649 throw new AssertionError( 650 "Failed to successfully run device tests for " 651 + runResult.getName() 652 + ": " 653 + runResult.getRunFailureMessage()); 654 } 655 if (runResult.getNumTests() == 0) { 656 throw new AssertionError("No tests were run on the device"); 657 } 658 if (runResult.hasFailedTests()) { 659 // build a meaningful error message 660 StringBuilder errorBuilder = new StringBuilder("on-device tests failed:\n"); 661 for (Map.Entry<TestDescription, TestResult> resultEntry : 662 runResult.getTestResults().entrySet()) { 663 if (!TestStatus.PASSED.equals(resultEntry.getValue().getStatus())) { 664 errorBuilder.append(resultEntry.getKey().toString()); 665 errorBuilder.append(":\n"); 666 errorBuilder.append(resultEntry.getValue().getStackTrace()); 667 } 668 } 669 throw new AssertionError(errorBuilder.toString()); 670 } 671 // Assume not all tests have skipped (and rethrow AssumptionViolatedException if so) 672 List<TestResult> assumpFail = 673 runResult.getTestsResultsInState(TestStatus.ASSUMPTION_FAILURE); 674 List<String> messages = 675 assumpFail.stream().map(r -> r.getStackTrace()).collect(Collectors.toList()); 676 String errors = String.join("\n\n", messages); 677 Assume.assumeTrue( 678 errors, 679 runResult.getNumTests() 680 != runResult.getNumTestsInState(TestStatus.ASSUMPTION_FAILURE)); 681 } 682 return !runResult.hasFailedTests() && runResult.getNumTests() > 0; 683 } 684 685 /** 686 * Returns the {@link TestRunResult} resulting from the latest runDeviceTests that ran. Or null 687 * if no results available. 688 */ getLastDeviceRunResults()689 public final TestRunResult getLastDeviceRunResults() { 690 return mLatestInstruRes; 691 } 692 693 /** Helper method to run tests and return the listener that collected the results. */ doRunTests( ITestDevice device, String runner, String pkgName, String testClassName, String testMethodName, Integer userId, Long testTimeoutMs, Long maxTimeToOutputMs, Long maxInstrumentationTimeoutMs, boolean isHiddenApiCheckDisabled, boolean isTestApiCheckDisabled, boolean isIsolatedStorageDisabled, boolean isWindowAnimationDisabled, Map<String, String> instrumentationArgs, List<ITestLifeCycleReceiver> extraListeners)694 private TestRunResult doRunTests( 695 ITestDevice device, 696 String runner, 697 String pkgName, 698 String testClassName, 699 String testMethodName, 700 Integer userId, 701 Long testTimeoutMs, 702 Long maxTimeToOutputMs, 703 Long maxInstrumentationTimeoutMs, 704 boolean isHiddenApiCheckDisabled, 705 boolean isTestApiCheckDisabled, 706 boolean isIsolatedStorageDisabled, 707 boolean isWindowAnimationDisabled, 708 Map<String, String> instrumentationArgs, 709 List<ITestLifeCycleReceiver> extraListeners) 710 throws DeviceNotAvailableException { 711 RemoteAndroidTestRunner testRunner = createTestRunner(pkgName, runner, device); 712 String runOptions = ""; 713 // hidden-api-checks flag only exists in P and after. 714 // Using a temp variable to consolidate the dynamic checks 715 int apiLevel = isHiddenApiCheckDisabled || isWindowAnimationDisabled 716 ? device.getApiLevel() : 0; 717 if (isHiddenApiCheckDisabled && (apiLevel >= 28)) { 718 runOptions += "--no-hidden-api-checks "; 719 } 720 if (!isHiddenApiCheckDisabled 721 && !isTestApiCheckDisabled 722 && device.checkApiLevelAgainstNextRelease(30)) { 723 runOptions += "--no-test-api-access "; 724 } 725 // isolated-storage flag only exists in Q and after. 726 if (isIsolatedStorageDisabled && device.checkApiLevelAgainstNextRelease(29)) { 727 runOptions += "--no-isolated-storage "; 728 } 729 // window-animation flag only exists in ICS and after 730 if (isWindowAnimationDisabled && apiLevel >= 14) { 731 runOptions += "--no-window-animation "; 732 } 733 734 if (getAbi() != null) { 735 runOptions += String.format("--abi %s", getAbi().getName()); 736 } 737 // Set the run options if any. 738 if (!runOptions.isEmpty()) { 739 testRunner.setRunOptions(runOptions); 740 } 741 742 if (testClassName != null && testMethodName != null) { 743 testRunner.setMethodName(testClassName, testMethodName); 744 } else if (testClassName != null) { 745 testRunner.setClassName(testClassName); 746 } 747 748 if (testTimeoutMs != null) { 749 testRunner.addInstrumentationArg("timeout_msec", Long.toString(testTimeoutMs)); 750 } else { 751 testRunner.addInstrumentationArg( 752 "timeout_msec", Long.toString(DEFAULT_TEST_TIMEOUT_MS)); 753 } 754 if (maxTimeToOutputMs != null) { 755 testRunner.setMaxTimeToOutputResponse(maxTimeToOutputMs, TimeUnit.MILLISECONDS); 756 } 757 if (maxInstrumentationTimeoutMs != null) { 758 testRunner.setMaxTimeout(maxInstrumentationTimeoutMs, TimeUnit.MILLISECONDS); 759 } 760 // Pass all the instrumentation arguments 761 for (String key : instrumentationArgs.keySet()) { 762 testRunner.addInstrumentationArg(key, instrumentationArgs.get(key)); 763 } 764 765 CollectingTestListener listener = createListener(); 766 List<ITestLifeCycleReceiver> allReceiver = new ArrayList<>(); 767 allReceiver.add(listener); 768 allReceiver.addAll(extraListeners); 769 if (userId == null) { 770 assertTrue(device.runInstrumentationTests(testRunner, allReceiver)); 771 } else { 772 assertTrue(device.runInstrumentationTestsAsUser(testRunner, userId, allReceiver)); 773 } 774 return listener.getCurrentRunResults(); 775 } 776 777 @VisibleForTesting createTestRunner( String packageName, String runnerName, ITestDevice device)778 RemoteAndroidTestRunner createTestRunner( 779 String packageName, String runnerName, ITestDevice device) 780 throws DeviceNotAvailableException { 781 if (runnerName == null) { 782 ListInstrumentationParser parser = getListInstrumentationParser(); 783 device.executeShellCommand("pm list instrumentation", parser); 784 for (InstrumentationTarget target : parser.getInstrumentationTargets()) { 785 if (packageName.equals(target.packageName)) { 786 runnerName = target.runnerName; 787 } 788 } 789 } 790 // If the runner name is still null 791 if (runnerName == null) { 792 throw new RuntimeException("No runner was defined and couldn't dynamically find one."); 793 } 794 return new DefaultRemoteAndroidTestRunner(packageName, runnerName, device.getIDevice()); 795 } 796 797 @VisibleForTesting getListInstrumentationParser()798 ListInstrumentationParser getListInstrumentationParser() { 799 return new ListInstrumentationParser(); 800 } 801 802 @VisibleForTesting createListener()803 CollectingTestListener createListener() { 804 return new CollectingTestListener(); 805 } 806 printTestResult(TestRunResult runResult)807 private void printTestResult(TestRunResult runResult) { 808 for (Map.Entry<TestDescription, TestResult> testEntry : 809 runResult.getTestResults().entrySet()) { 810 TestResult testResult = testEntry.getValue(); 811 TestStatus testStatus = testResult.getStatus(); 812 CLog.logAndDisplay(LogLevel.INFO, "Test " + testEntry.getKey() + ": " + testStatus); 813 if (!TestStatus.PASSED.equals(testStatus) 814 && !TestStatus.ASSUMPTION_FAILURE.equals(testStatus)) { 815 CLog.logAndDisplay(LogLevel.WARN, testResult.getStackTrace()); 816 } 817 } 818 } 819 820 /** 821 * Uninstalls a package on the device. 822 * 823 * @param pkgName the Android package to uninstall 824 * @return a {@link String} with an error code, or <code>null</code> if success 825 */ uninstallPackage(String pkgName)826 public final String uninstallPackage(String pkgName) throws DeviceNotAvailableException { 827 return getDevice().uninstallPackage(pkgName); 828 } 829 830 /** 831 * Uninstalls a package on the device 832 * 833 * @param device the device that should uninstall the package. 834 * @param pkgName the Android package to uninstall 835 * @return a {@link String} with an error code, or <code>null</code> if success 836 */ uninstallPackage(ITestDevice device, String pkgName)837 public final String uninstallPackage(ITestDevice device, String pkgName) 838 throws DeviceNotAvailableException { 839 return device.uninstallPackage(pkgName); 840 } 841 842 /** 843 * Checks if a package of a given name is installed on the device 844 * 845 * @param pkg the name of the package 846 * @return true if the package is found on the device 847 */ isPackageInstalled(String pkg)848 public final boolean isPackageInstalled(String pkg) throws DeviceNotAvailableException { 849 return isPackageInstalled(getDevice(), pkg); 850 } 851 852 /** 853 * Checks if a package of a given name is installed on the device 854 * 855 * @param device the device that should uninstall the package. 856 * @param pkg the name of the package 857 * @return true if the package is found on the device 858 */ isPackageInstalled(ITestDevice device, String pkg)859 public final boolean isPackageInstalled(ITestDevice device, String pkg) 860 throws DeviceNotAvailableException { 861 for (String installedPackage : device.getInstalledPackageNames()) { 862 if (pkg.equals(installedPackage)) { 863 return true; 864 } 865 } 866 return false; 867 } 868 hasDeviceFeature(String feature)869 public boolean hasDeviceFeature(String feature) throws DeviceNotAvailableException { 870 return getDevice().hasFeature("feature:" + feature); 871 } 872 873 @VisibleForTesting createSuiteApkInstaller()874 SuiteApkInstaller createSuiteApkInstaller() { 875 return new SuiteApkInstaller(); 876 } 877 } 878