1 /* 2 * Copyright (C) 2015 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package com.android.compatibility.common.tradefed.testtype; 18 19 import com.android.compatibility.common.tradefed.build.CompatibilityBuildHelper; 20 import com.android.compatibility.common.tradefed.result.InvocationFailureHandler; 21 import com.android.compatibility.common.tradefed.result.SubPlanHelper; 22 import com.android.compatibility.common.tradefed.targetprep.NetworkConnectivityChecker; 23 import com.android.compatibility.common.tradefed.testtype.suite.CompatibilityTestSuite; 24 import com.android.compatibility.common.tradefed.util.RetryFilterHelper; 25 import com.android.compatibility.common.tradefed.util.RetryType; 26 import com.android.compatibility.common.tradefed.util.UniqueModuleCountUtil; 27 import com.android.compatibility.common.util.IInvocationResult; 28 import com.android.compatibility.common.util.ResultHandler; 29 import com.android.compatibility.common.util.TestFilter; 30 import com.android.ddmlib.Log.LogLevel; 31 import com.android.tradefed.build.IBuildInfo; 32 import com.android.tradefed.config.ConfigurationException; 33 import com.android.tradefed.config.Option; 34 import com.android.tradefed.config.Option.Importance; 35 import com.android.tradefed.config.OptionClass; 36 import com.android.tradefed.config.OptionCopier; 37 import com.android.tradefed.device.DeviceNotAvailableException; 38 import com.android.tradefed.device.DeviceUnresponsiveException; 39 import com.android.tradefed.device.ITestDevice; 40 import com.android.tradefed.invoker.IInvocationContext; 41 import com.android.tradefed.invoker.InvocationContext; 42 import com.android.tradefed.log.ITestLogger; 43 import com.android.tradefed.log.LogUtil.CLog; 44 import com.android.tradefed.result.ITestInvocationListener; 45 import com.android.tradefed.result.InputStreamSource; 46 import com.android.tradefed.result.LogDataType; 47 import com.android.tradefed.suite.checker.ISystemStatusChecker; 48 import com.android.tradefed.suite.checker.ISystemStatusCheckerReceiver; 49 import com.android.tradefed.suite.checker.StatusCheckerResult; 50 import com.android.tradefed.suite.checker.StatusCheckerResult.CheckStatus; 51 import com.android.tradefed.testtype.Abi; 52 import com.android.tradefed.testtype.IAbi; 53 import com.android.tradefed.testtype.IBuildReceiver; 54 import com.android.tradefed.testtype.IDeviceTest; 55 import com.android.tradefed.testtype.IInvocationContextReceiver; 56 import com.android.tradefed.testtype.IRemoteTest; 57 import com.android.tradefed.testtype.IShardableTest; 58 import com.android.tradefed.testtype.ITestCollector; 59 import com.android.tradefed.testtype.suite.TestSuiteInfo; 60 import com.android.tradefed.util.AbiFormatter; 61 import com.android.tradefed.util.AbiUtils; 62 import com.android.tradefed.util.ArrayUtil; 63 import com.android.tradefed.util.MultiMap; 64 import com.android.tradefed.util.StreamUtil; 65 import com.android.tradefed.util.TimeUtil; 66 67 import com.google.common.annotations.VisibleForTesting; 68 69 import java.io.ByteArrayOutputStream; 70 import java.io.FileNotFoundException; 71 import java.io.PrintWriter; 72 import java.util.ArrayList; 73 import java.util.Arrays; 74 import java.util.Collection; 75 import java.util.HashSet; 76 import java.util.LinkedHashSet; 77 import java.util.LinkedList; 78 import java.util.List; 79 import java.util.Set; 80 import java.util.concurrent.CountDownLatch; 81 import java.util.concurrent.TimeUnit; 82 83 /** 84 * A Test for running Compatibility Suites. 85 * @deprecated use {@link CompatibilityTestSuite} instead. 86 */ 87 @Deprecated 88 @OptionClass(alias = "compatibility") 89 public class CompatibilityTest implements IDeviceTest, IShardableTest, IBuildReceiver, 90 ISystemStatusCheckerReceiver, ITestCollector, 91 IInvocationContextReceiver { 92 93 public static final String INCLUDE_FILTER_OPTION = "include-filter"; 94 public static final String EXCLUDE_FILTER_OPTION = "exclude-filter"; 95 public static final String SUBPLAN_OPTION = "subplan"; 96 public static final String MODULE_OPTION = "module"; 97 public static final String TEST_OPTION = "test"; 98 public static final String PRECONDITION_ARG_OPTION = "precondition-arg"; 99 public static final String MODULE_ARG_OPTION = "module-arg"; 100 public static final String TEST_ARG_OPTION = "test-arg"; 101 public static final char TEST_OPTION_SHORT_NAME = 't'; 102 public static final String RETRY_OPTION = "retry"; 103 public static final String RETRY_TYPE_OPTION = "retry-type"; 104 public static final String ABI_OPTION = "abi"; 105 public static final String SHARD_OPTION = "shards"; 106 public static final String SKIP_DEVICE_INFO_OPTION = "skip-device-info"; 107 public static final String SKIP_PRECONDITIONS_OPTION = "skip-preconditions"; 108 public static final String SKIP_HOST_ARCH_CHECK = "skip-host-arch-check"; 109 public static final String PRIMARY_ABI_RUN = "primary-abi-only"; 110 public static final String DEVICE_TOKEN_OPTION = "device-token"; 111 public static final String LOGCAT_ON_FAILURE_SIZE_OPTION = "logcat-on-failure-size"; 112 113 // Constants for checking invocation or preconditions preparation failure 114 private static final int NUM_PREP_ATTEMPTS = 10; 115 private static final int MINUTES_PER_PREP_ATTEMPT = 2; 116 117 @Option(name = SUBPLAN_OPTION, 118 description = "the subplan to run", 119 importance = Importance.IF_UNSET) 120 private String mSubPlan; 121 122 @Option(name = INCLUDE_FILTER_OPTION, 123 description = "the include module filters to apply.", 124 importance = Importance.ALWAYS) 125 private Set<String> mIncludeFilters = new HashSet<>(); 126 127 @Option(name = EXCLUDE_FILTER_OPTION, 128 description = "the exclude module filters to apply.", 129 importance = Importance.ALWAYS) 130 private Set<String> mExcludeFilters = new HashSet<>(); 131 132 @Option(name = MODULE_OPTION, 133 shortName = 'm', 134 description = "the test module to run.", 135 importance = Importance.IF_UNSET) 136 private String mModuleName = null; 137 138 @Option(name = TEST_OPTION, 139 shortName = TEST_OPTION_SHORT_NAME, 140 description = "the test run.", 141 importance = Importance.IF_UNSET) 142 private String mTestName = null; 143 144 @Option(name = PRECONDITION_ARG_OPTION, 145 description = "the arguments to pass to a precondition. The expected format is" 146 + "\"<arg-name>:<arg-value>\"", 147 importance = Importance.ALWAYS) 148 private List<String> mPreconditionArgs = new ArrayList<>(); 149 150 @Option(name = MODULE_ARG_OPTION, 151 description = "the arguments to pass to a module. The expected format is" 152 + "\"<module-name>:<arg-name>:[<arg-key>:=]<arg-value>\"", 153 importance = Importance.ALWAYS) 154 private List<String> mModuleArgs = new ArrayList<>(); 155 156 @Option(name = TEST_ARG_OPTION, 157 description = "the arguments to pass to a test. The expected format is" 158 + "\"<test-class>:<arg-name>:[<arg-key>:=]<arg-value>\"", 159 importance = Importance.ALWAYS) 160 private List<String> mTestArgs = new ArrayList<>(); 161 162 @Option(name = RETRY_OPTION, 163 shortName = 'r', 164 description = "retry a previous session's failed and not executed tests.", 165 importance = Importance.IF_UNSET) 166 private Integer mRetrySessionId = null; 167 168 @Option(name = RETRY_TYPE_OPTION, 169 description = "used with " + RETRY_OPTION + ", retry tests of a certain status. " 170 + "Possible values include \"failed\", \"not_executed\", and \"custom\".", 171 importance = Importance.IF_UNSET) 172 private RetryType mRetryType = null; 173 174 @Option(name = ABI_OPTION, 175 shortName = 'a', 176 description = "the abi to test.", 177 importance = Importance.IF_UNSET) 178 private String mAbiName = null; 179 180 @Option(name = SHARD_OPTION, 181 description = "split the modules up to run on multiple devices concurrently. " 182 + "Deprecated, use --shard-count instead.") 183 @Deprecated 184 private int mShards = 1; 185 186 @Option(name = SKIP_DEVICE_INFO_OPTION, 187 shortName = 'd', 188 description = "Whether device info collection should be skipped") 189 private boolean mSkipDeviceInfo = false; 190 191 @Option(name = SKIP_HOST_ARCH_CHECK, 192 description = "Whether host architecture check should be skipped") 193 private boolean mSkipHostArchCheck = false; 194 195 @Option(name = SKIP_PRECONDITIONS_OPTION, 196 shortName = 'o', 197 description = "Whether preconditions should be skipped") 198 private boolean mSkipPreconditions = false; 199 200 @Option(name = PRIMARY_ABI_RUN, 201 description = "Whether to run tests with only the device primary abi. " 202 + "This override the --abi option.") 203 private boolean mPrimaryAbiRun = false; 204 205 @Option(name = DEVICE_TOKEN_OPTION, 206 description = "Holds the devices' tokens, used when scheduling tests that have" 207 + "prerequisites such as requiring a SIM card. Format is <serial>:<token>", 208 importance = Importance.ALWAYS) 209 private List<String> mDeviceTokens = new ArrayList<>(); 210 211 @Option(name = "bugreport-on-failure", 212 description = "Take a bugreport on every test failure. " + 213 "Warning: can potentially use a lot of disk space.") 214 private boolean mBugReportOnFailure = false; 215 216 @Option(name = "logcat-on-failure", 217 description = "Take a logcat snapshot on every test failure.") 218 private boolean mLogcatOnFailure = false; 219 220 @Option(name = LOGCAT_ON_FAILURE_SIZE_OPTION, 221 description = "The max number of logcat data in bytes to capture when " 222 + "--logcat-on-failure is on. Should be an amount that can comfortably fit in memory.") 223 private int mMaxLogcatBytes = 500 * 1024; // 500K 224 225 @Option(name = "screenshot-on-failure", 226 description = "Take a screenshot on every test failure.") 227 private boolean mScreenshotOnFailure = false; 228 229 @Option(name = "reboot-before-test", 230 description = "Reboot the device before the test suite starts.") 231 private boolean mRebootBeforeTest = false; 232 233 @Option(name = "reboot-on-failure", 234 description = "Reboot the device after every test failure.") 235 private boolean mRebootOnFailure = false; 236 237 @Option(name = "reboot-per-module", 238 description = "Reboot the device before every module run.") 239 private boolean mRebootPerModule = false; 240 241 @Option(name = "skip-connectivity-check", 242 description = "Don't verify device connectivity between module execution.") 243 private boolean mSkipConnectivityCheck = false; 244 245 @Option( 246 name = "preparer-whitelist", 247 description = 248 "Only run specific preparers." 249 + "Specify zero or more ITargetPreparers as canonical class names. " 250 + "e.g. \"com.android.compatibility.common.tradefed.targetprep.ApkInstaller\" " 251 + "If not specified, all configured preparers are run.") 252 private Set<String> mPreparerAllowlist = new HashSet<>(); 253 254 @Option(name = "skip-all-system-status-check", 255 description = "Whether all system status check between modules should be skipped") 256 private boolean mSkipAllSystemStatusCheck = false; 257 258 @Option(name = "skip-system-status-check", 259 description = "Disable specific system status checkers." 260 + "Specify zero or more SystemStatusChecker as canonical class names. e.g. " 261 + "\"com.android.compatibility.common.tradefed.targetprep.NetworkConnectivityChecker\" " 262 + "If not specified, all configured or whitelisted system status checkers are run.") 263 private Set<String> mSystemStatusCheckBlacklist = new HashSet<>(); 264 265 @Option(name = "system-status-check-whitelist", 266 description = "Only run specific system status checkers." 267 + "Specify zero or more SystemStatusChecker as canonical class names. e.g. " 268 + "\"com.android.compatibility.common.tradefed.targetprep.NetworkConnectivityChecker\" " 269 + "If not specified, all configured system status checkers are run.") 270 private Set<String> mSystemStatusCheckWhitelist = new HashSet<>(); 271 272 private List<ISystemStatusChecker> mListCheckers = new ArrayList<>(); 273 274 @Option(name = "collect-tests-only", 275 description = "Only invoke the suite to collect list of applicable test cases. All " 276 + "test run callbacks will be triggered, but test execution will not be " 277 + "actually carried out.") 278 private Boolean mCollectTestsOnly = null; 279 280 @Option(name = "module-metadata-include-filter", 281 description = "Include modules for execution based on matching of metadata fields: " 282 + "for any of the specified filter name and value, if a module has a metadata " 283 + "field with the same name and value, it will be included. When both module " 284 + "inclusion and exclusion rules are applied, inclusion rules will be " 285 + "evaluated first. Using this together with test filter inclusion rules may " 286 + "result in no tests to execute if the rules don't overlap.") 287 private MultiMap<String, String> mModuleMetadataIncludeFilter = new MultiMap<>(); 288 289 @Option(name = "module-metadata-exclude-filter", 290 description = "Exclude modules for execution based on matching of metadata fields: " 291 + "for any of the specified filter name and value, if a module has a metadata " 292 + "field with the same name and value, it will be excluded. When both module " 293 + "inclusion and exclusion rules are applied, inclusion rules will be " 294 + "evaluated first.") 295 private MultiMap<String, String> mModuleMetadataExcludeFilter = new MultiMap<>(); 296 297 private int mTotalShards; 298 private Integer mShardIndex = null; 299 private IModuleRepo mModuleRepo; 300 private ITestDevice mDevice; 301 private CompatibilityBuildHelper mBuildHelper; 302 303 // variables used for local sharding scenario 304 private static CountDownLatch sPreparedLatch; 305 private boolean mIsLocalSharding = false; 306 private boolean mIsSharded = false; 307 308 private IInvocationContext mInvocationContext; 309 310 /** 311 * Create a new {@link CompatibilityTest} that will run the default list of 312 * modules. 313 */ CompatibilityTest()314 public CompatibilityTest() { 315 this(1 /* totalShards */, new ModuleRepo(), 0); 316 } 317 318 /** 319 * Create a new {@link CompatibilityTest} that will run a sublist of 320 * modules. 321 */ CompatibilityTest(int totalShards, IModuleRepo moduleRepo, Integer shardIndex)322 public CompatibilityTest(int totalShards, IModuleRepo moduleRepo, Integer shardIndex) { 323 if (totalShards < 1) { 324 throw new IllegalArgumentException( 325 "Must be at least 1 shard. Given:" + totalShards); 326 } 327 mTotalShards = totalShards; 328 mModuleRepo = moduleRepo; 329 mShardIndex = shardIndex; 330 } 331 332 /** 333 * {@inheritDoc} 334 */ 335 @Override getDevice()336 public ITestDevice getDevice() { 337 return mDevice; 338 } 339 340 /** 341 * {@inheritDoc} 342 */ 343 @Override setDevice(ITestDevice device)344 public void setDevice(ITestDevice device) { 345 mDevice = device; 346 } 347 348 /** 349 * {@inheritDoc} 350 */ 351 @Override setBuild(IBuildInfo buildInfo)352 public void setBuild(IBuildInfo buildInfo) { 353 mBuildHelper = new CompatibilityBuildHelper(buildInfo); 354 } 355 356 /** 357 * {@inheritDoc} 358 */ 359 @Override run(ITestInvocationListener listener)360 public void run(ITestInvocationListener listener) throws DeviceNotAvailableException { 361 try { 362 List<ISystemStatusChecker> checkers = new ArrayList<>(); 363 // Get system status checkers 364 if (mSkipAllSystemStatusCheck) { 365 CLog.d("Skipping system status checkers"); 366 } else { 367 checkSystemStatusBlackAndWhiteList(); 368 for (ISystemStatusChecker checker : mListCheckers) { 369 if(shouldIncludeSystemStatusChecker(checker)) { 370 checkers.add(checker); 371 } 372 } 373 } 374 375 LinkedList<IModuleDef> modules = initializeModuleRepo(); 376 377 mExcludeFilters.clear(); 378 mIncludeFilters.clear(); 379 // Update BuildInfo in each shard to store the original command-line arguments from 380 // the session to be retried. These arguments will be serialized in the report later. 381 if (mRetrySessionId != null) { 382 loadRetryCommandLineArgs(mRetrySessionId); 383 } 384 385 listener = new FailureListener(listener, getDevice(), mBugReportOnFailure, 386 mLogcatOnFailure, mScreenshotOnFailure, mRebootOnFailure, mMaxLogcatBytes); 387 int moduleCount = modules.size(); 388 if (moduleCount == 0) { 389 CLog.logAndDisplay(LogLevel.INFO, "No module to run on %s.", 390 mDevice.getSerialNumber()); 391 // Make sure we unlock other shards. 392 if (sPreparedLatch != null) { 393 sPreparedLatch.countDown(); 394 } 395 return; 396 } else { 397 int uniqueModuleCount = UniqueModuleCountUtil.countUniqueModules(modules); 398 CLog.logAndDisplay(LogLevel.INFO, "Starting %d test sub-module%s on %s", 399 uniqueModuleCount, (uniqueModuleCount > 1) ? "s" : "", 400 mDevice.getSerialNumber()); 401 } 402 403 if (mRebootBeforeTest) { 404 CLog.d("Rebooting device before test starts as requested."); 405 mDevice.reboot(); 406 } 407 408 if (mSkipConnectivityCheck) { 409 String clazz = NetworkConnectivityChecker.class.getCanonicalName(); 410 CLog.logAndDisplay(LogLevel.INFO, "\"--skip-connectivity-check\" is deprecated, " 411 + "please use \"--skip-system-status-check %s\" instead", clazz); 412 mSystemStatusCheckBlacklist.add(clazz); 413 } 414 415 // Set values and run preconditions 416 boolean isPrepared = true; // whether the device has been successfully prepared 417 for (int i = 0; i < moduleCount; i++) { 418 IModuleDef module = modules.get(i); 419 module.setBuild(mBuildHelper.getBuildInfo()); 420 module.setDevice(mDevice); 421 module.setPreparerAllowlist(mPreparerAllowlist); 422 // don't set a value if unspecified 423 if (mCollectTestsOnly != null) { 424 module.setCollectTestsOnly(mCollectTestsOnly); 425 } 426 isPrepared &= module.prepare(mSkipPreconditions, mPreconditionArgs); 427 } 428 if (!isPrepared) { 429 throw new RuntimeException(String.format("Failed preconditions on %s", 430 mDevice.getSerialNumber())); 431 } 432 if (mIsLocalSharding) { 433 try { 434 sPreparedLatch.countDown(); 435 int attempt = 1; 436 while(!sPreparedLatch.await(MINUTES_PER_PREP_ATTEMPT, TimeUnit.MINUTES)) { 437 if (attempt > NUM_PREP_ATTEMPTS || 438 InvocationFailureHandler.hasFailed(mBuildHelper)) { 439 CLog.logAndDisplay(LogLevel.ERROR, 440 "Incorrect preparation detected, exiting test run from %s", 441 mDevice.getSerialNumber()); 442 return; 443 } 444 CLog.logAndDisplay(LogLevel.WARN, "waiting on preconditions"); 445 attempt++; 446 } 447 } catch (InterruptedException e) { 448 throw new RuntimeException(e); 449 } 450 } 451 // Module Repo is not useful anymore 452 mModuleRepo.tearDown(); 453 mModuleRepo = null; 454 // Run the tests 455 while (!modules.isEmpty()) { 456 // Make sure we remove the modules from the reference list when we are done with 457 // them. 458 IModuleDef module = modules.poll(); 459 long start = System.currentTimeMillis(); 460 461 if (mRebootPerModule) { 462 if ("user".equals(mDevice.getProperty("ro.build.type"))) { 463 CLog.e("reboot-per-module should only be used during development, " 464 + "this is a\" user\" build device"); 465 } else { 466 CLog.logAndDisplay(LogLevel.INFO, "Rebooting device before starting next " 467 + "module"); 468 mDevice.reboot(); 469 } 470 } 471 472 // execute pre module execution checker 473 if (checkers != null && !checkers.isEmpty()) { 474 runPreModuleCheck(module.getName(), checkers, mDevice, listener); 475 } 476 IInvocationContext moduleContext = new InvocationContext(); 477 moduleContext.setConfigurationDescriptor(module.getConfigurationDescriptor()); 478 moduleContext.addInvocationAttribute(IModuleDef.MODULE_NAME, module.getName()); 479 moduleContext.addInvocationAttribute(IModuleDef.MODULE_ABI, 480 module.getAbi().getName()); 481 // This format is not always true but for the deprecated runner this is best effort. 482 moduleContext.addInvocationAttribute( 483 IModuleDef.MODULE_ID, 484 String.format("%s %s", module.getAbi().getName(), module.getName())); 485 mInvocationContext.setModuleInvocationContext(moduleContext); 486 // Populate the module context with devices and builds 487 for (String deviceName : mInvocationContext.getDeviceConfigNames()) { 488 moduleContext.addAllocatedDevice( 489 deviceName, mInvocationContext.getDevice(deviceName)); 490 moduleContext.addDeviceBuildInfo( 491 deviceName, mInvocationContext.getBuildInfo(deviceName)); 492 } 493 module.setInvocationContext(moduleContext); 494 try { 495 listener.testModuleStarted(moduleContext); 496 module.run(listener); 497 } catch (DeviceUnresponsiveException due) { 498 // being able to catch a DeviceUnresponsiveException here implies that recovery 499 // was successful, and test execution should proceed to next module 500 ByteArrayOutputStream stack = new ByteArrayOutputStream(); 501 due.printStackTrace(new PrintWriter(stack, true)); 502 StreamUtil.close(stack); 503 CLog.w("Ignored DeviceUnresponsiveException because recovery was successful, " 504 + "proceeding with next module. Stack trace: %s", 505 stack.toString()); 506 CLog.w("This may be due to incorrect timeout setting on module %s", 507 module.getName()); 508 } finally { 509 // clear out module invocation context since we are now done with module 510 // execution 511 mInvocationContext.setModuleInvocationContext(null); 512 listener.testModuleEnded(); 513 } 514 long duration = System.currentTimeMillis() - start; 515 long expected = module.getRuntimeHint(); 516 long delta = Math.abs(duration - expected); 517 // Show warning if delta is more than 10% of expected 518 if (expected > 0 && ((float)delta / (float)expected) > 0.1f) { 519 CLog.logAndDisplay(LogLevel.WARN, 520 "Inaccurate runtime hint for %s, expected %s was %s", 521 module.getId(), 522 TimeUtil.formatElapsedTime(expected), 523 TimeUtil.formatElapsedTime(duration)); 524 } 525 if (checkers != null && !checkers.isEmpty()) { 526 runPostModuleCheck(module.getName(), checkers, mDevice, listener); 527 } 528 module = null; 529 } 530 } catch (FileNotFoundException fnfe) { 531 throw new RuntimeException("Failed to initialize modules", fnfe); 532 } 533 } 534 535 /** 536 * Initialize module repo. 537 * 538 * @return A list of module definition 539 * @throws DeviceNotAvailableException 540 * @throws FileNotFoundException 541 */ initializeModuleRepo()542 protected LinkedList<IModuleDef> initializeModuleRepo() 543 throws DeviceNotAvailableException, FileNotFoundException { 544 // FIXME: Each shard will do a full initialization which is not optimal. Need a way 545 // to be more specific on what to initialize. 546 synchronized (mModuleRepo) { 547 if (!mModuleRepo.isInitialized()) { 548 setupFilters(); 549 // Initialize the repository, {@link CompatibilityBuildHelper#getTestsDir} can 550 // throw a {@link FileNotFoundException} 551 mModuleRepo.initialize(mTotalShards, mShardIndex, mBuildHelper.getTestsDir(), 552 getAbis(), mDeviceTokens, mTestArgs, mModuleArgs, mIncludeFilters, 553 mExcludeFilters, mModuleMetadataIncludeFilter, mModuleMetadataExcludeFilter, 554 mBuildHelper.getBuildInfo()); 555 556 // Add the entire list of modules to the CompatibilityBuildHelper for reporting 557 mBuildHelper.setModuleIds(mModuleRepo.getModuleIds()); 558 559 int count = UniqueModuleCountUtil.countUniqueModules(mModuleRepo.getTokenModules()) 560 + UniqueModuleCountUtil.countUniqueModules( 561 mModuleRepo.getNonTokenModules()); 562 CLog.logAndDisplay(LogLevel.INFO, "========================================"); 563 CLog.logAndDisplay(LogLevel.INFO, "Starting a run with %s unique modules.", count); 564 CLog.logAndDisplay(LogLevel.INFO, "========================================"); 565 } else { 566 CLog.d("ModuleRepo already initialized."); 567 } 568 // Get the tests to run in this shard 569 return mModuleRepo.getModules(getDevice().getSerialNumber(), mShardIndex); 570 } 571 } 572 573 /** 574 * Gets the set of ABIs supported by both Compatibility and the device under test 575 * 576 * @return The set of ABIs to run the tests on 577 * @throws DeviceNotAvailableException 578 */ getAbis()579 Set<IAbi> getAbis() throws DeviceNotAvailableException { 580 Set<IAbi> abis = new LinkedHashSet<>(); 581 Set<String> archAbis = getAbisForBuildTargetArch(); 582 if (mPrimaryAbiRun) { 583 if (mAbiName == null) { 584 // Get the primary from the device and make it the --abi to run. 585 mAbiName = mDevice.getProperty("ro.product.cpu.abi").trim(); 586 } else { 587 CLog.d("Option --%s supersedes the option --%s, using abi: %s", ABI_OPTION, 588 PRIMARY_ABI_RUN, mAbiName); 589 } 590 } 591 if (mAbiName != null) { 592 // A particular abi was requested, it still need to be supported by the build. 593 if ((!mSkipHostArchCheck && !archAbis.contains(mAbiName)) || 594 !AbiUtils.isAbiSupportedByCompatibility(mAbiName)) { 595 throw new IllegalArgumentException(String.format("Your CTS hasn't been built with " 596 + "abi '%s' support, this CTS currently supports '%s'.", 597 mAbiName, archAbis)); 598 } else { 599 abis.add(new Abi(mAbiName, AbiUtils.getBitness(mAbiName))); 600 return abis; 601 } 602 } else { 603 // Run on all abi in common between the device and CTS. 604 List<String> deviceAbis = Arrays.asList(AbiFormatter.getSupportedAbis(mDevice, "")); 605 for (String abi : deviceAbis) { 606 if ((mSkipHostArchCheck || archAbis.contains(abi)) && 607 AbiUtils.isAbiSupportedByCompatibility(abi)) { 608 abis.add(new Abi(abi, AbiUtils.getBitness(abi))); 609 } else { 610 CLog.d("abi '%s' is supported by device but not by this CTS build (%s), tests " 611 + "will not run against it.", abi, archAbis); 612 } 613 } 614 if (abis.isEmpty()) { 615 throw new IllegalArgumentException(String.format("None of the abi supported by this" 616 + " CTS build ('%s') are supported by the device ('%s').", 617 archAbis, deviceAbis)); 618 } 619 return abis; 620 } 621 } 622 623 /** 624 * Return the abis supported by the Host build target architecture. 625 * Exposed for testing. 626 */ getAbisForBuildTargetArch()627 protected Set<String> getAbisForBuildTargetArch() { 628 return AbiUtils.getAbisForArch(TestSuiteInfo.getInstance().getTargetArchs().get(0)); 629 } 630 631 /** 632 * Check that the system status checker specified by option are valid. 633 */ checkSystemStatusBlackAndWhiteList()634 protected void checkSystemStatusBlackAndWhiteList() { 635 for (String checker : mSystemStatusCheckWhitelist) { 636 try { 637 Class.forName(checker); 638 } catch (ClassNotFoundException e) { 639 ConfigurationException ex = new ConfigurationException( 640 String.format("--system-status-check-whitelist must contains valid class, " 641 + "%s was not found", checker), e); 642 throw new RuntimeException(ex); 643 } 644 } 645 for (String checker : mSystemStatusCheckBlacklist) { 646 try { 647 Class.forName(checker); 648 } catch (ClassNotFoundException e) { 649 ConfigurationException ex = new ConfigurationException( 650 String.format("--skip-system-status-check must contains valid class, " 651 + "%s was not found", checker), e); 652 throw new RuntimeException(ex); 653 } 654 } 655 } 656 657 /** 658 * Resolve the inclusion and exclusion logic of system status checkers 659 * 660 * @param s the {@link ISystemStatusChecker} to perform filtering logic on 661 * @return True if the {@link ISystemStatusChecker} should be included, false otherwise. 662 */ shouldIncludeSystemStatusChecker(ISystemStatusChecker s)663 private boolean shouldIncludeSystemStatusChecker(ISystemStatusChecker s) { 664 String clazz = s.getClass().getCanonicalName(); 665 boolean shouldInclude = mSystemStatusCheckWhitelist.isEmpty() 666 || mSystemStatusCheckWhitelist.contains(clazz); 667 boolean shouldExclude = !mSystemStatusCheckBlacklist.isEmpty() 668 && mSystemStatusCheckBlacklist.contains(clazz); 669 return shouldInclude && !shouldExclude; 670 } 671 672 @VisibleForTesting runPreModuleCheck(String moduleName, List<ISystemStatusChecker> checkers, ITestDevice device, ITestLogger logger)673 void runPreModuleCheck(String moduleName, List<ISystemStatusChecker> checkers, 674 ITestDevice device, ITestLogger logger) throws DeviceNotAvailableException { 675 CLog.i("Running system status checker before module execution: %s", moduleName); 676 List<String> failures = new ArrayList<>(); 677 for (ISystemStatusChecker checker : checkers) { 678 StatusCheckerResult result = checker.preExecutionCheck(device); 679 if (!CheckStatus.SUCCESS.equals(result.getStatus())) { 680 failures.add(checker.getClass().getCanonicalName()); 681 CLog.w("System status checker [%s] failed", checker.getClass().getCanonicalName()); 682 } 683 } 684 if (!failures.isEmpty()) { 685 CLog.w("There are failed system status checkers: %s capturing a bugreport", 686 failures.toString()); 687 try (InputStreamSource bugSource = device.getBugreport()) { 688 logger.testLog(String.format("bugreport-checker-pre-module-%s", moduleName), 689 LogDataType.BUGREPORT, bugSource); 690 } 691 } 692 } 693 694 @VisibleForTesting runPostModuleCheck(String moduleName, List<ISystemStatusChecker> checkers, ITestDevice device, ITestLogger logger)695 void runPostModuleCheck(String moduleName, List<ISystemStatusChecker> checkers, 696 ITestDevice device, ITestLogger logger) throws DeviceNotAvailableException { 697 CLog.i("Running system status checker after module execution: %s", moduleName); 698 List<String> failures = new ArrayList<>(); 699 for (ISystemStatusChecker checker : checkers) { 700 StatusCheckerResult result = checker.postExecutionCheck(device); 701 if (!CheckStatus.SUCCESS.equals(result.getStatus())) { 702 failures.add(checker.getClass().getCanonicalName()); 703 CLog.w("System status checker [%s] failed", checker.getClass().getCanonicalName()); 704 } 705 } 706 if (!failures.isEmpty()) { 707 CLog.w("There are failed system status checkers: %s capturing a bugreport", 708 failures.toString()); 709 try (InputStreamSource bugSource = device.getBugreport()) { 710 logger.testLog(String.format("bugreport-checker-post-module-%s", moduleName), 711 LogDataType.BUGREPORT, bugSource); 712 } 713 } 714 } 715 716 /** 717 * Sets the retry command-line args to be stored in the BuildInfo and serialized into the 718 * report upon completion of the invocation. 719 */ loadRetryCommandLineArgs(Integer sessionId)720 void loadRetryCommandLineArgs(Integer sessionId) { 721 IInvocationResult result = null; 722 try { 723 result = ResultHandler.findResult(mBuildHelper.getResultsDir(), sessionId); 724 } catch (FileNotFoundException e) { 725 // We should never reach this point, because this method should only be called 726 // after setupFilters(), so result exists if we've gotten this far 727 throw new RuntimeException(e); 728 } 729 if (result == null) { 730 // Again, this should never happen 731 throw new IllegalArgumentException(String.format( 732 "Could not find session with id %d", sessionId)); 733 } 734 String retryCommandLineArgs = result.getCommandLineArgs(); 735 if (retryCommandLineArgs != null) { 736 mBuildHelper.setRetryCommandLineArgs(retryCommandLineArgs); 737 } 738 } 739 740 /** 741 * Sets the include/exclude filters up based on if a module name was given or whether this is a 742 * retry run. 743 */ setupFilters()744 void setupFilters() throws DeviceNotAvailableException { 745 if (mRetrySessionId != null) { 746 // Load the invocation result 747 RetryFilterHelper helper = createRetryFilterHelper(mRetrySessionId); 748 helper.validateBuildFingerprint(mDevice); 749 helper.setCommandLineOptionsFor(this); 750 helper.populateRetryFilters(); 751 mIncludeFilters = helper.getIncludeFilters(); 752 mExcludeFilters = helper.getExcludeFilters(); 753 helper.tearDown(); 754 } else { 755 if (mSubPlan != null) { 756 ISubPlan subPlan = SubPlanHelper.getSubPlanByName(mBuildHelper, mSubPlan); 757 mIncludeFilters.addAll(subPlan.getIncludeFilters()); 758 mExcludeFilters.addAll(subPlan.getExcludeFilters()); 759 } 760 if (mModuleName != null) { 761 try { 762 List<String> modules = ModuleRepo.getModuleNamesMatching( 763 mBuildHelper.getTestsDir(), mModuleName); 764 if (modules.size() == 0) { 765 throw new IllegalArgumentException( 766 String.format("No modules found matching %s", mModuleName)); 767 } else if (modules.size() > 1) { 768 throw new IllegalArgumentException(String.format("Multiple modules found" 769 + " matching %s:\n%s\nWhich one did you mean?\n", 770 mModuleName, ArrayUtil.join("\n", modules))); 771 } else { 772 String module = modules.get(0); 773 cleanFilters(mIncludeFilters, module); 774 cleanFilters(mExcludeFilters, module); 775 mIncludeFilters.add( 776 new TestFilter(mAbiName, module, mTestName).toString()); 777 } 778 } catch (FileNotFoundException e) { 779 throw new RuntimeException(e); 780 } 781 } else if (mTestName != null) { 782 throw new IllegalArgumentException( 783 "Test name given without module name. Add --module <module-name>"); 784 } 785 } 786 } 787 788 /* Creates a new {@link RetryFilterHelper} from attributes of this object. */ createRetryFilterHelper(Integer retrySessionId)789 protected RetryFilterHelper createRetryFilterHelper(Integer retrySessionId) { 790 return new RetryFilterHelper(mBuildHelper, retrySessionId, 791 mSubPlan, mIncludeFilters, mExcludeFilters, mAbiName, mModuleName, mTestName, 792 mRetryType); 793 } 794 795 /* Helper method designed to remove filters in a list not applicable to the given module */ cleanFilters(Set<String> filters, String module)796 private static void cleanFilters(Set<String> filters, String module) { 797 Set<String> cleanedFilters = new HashSet<String>(); 798 for (String filter : filters) { 799 if (module.equals(TestFilter.createFrom(filter).getName())) { 800 cleanedFilters.add(filter); // Module name matches, filter passes 801 } 802 } 803 filters.clear(); 804 filters.addAll(cleanedFilters); 805 } 806 807 /** 808 * {@inheritDoc} 809 */ 810 @Override split()811 public Collection<IRemoteTest> split() { 812 if (mShards <= 1) { 813 return null; 814 } 815 mIsLocalSharding = true; 816 List<IRemoteTest> shardQueue = new LinkedList<>(); 817 for (int i = 0; i < mShards; i++) { 818 CompatibilityTest test = (CompatibilityTest) getTestShard(mShards, i); 819 test.mIsLocalSharding = true; 820 shardQueue.add(test); 821 } 822 sPreparedLatch = new CountDownLatch(shardQueue.size()); 823 return shardQueue; 824 } 825 826 /** 827 * {@inheritDoc} 828 */ 829 @Override split(int shardCount)830 public Collection<IRemoteTest> split(int shardCount) { 831 if (shardCount <= 1 || mIsSharded) { 832 return null; 833 } 834 mIsSharded = true; 835 List<IRemoteTest> shardQueue = new LinkedList<>(); 836 for (int i = 0; i < shardCount; i++) { 837 CompatibilityTest test = (CompatibilityTest) getTestShard(shardCount, i); 838 shardQueue.add(test); 839 test.mIsSharded = true; 840 } 841 return shardQueue; 842 } 843 getTestShard(int shardCount, int shardIndex)844 private IRemoteTest getTestShard(int shardCount, int shardIndex) { 845 CompatibilityTest test = new CompatibilityTest(shardCount, mModuleRepo, shardIndex); 846 OptionCopier.copyOptionsNoThrow(this, test); 847 // Set the shard count because the copy option on the previous line 848 // copies over the mShard value 849 test.mShards = 0; 850 return test; 851 } 852 853 /** 854 * {@inheritDoc} 855 */ 856 @Override setSystemStatusChecker(List<ISystemStatusChecker> systemCheckers)857 public void setSystemStatusChecker(List<ISystemStatusChecker> systemCheckers) { 858 mListCheckers = systemCheckers; 859 } 860 861 @Override setCollectTestsOnly(boolean collectTestsOnly)862 public void setCollectTestsOnly(boolean collectTestsOnly) { 863 mCollectTestsOnly = collectTestsOnly; 864 } 865 866 /** 867 * Sets include-filters for the compatibility test 868 */ setIncludeFilter(Set<String> includeFilters)869 public void setIncludeFilter(Set<String> includeFilters) { 870 mIncludeFilters.addAll(includeFilters); 871 } 872 873 /** 874 * Sets exclude-filters for the compatibility test 875 */ setExcludeFilter(Set<String> excludeFilters)876 public void setExcludeFilter(Set<String> excludeFilters) { 877 mExcludeFilters.addAll(excludeFilters); 878 } 879 880 @Override setInvocationContext(IInvocationContext invocationContext)881 public void setInvocationContext(IInvocationContext invocationContext) { 882 mInvocationContext = invocationContext; 883 } 884 885 /** 886 * @return the mSubPlan 887 */ getSubPlan()888 protected String getSubPlan() { 889 return mSubPlan; 890 } 891 892 /** 893 * @return the mIncludeFilters 894 */ getIncludeFilters()895 protected Set<String> getIncludeFilters() { 896 return mIncludeFilters; 897 } 898 899 /** 900 * @return the mExcludeFilters 901 */ getExcludeFilters()902 protected Set<String> getExcludeFilters() { 903 return mExcludeFilters; 904 } 905 906 /** 907 * @return the mModuleName 908 */ getModuleName()909 protected String getModuleName() { 910 return mModuleName; 911 } 912 913 /** 914 * @return the mTestName 915 */ getTestName()916 protected String getTestName() { 917 return mTestName; 918 } 919 920 /** 921 * @return the mModuleArgs 922 */ getModuleArgs()923 protected List<String> getModuleArgs() { 924 return mModuleArgs; 925 } 926 927 /** 928 * @return the mTestArgs 929 */ getTestArgs()930 protected List<String> getTestArgs() { 931 return mTestArgs; 932 } 933 934 /** 935 * @return the mRetryType 936 */ getRetryType()937 protected RetryType getRetryType() { 938 return mRetryType; 939 } 940 941 /** 942 * @return the mAbiName 943 */ getAbiName()944 protected String getAbiName() { 945 return mAbiName; 946 } 947 948 /** 949 * @return the mDeviceTokens 950 */ getDeviceTokens()951 protected List<String> getDeviceTokens() { 952 return mDeviceTokens; 953 } 954 955 /** 956 * @return the mModuleMetadataIncludeFilter 957 */ getModuleMetadataIncludeFilter()958 protected MultiMap<String, String> getModuleMetadataIncludeFilter() { 959 return mModuleMetadataIncludeFilter; 960 } 961 962 /** 963 * @return the mModuleMetadataExcludeFilter 964 */ getModuleMetadataExcludeFilter()965 protected MultiMap<String, String> getModuleMetadataExcludeFilter() { 966 return mModuleMetadataExcludeFilter; 967 } 968 969 /** 970 * @return the mTotalShards 971 */ getTotalShards()972 protected int getTotalShards() { 973 return mTotalShards; 974 } 975 976 /** 977 * @return the mShardIndex 978 */ getShardIndex()979 protected Integer getShardIndex() { 980 return mShardIndex; 981 } 982 983 /** 984 * @return the mBuildHelper 985 */ getBuildHelper()986 protected CompatibilityBuildHelper getBuildHelper() { 987 return mBuildHelper; 988 } 989 990 /** 991 * @return the mInvocationContext 992 */ getInvocationContext()993 protected IInvocationContext getInvocationContext() { 994 return mInvocationContext; 995 } 996 997 /** 998 * @return the mModuleRepo 999 */ getModuleRepo()1000 protected IModuleRepo getModuleRepo() { 1001 return mModuleRepo; 1002 } 1003 } 1004