1 /* 2 * Copyright (C) 2016 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package com.android.tradefed.testtype.suite; 17 18 import com.android.annotations.VisibleForTesting; 19 import com.android.ddmlib.Log.LogLevel; 20 import com.android.tradefed.build.BuildRetrievalError; 21 import com.android.tradefed.build.IBuildInfo; 22 import com.android.tradefed.build.IDeviceBuildInfo; 23 import com.android.tradefed.config.ConfigurationException; 24 import com.android.tradefed.config.DynamicRemoteFileResolver; 25 import com.android.tradefed.config.IConfiguration; 26 import com.android.tradefed.config.IConfigurationReceiver; 27 import com.android.tradefed.config.IDeviceConfiguration; 28 import com.android.tradefed.config.Option; 29 import com.android.tradefed.config.Option.Importance; 30 import com.android.tradefed.config.OptionCopier; 31 import com.android.tradefed.device.DeviceNotAvailableException; 32 import com.android.tradefed.device.DeviceProperties; 33 import com.android.tradefed.device.ITestDevice; 34 import com.android.tradefed.device.NullDevice; 35 import com.android.tradefed.device.StubDevice; 36 import com.android.tradefed.device.cloud.NestedRemoteDevice; 37 import com.android.tradefed.device.metric.CollectorHelper; 38 import com.android.tradefed.device.metric.IMetricCollector; 39 import com.android.tradefed.device.metric.IMetricCollectorReceiver; 40 import com.android.tradefed.invoker.IInvocationContext; 41 import com.android.tradefed.invoker.TestInformation; 42 import com.android.tradefed.invoker.logger.InvocationMetricLogger; 43 import com.android.tradefed.invoker.logger.InvocationMetricLogger.InvocationMetricKey; 44 import com.android.tradefed.invoker.shard.token.ITokenRequest; 45 import com.android.tradefed.invoker.shard.token.TokenProperty; 46 import com.android.tradefed.log.ITestLogger; 47 import com.android.tradefed.log.LogUtil.CLog; 48 import com.android.tradefed.metrics.proto.MetricMeasurement.Metric; 49 import com.android.tradefed.result.ITestInvocationListener; 50 import com.android.tradefed.result.ITestLoggerReceiver; 51 import com.android.tradefed.result.ResultForwarder; 52 import com.android.tradefed.retry.IRetryDecision; 53 import com.android.tradefed.retry.RetryStrategy; 54 import com.android.tradefed.suite.checker.ISystemStatusChecker; 55 import com.android.tradefed.suite.checker.ISystemStatusCheckerReceiver; 56 import com.android.tradefed.suite.checker.StatusCheckerResult; 57 import com.android.tradefed.suite.checker.StatusCheckerResult.CheckStatus; 58 import com.android.tradefed.targetprep.ITargetPreparer; 59 import com.android.tradefed.testtype.Abi; 60 import com.android.tradefed.testtype.IAbi; 61 import com.android.tradefed.testtype.IBuildReceiver; 62 import com.android.tradefed.testtype.IDeviceTest; 63 import com.android.tradefed.testtype.IInvocationContextReceiver; 64 import com.android.tradefed.testtype.IRemoteTest; 65 import com.android.tradefed.testtype.IReportNotExecuted; 66 import com.android.tradefed.testtype.IRuntimeHintProvider; 67 import com.android.tradefed.testtype.IShardableTest; 68 import com.android.tradefed.testtype.ITestCollector; 69 import com.android.tradefed.util.AbiFormatter; 70 import com.android.tradefed.util.AbiUtils; 71 import com.android.tradefed.util.MultiMap; 72 import com.android.tradefed.util.TimeUtil; 73 74 import com.google.inject.Inject; 75 import com.google.inject.Injector; 76 77 import java.io.File; 78 import java.io.FileNotFoundException; 79 import java.lang.reflect.InvocationTargetException; 80 import java.util.ArrayList; 81 import java.util.Arrays; 82 import java.util.Collection; 83 import java.util.Collections; 84 import java.util.HashMap; 85 import java.util.HashSet; 86 import java.util.Iterator; 87 import java.util.LinkedHashMap; 88 import java.util.LinkedHashSet; 89 import java.util.List; 90 import java.util.Map; 91 import java.util.Map.Entry; 92 import java.util.Random; 93 import java.util.Set; 94 import java.util.stream.Collectors; 95 96 /** 97 * Abstract class used to run Test Suite. This class provide the base of how the Suite will be run. 98 * Each implementation can define the list of tests via the {@link #loadTests()} method. 99 */ 100 public abstract class ITestSuite 101 implements IRemoteTest, 102 IDeviceTest, 103 IBuildReceiver, 104 ISystemStatusCheckerReceiver, 105 IShardableTest, 106 ITestCollector, 107 IInvocationContextReceiver, 108 IRuntimeHintProvider, 109 IMetricCollectorReceiver, 110 IConfigurationReceiver, 111 IReportNotExecuted, 112 ITokenRequest, 113 ITestLoggerReceiver { 114 115 public static final String SKIP_SYSTEM_STATUS_CHECKER = "skip-system-status-check"; 116 public static final String RUNNER_WHITELIST = "runner-whitelist"; 117 public static final String PREPARER_WHITELIST = "preparer-whitelist"; 118 public static final String MODULE_CHECKER_PRE = "PreModuleChecker"; 119 public static final String MODULE_CHECKER_POST = "PostModuleChecker"; 120 public static final String ABI_OPTION = "abi"; 121 public static final String SKIP_HOST_ARCH_CHECK = "skip-host-arch-check"; 122 public static final String PRIMARY_ABI_RUN = "primary-abi-only"; 123 public static final String PARAMETER_KEY = "parameter"; 124 public static final String MAINLINE_PARAMETER_KEY = "mainline-param"; 125 public static final String ACTIVE_MAINLINE_PARAMETER_KEY = "active-mainline-parameter"; 126 public static final String TOKEN_KEY = "token"; 127 public static final String MODULE_METADATA_INCLUDE_FILTER = "module-metadata-include-filter"; 128 public static final String MODULE_METADATA_EXCLUDE_FILTER = "module-metadata-exclude-filter"; 129 public static final String RANDOM_SEED = "random-seed"; 130 public static final String REBOOT_BEFORE_TEST = "reboot-before-test"; 131 132 private static final String PRODUCT_CPU_ABI_KEY = "ro.product.cpu.abi"; 133 134 // Options for test failure case 135 @Option( 136 name = "bugreport-on-failure", 137 description = 138 "Take a bugreport on every test failure. Warning: This may require a lot" 139 + "of storage space of the machine running the tests." 140 ) 141 private boolean mBugReportOnFailure = false; 142 143 @Deprecated 144 @Option( 145 name = "logcat-on-failure", 146 description = "Take a logcat snapshot on every test failure." 147 ) 148 private boolean mLogcatOnFailure = false; 149 150 @Deprecated 151 @Option( 152 name = "logcat-on-failure-size", 153 description = 154 "The max number of logcat data in bytes to capture when " 155 + "--logcat-on-failure is on. Should be an amount that can comfortably fit in memory." 156 ) 157 private int mMaxLogcatBytes = 500 * 1024; // 500K 158 159 @Deprecated 160 @Option( 161 name = "screenshot-on-failure", 162 description = "Take a screenshot on every test failure." 163 ) 164 private boolean mScreenshotOnFailure = false; 165 166 @Option(name = "reboot-on-failure", 167 description = "Reboot the device after every test failure.") 168 private boolean mRebootOnFailure = false; 169 170 // Options for suite runner behavior 171 @Option(name = "reboot-per-module", description = "Reboot the device before every module run.") 172 private boolean mRebootPerModule = false; 173 174 @Option( 175 name = REBOOT_BEFORE_TEST, 176 description = "Reboot the device before the test suite starts." 177 ) 178 private boolean mRebootBeforeTest = false; 179 180 @Option(name = "skip-all-system-status-check", 181 description = "Whether all system status check between modules should be skipped") 182 private boolean mSkipAllSystemStatusCheck = false; 183 184 @Option( 185 name = SKIP_SYSTEM_STATUS_CHECKER, 186 description = 187 "Disable specific system status checkers." 188 + "Specify zero or more SystemStatusChecker as canonical class names. e.g. " 189 + "\"com.android.tradefed.suite.checker.KeyguardStatusChecker\" If not " 190 + "specified, all configured or whitelisted system status checkers will " 191 + "run." 192 ) 193 private Set<String> mSystemStatusCheckBlacklist = new HashSet<>(); 194 195 @Option( 196 name = "report-system-checkers", 197 description = "Whether reporting system checkers as test or not." 198 ) 199 private boolean mReportSystemChecker = false; 200 201 @Option( 202 name = "random-order", 203 description = "Whether randomizing the order of the modules to be ran or not." 204 ) 205 private boolean mRandomOrder = false; 206 207 @Option( 208 name = RANDOM_SEED, 209 description = "Seed to randomize the order of the modules." 210 ) 211 private long mRandomSeed = -1; 212 213 @Option( 214 name = "collect-tests-only", 215 description = 216 "Only invoke the suite to collect list of applicable test cases. All " 217 + "test run callbacks will be triggered, but test execution will not be " 218 + "actually carried out." 219 ) 220 private boolean mCollectTestsOnly = false; 221 222 // Abi related options 223 @Option( 224 name = ABI_OPTION, 225 shortName = 'a', 226 description = "the abi to test. For example: 'arm64-v8a'.", 227 importance = Importance.IF_UNSET 228 ) 229 private String mAbiName = null; 230 231 @Option( 232 name = SKIP_HOST_ARCH_CHECK, 233 description = "Whether host architecture check should be skipped." 234 ) 235 private boolean mSkipHostArchCheck = false; 236 237 @Option( 238 name = PRIMARY_ABI_RUN, 239 description = 240 "Whether to run tests with only the device primary abi. " 241 + "This is overriden by the --abi option." 242 ) 243 private boolean mPrimaryAbiRun = false; 244 245 @Option( 246 name = MODULE_METADATA_INCLUDE_FILTER, 247 description = 248 "Include modules for execution based on matching of metadata fields: for any of " 249 + "the specified filter name and value, if a module has a metadata field " 250 + "with the same name and value, it will be included. When both module " 251 + "inclusion and exclusion rules are applied, inclusion rules will be " 252 + "evaluated first. Using this together with test filter inclusion rules " 253 + "may result in no tests to execute if the rules don't overlap." 254 ) 255 private MultiMap<String, String> mModuleMetadataIncludeFilter = new MultiMap<>(); 256 257 @Option( 258 name = MODULE_METADATA_EXCLUDE_FILTER, 259 description = 260 "Exclude modules for execution based on matching of metadata fields: for any of " 261 + "the specified filter name and value, if a module has a metadata field " 262 + "with the same name and value, it will be excluded. When both module " 263 + "inclusion and exclusion rules are applied, inclusion rules will be " 264 + "evaluated first." 265 ) 266 private MultiMap<String, String> mModuleMetadataExcludeFilter = new MultiMap<>(); 267 268 @Option(name = RUNNER_WHITELIST, description = "Runner class(es) that are allowed to run.") 269 private Set<String> mAllowedRunners = new HashSet<>(); 270 271 @Option( 272 name = PREPARER_WHITELIST, 273 description = 274 "Preparer class(es) that are allowed to run. This mostly usefeul for dry-runs." 275 ) 276 private Set<String> mAllowedPreparers = new HashSet<>(); 277 278 @Option( 279 name = "enable-module-dynamic-download", 280 description = 281 "Whether or not to allow the downloading of dynamic @option files at module level." 282 ) 283 private boolean mEnableDynamicDownload = false; 284 285 @Option( 286 name = "intra-module-sharding", 287 description = "Whether or not to allow intra-module sharding." 288 ) 289 private boolean mIntraModuleSharding = true; 290 291 @Option( 292 name = "isolated-module", 293 description = "Whether or not to attempt the module isolation between modules" 294 ) 295 private boolean mIsolatedModule = false; 296 297 /** @deprecated to be deleted when next version is deployed */ 298 @Deprecated 299 @Option( 300 name = "max-testcase-run-count", 301 description = 302 "If the IRemoteTest can have its testcases run multiple times, " 303 + "the max number of runs for each testcase." 304 ) 305 private int mMaxRunLimit = 1; 306 307 /** @deprecated to be deleted when next version is deployed */ 308 @Deprecated 309 @Option( 310 name = "retry-strategy", 311 description = 312 "The retry strategy to be used when re-running some tests with " 313 + "--max-testcase-run-count" 314 ) 315 private RetryStrategy mRetryStrategy = RetryStrategy.NO_RETRY; 316 317 // [Options relate to module retry and intra-module retry][ 318 @Option( 319 name = "merge-attempts", 320 description = "Whether or not to use the merge the results of the different attempts." 321 ) 322 private boolean mMergeAttempts = true; 323 // end [Options relate to module retry and intra-module retry] 324 325 private ITestDevice mDevice; 326 private IBuildInfo mBuildInfo; 327 private List<ISystemStatusChecker> mSystemStatusCheckers; 328 private IInvocationContext mContext; 329 private List<IMetricCollector> mMetricCollectors; 330 private IConfiguration mMainConfiguration; 331 332 // Sharding attributes 333 private boolean mIsSharded = false; 334 private ModuleDefinition mDirectModule = null; 335 private boolean mShouldMakeDynamicModule = true; 336 337 // Guice object 338 private Injector mInjector; 339 340 // Current modules to run, null if not started to run yet. 341 private List<ModuleDefinition> mRunModules = null; 342 private ModuleDefinition mModuleInProgress = null; 343 // Logger to be used to files. 344 private ITestLogger mCurrentLogger = null; 345 // Whether or not we are currently in split 346 private boolean mIsSplitting = false; 347 348 private boolean mDisableAutoRetryTimeReporting = false; 349 350 private DynamicRemoteFileResolver mDynamicResolver = new DynamicRemoteFileResolver(); 351 352 @VisibleForTesting setDynamicResolver(DynamicRemoteFileResolver resolver)353 void setDynamicResolver(DynamicRemoteFileResolver resolver) { 354 mDynamicResolver = resolver; 355 } 356 357 /** 358 * Get the current Guice {@link Injector} from the invocation. It should allow us to continue 359 * the object injection of modules. 360 */ 361 @Inject setInvocationInjector(Injector injector)362 public void setInvocationInjector(Injector injector) { 363 mInjector = injector; 364 } 365 366 /** Forward our invocation scope guice objects to whoever needs them in modules. */ applyGuiceInjection(LinkedHashMap<String, IConfiguration> runConfig)367 private void applyGuiceInjection(LinkedHashMap<String, IConfiguration> runConfig) { 368 if (mInjector == null) { 369 // TODO: Convert to a strong failure 370 CLog.d("No injector received by the suite."); 371 return; 372 } 373 for (IConfiguration config : runConfig.values()) { 374 for (IRemoteTest test : config.getTests()) { 375 mInjector.injectMembers(test); 376 } 377 } 378 } 379 380 /** 381 * Abstract method to load the tests configuration that will be run. Each tests is defined by a 382 * {@link IConfiguration} and a unique name under which it will report results. 383 */ loadTests()384 public abstract LinkedHashMap<String, IConfiguration> loadTests(); 385 386 /** 387 * Return an instance of the class implementing {@link ITestSuite}. 388 */ createInstance()389 private ITestSuite createInstance() { 390 try { 391 return this.getClass().getDeclaredConstructor().newInstance(); 392 } catch (InstantiationException 393 | IllegalAccessException 394 | InvocationTargetException 395 | NoSuchMethodException e) { 396 throw new RuntimeException(e); 397 } 398 } 399 getTestsDir()400 public File getTestsDir() throws FileNotFoundException { 401 IBuildInfo build = getBuildInfo(); 402 if (build instanceof IDeviceBuildInfo) { 403 return ((IDeviceBuildInfo) build).getTestsDir(); 404 } 405 // TODO: handle multi build? 406 throw new FileNotFoundException("Could not found a tests dir folder."); 407 } 408 loadAndFilter()409 private LinkedHashMap<String, IConfiguration> loadAndFilter() { 410 LinkedHashMap<String, IConfiguration> runConfig = loadTests(); 411 if (runConfig.isEmpty()) { 412 CLog.i("No config were loaded. Nothing to run."); 413 return runConfig; 414 } 415 // Apply our guice scope to all modules objects 416 applyGuiceInjection(runConfig); 417 418 Set<String> moduleNames = new HashSet<>(); 419 LinkedHashMap<String, IConfiguration> filteredConfig = new LinkedHashMap<>(); 420 for (Entry<String, IConfiguration> config : runConfig.entrySet()) { 421 if (!mModuleMetadataIncludeFilter.isEmpty() 422 || !mModuleMetadataExcludeFilter.isEmpty()) { 423 if (!filterByConfigMetadata( 424 config.getValue(), 425 mModuleMetadataIncludeFilter, 426 mModuleMetadataExcludeFilter)) { 427 // if the module config did not pass the metadata filters, it's excluded 428 // from execution. 429 continue; 430 } 431 } 432 if (!filterByRunnerType(config.getValue(), mAllowedRunners)) { 433 // if the module config did not pass the runner type filter, it's excluded from 434 // execution. 435 continue; 436 } 437 filterPreparers(config.getValue(), mAllowedPreparers); 438 439 // Copy the CoverageOptions from the main configuration to the module configuration. 440 if (mMainConfiguration != null) { 441 config.getValue().setCoverageOptions(mMainConfiguration.getCoverageOptions()); 442 } 443 444 filteredConfig.put(config.getKey(), config.getValue()); 445 moduleNames.add(config.getValue().getConfigurationDescription().getModuleName()); 446 } 447 448 if (mBuildInfo != null 449 && mBuildInfo.getRemoteFiles() != null 450 && mBuildInfo.getRemoteFiles().size() > 0) { 451 stageTestArtifacts(mDevice, moduleNames); 452 } 453 454 runConfig.clear(); 455 return filteredConfig; 456 } 457 458 /** Helper to download all artifacts for the given modules. */ stageTestArtifacts(ITestDevice device, Set<String> modules)459 private void stageTestArtifacts(ITestDevice device, Set<String> modules) { 460 CLog.i(String.format("Start to stage test artifacts for %d modules.", modules.size())); 461 long startTime = System.currentTimeMillis(); 462 // Include the file if its path contains a folder name matching any of the module. 463 String moduleRegex = 464 modules.stream() 465 .map(m -> String.format("/%s/", m)) 466 .collect(Collectors.joining("|")); 467 List<String> includeFilters = Arrays.asList(moduleRegex); 468 // Ignore config file as it's part of config zip artifact that's staged already. 469 List<String> excludeFilters = Arrays.asList("[.]config$"); 470 mDynamicResolver.setDevice(device); 471 mDynamicResolver.addExtraArgs( 472 mMainConfiguration.getCommandOptions().getDynamicDownloadArgs()); 473 for (File remoteFile : mBuildInfo.getRemoteFiles()) { 474 try { 475 mDynamicResolver.resolvePartialDownloadZip( 476 getTestsDir(), remoteFile.toString(), includeFilters, excludeFilters); 477 } catch (BuildRetrievalError | FileNotFoundException e) { 478 CLog.e( 479 String.format( 480 "Failed to download partial zip from %s for modules: %s", 481 remoteFile, String.join(", ", modules))); 482 CLog.e(e); 483 throw new RuntimeException(e); 484 } 485 } 486 long elapsedTime = System.currentTimeMillis() - startTime; 487 InvocationMetricLogger.addInvocationMetrics( 488 InvocationMetricKey.STAGE_TESTS_TIME, elapsedTime); 489 CLog.i( 490 String.format( 491 "Staging test artifacts for %d modules finished in %s.", 492 modules.size(), TimeUtil.formatElapsedTime(elapsedTime))); 493 } 494 495 /** Helper that creates and returns the list of {@link ModuleDefinition} to be executed. */ createExecutionList()496 private List<ModuleDefinition> createExecutionList() { 497 List<ModuleDefinition> runModules = new ArrayList<>(); 498 if (mDirectModule != null) { 499 // If we are sharded and already know what to run then we just do it. 500 runModules.add(mDirectModule); 501 mDirectModule.setDevice(mDevice); 502 mDirectModule.setBuild(mBuildInfo); 503 return runModules; 504 } 505 506 LinkedHashMap<String, IConfiguration> runConfig = loadAndFilter(); 507 if (runConfig.isEmpty()) { 508 CLog.i("No config were loaded. Nothing to run."); 509 return runModules; 510 } 511 512 for (Entry<String, IConfiguration> config : runConfig.entrySet()) { 513 // Validate the configuration, it will throw if not valid. 514 ValidateSuiteConfigHelper.validateConfig(config.getValue()); 515 Map<String, List<ITargetPreparer>> preparersPerDevice = 516 getPreparerPerDevice(config.getValue()); 517 ModuleDefinition module = 518 new ModuleDefinition( 519 config.getKey(), 520 config.getValue().getTests(), 521 preparersPerDevice, 522 config.getValue().getMultiTargetPreparers(), 523 config.getValue()); 524 if (mDisableAutoRetryTimeReporting) { 525 module.disableAutoRetryReportingTime(); 526 } 527 module.setDevice(mDevice); 528 module.setBuild(mBuildInfo); 529 runModules.add(module); 530 } 531 532 /** Randomize all the modules to be ran if random-order is set and no sharding.*/ 533 if (mRandomOrder) { 534 randomizeTestModules(runModules, mRandomSeed); 535 } 536 537 CLog.logAndDisplay(LogLevel.DEBUG, "[Total Unique Modules = %s]", runModules.size()); 538 // Free the map once we are done with it. 539 runConfig = null; 540 return runModules; 541 } 542 543 /** 544 * Helper method that handle randomizing the order of the modules. 545 * 546 * @param runModules The {@code List<ModuleDefinition>} of the test modules to be ran. 547 * @param randomSeed The {@code long} seed used to randomize the order of test modules, use the 548 * current time as seed if no specified seed provided. 549 */ 550 @VisibleForTesting randomizeTestModules(List<ModuleDefinition> runModules, long randomSeed)551 void randomizeTestModules(List<ModuleDefinition> runModules, long randomSeed) { 552 // Use current time as seed if no specified seed provided. 553 if (randomSeed == -1) { 554 randomSeed = System.currentTimeMillis(); 555 } 556 CLog.i("Randomizing all the modules with seed: %s", randomSeed); 557 Collections.shuffle(runModules, new Random(randomSeed)); 558 mBuildInfo.addBuildAttribute(RANDOM_SEED, String.valueOf(randomSeed)); 559 } 560 checkClassLoad(Set<String> classes, String type)561 private void checkClassLoad(Set<String> classes, String type) { 562 for (String c : classes) { 563 try { 564 Class.forName(c); 565 } catch (ClassNotFoundException e) { 566 ConfigurationException ex = 567 new ConfigurationException( 568 String.format( 569 "--%s must contains valid class, %s was not found", 570 type, c), 571 e); 572 throw new RuntimeException(ex); 573 } 574 } 575 } 576 577 /** Create the mapping of device to its target_preparer. */ getPreparerPerDevice(IConfiguration config)578 private Map<String, List<ITargetPreparer>> getPreparerPerDevice(IConfiguration config) { 579 Map<String, List<ITargetPreparer>> res = new LinkedHashMap<>(); 580 for (IDeviceConfiguration holder : config.getDeviceConfig()) { 581 List<ITargetPreparer> preparers = new ArrayList<>(); 582 res.put(holder.getDeviceName(), preparers); 583 preparers.addAll(holder.getTargetPreparers()); 584 } 585 return res; 586 } 587 588 /** 589 * Opportunity to clean up all the things that were needed during the suites setup but are not 590 * required to run the tests. 591 */ cleanUpSuiteSetup()592 void cleanUpSuiteSetup() { 593 // Empty by default. 594 } 595 596 /** Generic run method for all test loaded from {@link #loadTests()}. */ 597 @Override run(TestInformation testInfo, ITestInvocationListener listener)598 public final void run(TestInformation testInfo, ITestInvocationListener listener) 599 throws DeviceNotAvailableException { 600 mCurrentLogger = listener; 601 // Load and check the module checkers, runners and preparers in black and whitelist 602 checkClassLoad(mSystemStatusCheckBlacklist, SKIP_SYSTEM_STATUS_CHECKER); 603 checkClassLoad(mAllowedRunners, RUNNER_WHITELIST); 604 checkClassLoad(mAllowedPreparers, PREPARER_WHITELIST); 605 606 mRunModules = createExecutionList(); 607 // Check if we have something to run. 608 if (mRunModules.isEmpty()) { 609 CLog.i("No tests to be run."); 610 return; 611 } 612 613 // Allow checkers to log files for easier debugging. 614 for (ISystemStatusChecker checker : mSystemStatusCheckers) { 615 if (checker instanceof ITestLoggerReceiver) { 616 ((ITestLoggerReceiver) checker).setTestLogger(listener); 617 } 618 } 619 620 // If requested reboot each device before the testing starts. 621 if (mRebootBeforeTest) { 622 for (ITestDevice device : mContext.getDevices()) { 623 if (!(device.getIDevice() instanceof StubDevice)) { 624 CLog.d( 625 "Rebooting device '%s' before test starts as requested.", 626 device.getSerialNumber()); 627 mDevice.reboot(); 628 } 629 } 630 } 631 632 /** Setup a special listener to take actions on test failures. */ 633 TestFailureListener failureListener = 634 new TestFailureListener( 635 mContext.getDevices(), mBugReportOnFailure, mRebootOnFailure); 636 /** Create the list of listeners applicable at the module level. */ 637 List<ITestInvocationListener> moduleListeners = createModuleListeners(); 638 639 // Only print the running log if we are going to run something. 640 if (mRunModules.get(0).hasTests()) { 641 CLog.logAndDisplay( 642 LogLevel.INFO, 643 "%s running %s modules: %s", 644 mDevice.getSerialNumber(), 645 mRunModules.size(), 646 mRunModules); 647 } 648 649 /** Run all the module, make sure to reduce the list to release resources as we go. */ 650 try { 651 while (!mRunModules.isEmpty()) { 652 ModuleDefinition module = mRunModules.remove(0); 653 // Before running the module we ensure it has tests at this point or skip completely 654 // to avoid running SystemCheckers and preparation for nothing. 655 if (module.hasTests()) { 656 continue; 657 } 658 659 // Populate the module context with devices and builds 660 for (String deviceName : mContext.getDeviceConfigNames()) { 661 module.getModuleInvocationContext() 662 .addAllocatedDevice(deviceName, mContext.getDevice(deviceName)); 663 module.getModuleInvocationContext() 664 .addDeviceBuildInfo(deviceName, mContext.getBuildInfo(deviceName)); 665 } 666 listener.testModuleStarted(module.getModuleInvocationContext()); 667 mModuleInProgress = module; 668 // Trigger module start on module level listener too 669 new ResultForwarder(moduleListeners) 670 .testModuleStarted(module.getModuleInvocationContext()); 671 TestInformation moduleInfo = 672 TestInformation.createModuleTestInfo( 673 testInfo, module.getModuleInvocationContext()); 674 try { 675 runSingleModule(module, moduleInfo, listener, moduleListeners, failureListener); 676 } finally { 677 // Trigger module end on module level listener too 678 new ResultForwarder(moduleListeners).testModuleEnded(); 679 // clear out module invocation context since we are now done with module 680 // execution 681 listener.testModuleEnded(); 682 mModuleInProgress = null; 683 } 684 // Module isolation routine 685 moduleIsolation(mContext, listener); 686 } 687 } catch (DeviceNotAvailableException e) { 688 CLog.e( 689 "A DeviceNotAvailableException occurred, following modules did not run: %s", 690 mRunModules); 691 reportNotExecuted(listener, "Module did not run due to device not available."); 692 throw e; 693 } 694 } 695 696 /** 697 * Returns the list of {@link ITestInvocationListener} applicable to the {@link ModuleListener} 698 * level. These listeners will be re-used for each module, they will not be re-instantiated so 699 * they should not assume an internal state. 700 */ createModuleListeners()701 protected List<ITestInvocationListener> createModuleListeners() { 702 return new ArrayList<>(); 703 } 704 705 /** 706 * Routine that attempt to reset a device between modules in order to provide isolation. 707 * 708 * @param context The invocation context. 709 * @param logger A logger where extra logs can be saved. 710 * @throws DeviceNotAvailableException 711 */ moduleIsolation(IInvocationContext context, ITestLogger logger)712 private void moduleIsolation(IInvocationContext context, ITestLogger logger) 713 throws DeviceNotAvailableException { 714 // TODO: we can probably make it smarter: Did any test ran for example? 715 ITestDevice device = context.getDevices().get(0); 716 if (mIsolatedModule && (device instanceof NestedRemoteDevice)) { 717 boolean res = 718 ((NestedRemoteDevice) device) 719 .resetVirtualDevice( 720 logger, 721 context.getBuildInfos().get(0), 722 /* Do not collect the logs */ false); 723 if (!res) { 724 String serial = device.getSerialNumber(); 725 throw new DeviceNotAvailableException( 726 String.format( 727 "Failed to reset the AVD '%s' during module isolation.", serial), 728 serial); 729 } 730 } 731 } 732 733 /** 734 * Helper method that handle running a single module logic. 735 * 736 * @param module The {@link ModuleDefinition} to be ran. 737 * @param moduleInfo The {@link TestInformation} for the module. 738 * @param listener The {@link ITestInvocationListener} where to report results 739 * @param moduleListeners The {@link ITestInvocationListener}s that runs at the module level. 740 * @param failureListener special listener that we add to collect information on failures. 741 * @throws DeviceNotAvailableException 742 */ runSingleModule( ModuleDefinition module, TestInformation moduleInfo, ITestInvocationListener listener, List<ITestInvocationListener> moduleListeners, TestFailureListener failureListener)743 private void runSingleModule( 744 ModuleDefinition module, 745 TestInformation moduleInfo, 746 ITestInvocationListener listener, 747 List<ITestInvocationListener> moduleListeners, 748 TestFailureListener failureListener) 749 throws DeviceNotAvailableException { 750 if (mRebootPerModule) { 751 if ("user".equals(mDevice.getProperty(DeviceProperties.BUILD_TYPE))) { 752 CLog.e( 753 "reboot-per-module should only be used during development, " 754 + "this is a\" user\" build device"); 755 } else { 756 CLog.d("Rebooting device before starting next module"); 757 mDevice.reboot(); 758 } 759 } 760 761 if (!mSkipAllSystemStatusCheck) { 762 runPreModuleCheck(module.getId(), mSystemStatusCheckers, mDevice, listener); 763 } 764 if (mCollectTestsOnly) { 765 module.setCollectTestsOnly(mCollectTestsOnly); 766 } 767 // Pass the run defined collectors to be used. 768 module.setMetricCollectors(CollectorHelper.cloneCollectors(mMetricCollectors)); 769 // Pass the main invocation logSaver 770 module.setLogSaver(mMainConfiguration.getLogSaver()); 771 772 IRetryDecision decision = mMainConfiguration.getRetryDecision(); 773 // Pass whether we should merge the attempts of not 774 if (mMergeAttempts 775 && decision.getMaxRetryCount() > 1 776 && !RetryStrategy.NO_RETRY.equals(decision.getRetryStrategy())) { 777 CLog.d("Overriding '--merge-attempts' to false for auto-retry."); 778 mMergeAttempts = false; 779 } 780 module.setMergeAttemps(mMergeAttempts); 781 // Pass the retry decision to be used. 782 module.setRetryDecision(decision); 783 784 module.setEnableDynamicDownload(mEnableDynamicDownload); 785 module.addDynamicDownloadArgs( 786 mMainConfiguration.getCommandOptions().getDynamicDownloadArgs()); 787 // Actually run the module 788 module.run( 789 moduleInfo, 790 listener, 791 moduleListeners, 792 failureListener, 793 getConfiguration().getRetryDecision().getMaxRetryCount()); 794 795 if (!mSkipAllSystemStatusCheck) { 796 runPostModuleCheck(module.getId(), mSystemStatusCheckers, mDevice, listener); 797 } 798 } 799 800 /** 801 * Helper to run the System Status checkers preExecutionChecks defined for the test and log 802 * their failures. 803 */ runPreModuleCheck( String moduleName, List<ISystemStatusChecker> checkers, ITestDevice device, ITestInvocationListener listener)804 private void runPreModuleCheck( 805 String moduleName, 806 List<ISystemStatusChecker> checkers, 807 ITestDevice device, 808 ITestInvocationListener listener) 809 throws DeviceNotAvailableException { 810 long startTime = System.currentTimeMillis(); 811 CLog.i("Running system status checker before module execution: %s", moduleName); 812 Map<String, String> failures = new LinkedHashMap<>(); 813 boolean bugreportNeeded = false; 814 for (ISystemStatusChecker checker : checkers) { 815 // Check if the status checker should be skipped. 816 if (mSystemStatusCheckBlacklist.contains(checker.getClass().getName())) { 817 CLog.d( 818 "%s was skipped via %s", 819 checker.getClass().getName(), SKIP_SYSTEM_STATUS_CHECKER); 820 continue; 821 } 822 823 StatusCheckerResult result = new StatusCheckerResult(CheckStatus.FAILED); 824 try { 825 result = checker.preExecutionCheck(device); 826 } catch (RuntimeException e) { 827 // Catch RuntimeException to avoid leaking throws that go to the invocation. 828 result.setErrorMessage(e.getMessage()); 829 result.setBugreportNeeded(true); 830 } 831 if (!CheckStatus.SUCCESS.equals(result.getStatus())) { 832 String errorMessage = 833 (result.getErrorMessage() == null) ? "" : result.getErrorMessage(); 834 failures.put(checker.getClass().getCanonicalName(), errorMessage); 835 bugreportNeeded = bugreportNeeded | result.isBugreportNeeded(); 836 CLog.w("System status checker [%s] failed.", checker.getClass().getCanonicalName()); 837 } 838 } 839 if (!failures.isEmpty()) { 840 CLog.w("There are failed system status checkers: %s", failures.toString()); 841 if (bugreportNeeded && !(device.getIDevice() instanceof StubDevice)) { 842 device.logBugreport( 843 String.format("bugreport-checker-pre-module-%s", moduleName), listener); 844 } 845 } 846 847 // We report System checkers like tests. 848 reportModuleCheckerResult(MODULE_CHECKER_PRE, moduleName, failures, startTime, listener); 849 } 850 851 /** 852 * Helper to run the System Status checkers postExecutionCheck defined for the test and log 853 * their failures. 854 */ runPostModuleCheck( String moduleName, List<ISystemStatusChecker> checkers, ITestDevice device, ITestInvocationListener listener)855 private void runPostModuleCheck( 856 String moduleName, 857 List<ISystemStatusChecker> checkers, 858 ITestDevice device, 859 ITestInvocationListener listener) 860 throws DeviceNotAvailableException { 861 long startTime = System.currentTimeMillis(); 862 CLog.i("Running system status checker after module execution: %s", moduleName); 863 Map<String, String> failures = new LinkedHashMap<>(); 864 boolean bugreportNeeded = false; 865 for (ISystemStatusChecker checker : checkers) { 866 // Check if the status checker should be skipped. 867 if (mSystemStatusCheckBlacklist.contains(checker.getClass().getName())) { 868 continue; 869 } 870 871 StatusCheckerResult result = new StatusCheckerResult(CheckStatus.FAILED); 872 try { 873 result = checker.postExecutionCheck(device); 874 } catch (RuntimeException e) { 875 // Catch RuntimeException to avoid leaking throws that go to the invocation. 876 result.setErrorMessage(e.getMessage()); 877 result.setBugreportNeeded(true); 878 } 879 if (!CheckStatus.SUCCESS.equals(result.getStatus())) { 880 String errorMessage = 881 (result.getErrorMessage() == null) ? "" : result.getErrorMessage(); 882 failures.put(checker.getClass().getCanonicalName(), errorMessage); 883 bugreportNeeded = bugreportNeeded | result.isBugreportNeeded(); 884 CLog.w("System status checker [%s] failed", checker.getClass().getCanonicalName()); 885 } 886 } 887 if (!failures.isEmpty()) { 888 CLog.w("There are failed system status checkers: %s", failures.toString()); 889 if (bugreportNeeded && !(device.getIDevice() instanceof StubDevice)) { 890 device.logBugreport( 891 String.format("bugreport-checker-post-module-%s", moduleName), listener); 892 } 893 } 894 895 // We report System checkers like tests. 896 reportModuleCheckerResult(MODULE_CHECKER_POST, moduleName, failures, startTime, listener); 897 } 898 899 /** Helper to report status checker results as test results. */ reportModuleCheckerResult( String identifier, String moduleName, Map<String, String> failures, long startTime, ITestInvocationListener listener)900 private void reportModuleCheckerResult( 901 String identifier, 902 String moduleName, 903 Map<String, String> failures, 904 long startTime, 905 ITestInvocationListener listener) { 906 if (!mReportSystemChecker) { 907 // do not log here, otherwise it could be very verbose. 908 return; 909 } 910 // Avoid messing with the final test count by making them empty runs. 911 listener.testRunStarted(identifier + "_" + moduleName, 0, 0, System.currentTimeMillis()); 912 if (!failures.isEmpty()) { 913 listener.testRunFailed(String.format("%s failed '%s' checkers", moduleName, failures)); 914 } 915 listener.testRunEnded( 916 System.currentTimeMillis() - startTime, new HashMap<String, Metric>()); 917 } 918 919 /** Returns true if we are currently in {@link #split(int)}. */ isSplitting()920 public boolean isSplitting() { 921 return mIsSplitting; 922 } 923 924 /** {@inheritDoc} */ 925 @Override split(Integer shardCountHint, TestInformation testInfo)926 public Collection<IRemoteTest> split(Integer shardCountHint, TestInformation testInfo) { 927 if (shardCountHint == null || shardCountHint <= 1 || mIsSharded) { 928 // cannot shard or already sharded 929 return null; 930 } 931 // TODO: Replace by relying on testInfo directly 932 setBuild(testInfo.getBuildInfo()); 933 setDevice(testInfo.getDevice()); 934 setInvocationContext(testInfo.getContext()); 935 936 mIsSplitting = true; 937 try { 938 LinkedHashMap<String, IConfiguration> runConfig = loadAndFilter(); 939 if (runConfig.isEmpty()) { 940 CLog.i("No config were loaded. Nothing to run."); 941 return null; 942 } 943 injectInfo(runConfig, testInfo); 944 945 // We split individual tests on double the shardCountHint to provide better average. 946 // The test pool mechanism prevent this from creating too much overhead. 947 List<ModuleDefinition> splitModules = 948 ModuleSplitter.splitConfiguration( 949 testInfo, 950 runConfig, 951 shardCountHint, 952 mShouldMakeDynamicModule, 953 mIntraModuleSharding); 954 runConfig.clear(); 955 runConfig = null; 956 957 // Clean up the parent that will get sharded: It is fine to clean up before copying the 958 // options, because the sharded module is already created/populated so there is no need 959 // to carry these extra data. 960 cleanUpSuiteSetup(); 961 962 // create an association of one ITestSuite <=> one ModuleDefinition as the smallest 963 // execution unit supported. 964 List<IRemoteTest> splitTests = new ArrayList<>(); 965 for (ModuleDefinition m : splitModules) { 966 ITestSuite suite = createInstance(); 967 OptionCopier.copyOptionsNoThrow(this, suite); 968 suite.mIsSharded = true; 969 suite.mDirectModule = m; 970 splitTests.add(suite); 971 } 972 // return the list of ITestSuite with their ModuleDefinition assigned 973 return splitTests; 974 } finally { 975 // Done splitting at that point 976 mIsSplitting = false; 977 } 978 } 979 980 /** 981 * Inject {@link ITestDevice} and {@link IBuildInfo} to the {@link IRemoteTest}s in the config 982 * before sharding since they may be needed. 983 */ injectInfo( LinkedHashMap<String, IConfiguration> runConfig, TestInformation testInfo)984 private void injectInfo( 985 LinkedHashMap<String, IConfiguration> runConfig, TestInformation testInfo) { 986 for (IConfiguration config : runConfig.values()) { 987 for (IRemoteTest test : config.getTests()) { 988 if (test instanceof IBuildReceiver) { 989 ((IBuildReceiver) test).setBuild(testInfo.getBuildInfo()); 990 } 991 if (test instanceof IDeviceTest) { 992 ((IDeviceTest) test).setDevice(testInfo.getDevice()); 993 } 994 if (test instanceof IInvocationContextReceiver) { 995 ((IInvocationContextReceiver) test).setInvocationContext(testInfo.getContext()); 996 } 997 if (test instanceof ITestCollector) { 998 ((ITestCollector) test).setCollectTestsOnly(mCollectTestsOnly); 999 } 1000 } 1001 } 1002 } 1003 1004 /** {@inheritDoc} */ 1005 @Override setDevice(ITestDevice device)1006 public void setDevice(ITestDevice device) { 1007 mDevice = device; 1008 } 1009 1010 /** 1011 * {@inheritDoc} 1012 */ 1013 @Override getDevice()1014 public ITestDevice getDevice() { 1015 return mDevice; 1016 } 1017 1018 /** Set the value of mAbiName */ setAbiName(String abiName)1019 public void setAbiName(String abiName) { 1020 mAbiName = abiName; 1021 } 1022 1023 /** 1024 * {@inheritDoc} 1025 */ 1026 @Override setBuild(IBuildInfo buildInfo)1027 public void setBuild(IBuildInfo buildInfo) { 1028 mBuildInfo = buildInfo; 1029 } 1030 1031 /** 1032 * Implementation of {@link ITestSuite} may require the build info to load the tests. 1033 */ getBuildInfo()1034 public IBuildInfo getBuildInfo() { 1035 return mBuildInfo; 1036 } 1037 1038 /** Set the value of mPrimaryAbiRun */ setPrimaryAbiRun(boolean primaryAbiRun)1039 public void setPrimaryAbiRun(boolean primaryAbiRun) { 1040 mPrimaryAbiRun = primaryAbiRun; 1041 } 1042 1043 /** 1044 * {@inheritDoc} 1045 */ 1046 @Override setSystemStatusChecker(List<ISystemStatusChecker> systemCheckers)1047 public void setSystemStatusChecker(List<ISystemStatusChecker> systemCheckers) { 1048 mSystemStatusCheckers = systemCheckers; 1049 } 1050 1051 /** 1052 * Run the test suite in collector only mode, this requires all the sub-tests to implements this 1053 * interface too. 1054 */ 1055 @Override setCollectTestsOnly(boolean shouldCollectTest)1056 public void setCollectTestsOnly(boolean shouldCollectTest) { 1057 mCollectTestsOnly = shouldCollectTest; 1058 } 1059 1060 /** {@inheritDoc} */ 1061 @Override setMetricCollectors(List<IMetricCollector> collectors)1062 public void setMetricCollectors(List<IMetricCollector> collectors) { 1063 mMetricCollectors = collectors; 1064 } 1065 1066 /** 1067 * When doing distributed sharding, we cannot have ModuleDefinition that shares tests in a pool 1068 * otherwise intra-module sharding will not work, so we allow to disable it. 1069 */ setShouldMakeDynamicModule(boolean dynamicModule)1070 public void setShouldMakeDynamicModule(boolean dynamicModule) { 1071 mShouldMakeDynamicModule = dynamicModule; 1072 } 1073 1074 /** {@inheritDoc} */ 1075 @Override setInvocationContext(IInvocationContext invocationContext)1076 public void setInvocationContext(IInvocationContext invocationContext) { 1077 mContext = invocationContext; 1078 } 1079 1080 /** 1081 * Returns the invocation context. 1082 */ getInvocationContext()1083 public IInvocationContext getInvocationContext() { 1084 return mContext; 1085 } 1086 1087 /** {@inheritDoc} */ 1088 @Override setTestLogger(ITestLogger testLogger)1089 public void setTestLogger(ITestLogger testLogger) { 1090 mCurrentLogger = testLogger; 1091 } 1092 getCurrentTestLogger()1093 public ITestLogger getCurrentTestLogger() { 1094 return mCurrentLogger; 1095 } 1096 1097 /** {@inheritDoc} */ 1098 @Override getRuntimeHint()1099 public long getRuntimeHint() { 1100 if (mDirectModule != null) { 1101 CLog.d( 1102 " %s: %s", 1103 mDirectModule.getId(), 1104 TimeUtil.formatElapsedTime(mDirectModule.getRuntimeHint())); 1105 return mDirectModule.getRuntimeHint(); 1106 } 1107 return 0l; 1108 } 1109 1110 /** {@inheritDoc} */ 1111 @Override setConfiguration(IConfiguration configuration)1112 public void setConfiguration(IConfiguration configuration) { 1113 mMainConfiguration = configuration; 1114 } 1115 1116 /** Returns the invocation {@link IConfiguration}. */ getConfiguration()1117 public final IConfiguration getConfiguration() { 1118 return mMainConfiguration; 1119 } 1120 1121 /** {@inheritDoc} */ 1122 @Override reportNotExecuted(ITestInvocationListener listener)1123 public void reportNotExecuted(ITestInvocationListener listener) { 1124 reportNotExecuted(listener, IReportNotExecuted.NOT_EXECUTED_FAILURE); 1125 } 1126 1127 /** {@inheritDoc} */ 1128 @Override reportNotExecuted(ITestInvocationListener listener, String message)1129 public void reportNotExecuted(ITestInvocationListener listener, String message) { 1130 // If the runner is already in progress, report the remaining tests as not executed. 1131 List<ModuleDefinition> runModules = null; 1132 if (mRunModules != null) { 1133 runModules = new ArrayList<>(mRunModules); 1134 } 1135 if (runModules == null) { 1136 runModules = createExecutionList(); 1137 } 1138 1139 if (mModuleInProgress != null) { 1140 // TODO: Ensure in-progress data make sense 1141 String inProgressMessage = 1142 String.format( 1143 "Module %s was interrupted after starting. Results might not be " 1144 + "accurate or complete.", 1145 mModuleInProgress.getId()); 1146 mModuleInProgress.reportNotExecuted(listener, inProgressMessage); 1147 } 1148 1149 while (!runModules.isEmpty()) { 1150 ModuleDefinition module = runModules.remove(0); 1151 module.reportNotExecuted(listener, message); 1152 } 1153 } 1154 addModuleMetadataIncludeFilters(MultiMap<String, String> filters)1155 public void addModuleMetadataIncludeFilters(MultiMap<String, String> filters) { 1156 mModuleMetadataIncludeFilter.putAll(filters); 1157 } 1158 addModuleMetadataExcludeFilters(MultiMap<String, String> filters)1159 public void addModuleMetadataExcludeFilters(MultiMap<String, String> filters) { 1160 mModuleMetadataExcludeFilter.putAll(filters); 1161 } 1162 1163 /** 1164 * Returns the {@link ModuleDefinition} to be executed directly, or null if none yet (when the 1165 * ITestSuite has not been sharded yet). 1166 */ getDirectModule()1167 public ModuleDefinition getDirectModule() { 1168 return mDirectModule; 1169 } 1170 1171 @Override getRequiredTokens()1172 public Set<TokenProperty> getRequiredTokens() { 1173 if (mDirectModule == null) { 1174 return null; 1175 } 1176 return mDirectModule.getRequiredTokens(); 1177 } 1178 1179 /** 1180 * Gets the set of ABIs supported by both Compatibility testing {@link 1181 * AbiUtils#getAbisSupportedByCompatibility()} and the device under test. 1182 * 1183 * @return The set of ABIs to run the tests on 1184 * @throws DeviceNotAvailableException 1185 */ getAbis(ITestDevice device)1186 public Set<IAbi> getAbis(ITestDevice device) throws DeviceNotAvailableException { 1187 Set<IAbi> abis = new LinkedHashSet<>(); 1188 Set<String> archAbis = getAbisForBuildTargetArch(); 1189 // Handle null-device: use abi in common with host and suite build 1190 if (mPrimaryAbiRun) { 1191 if (mAbiName == null) { 1192 // Get the primary from the device and make it the --abi to run. 1193 mAbiName = getPrimaryAbi(device); 1194 } else { 1195 CLog.d( 1196 "Option --%s supersedes the option --%s, using abi: %s", 1197 ABI_OPTION, PRIMARY_ABI_RUN, mAbiName); 1198 } 1199 } 1200 if (mAbiName != null) { 1201 // A particular abi was requested, it still needs to be supported by the build. 1202 if ((!mSkipHostArchCheck && !archAbis.contains(mAbiName)) 1203 || !AbiUtils.isAbiSupportedByCompatibility(mAbiName)) { 1204 throw new IllegalArgumentException( 1205 String.format( 1206 "Your tests suite hasn't been built with " 1207 + "abi '%s' support, this suite currently supports '%s'.", 1208 mAbiName, archAbis)); 1209 } else { 1210 abis.add(new Abi(mAbiName, AbiUtils.getBitness(mAbiName))); 1211 return abis; 1212 } 1213 } else { 1214 // Run on all abi in common between the device and suite builds. 1215 List<String> deviceAbis = getDeviceAbis(device); 1216 if (deviceAbis.isEmpty()) { 1217 throw new IllegalArgumentException( 1218 String.format( 1219 "Couldn't determinate the abi of the device '%s'.", 1220 device.getSerialNumber())); 1221 } 1222 for (String abi : deviceAbis) { 1223 if ((mSkipHostArchCheck || archAbis.contains(abi)) 1224 && AbiUtils.isAbiSupportedByCompatibility(abi)) { 1225 abis.add(new Abi(abi, AbiUtils.getBitness(abi))); 1226 } else { 1227 CLog.d( 1228 "abi '%s' is supported by device but not by this suite build (%s), " 1229 + "tests will not run against it.", 1230 abi, archAbis); 1231 } 1232 } 1233 if (abis.isEmpty()) { 1234 throw new IllegalArgumentException( 1235 String.format( 1236 "None of the abi supported by this tests suite build ('%s') are " 1237 + "supported by the device ('%s').", 1238 archAbis, deviceAbis)); 1239 } 1240 return abis; 1241 } 1242 } 1243 1244 /** Returns the primary abi of the device or host if it's a null device. */ getPrimaryAbi(ITestDevice device)1245 private String getPrimaryAbi(ITestDevice device) throws DeviceNotAvailableException { 1246 if (device.getIDevice() instanceof NullDevice) { 1247 Set<String> hostAbis = getHostAbis(); 1248 return hostAbis.iterator().next(); 1249 } 1250 String property = device.getProperty(PRODUCT_CPU_ABI_KEY); 1251 if (property == null) { 1252 String serial = device.getSerialNumber(); 1253 throw new DeviceNotAvailableException( 1254 String.format( 1255 "Device '%s' was not online to query %s", serial, PRODUCT_CPU_ABI_KEY), 1256 serial); 1257 } 1258 return property.trim(); 1259 } 1260 1261 /** Returns the list of abis supported by the device or host if it's a null device. */ getDeviceAbis(ITestDevice device)1262 private List<String> getDeviceAbis(ITestDevice device) throws DeviceNotAvailableException { 1263 if (device.getIDevice() instanceof NullDevice) { 1264 return new ArrayList<>(getHostAbis()); 1265 } 1266 // Make it an arrayList to be able to modify the content. 1267 return new ArrayList<>(Arrays.asList(AbiFormatter.getSupportedAbis(device, ""))); 1268 } 1269 1270 /** Return the abis supported by the Host build target architecture. Exposed for testing. */ 1271 @VisibleForTesting getAbisForBuildTargetArch()1272 protected Set<String> getAbisForBuildTargetArch() { 1273 // If TestSuiteInfo does not exists, the stub arch will be replaced by all possible abis. 1274 Set<String> abis = new LinkedHashSet<>(); 1275 for (String arch : TestSuiteInfo.getInstance().getTargetArchs()) { 1276 abis.addAll(AbiUtils.getAbisForArch(arch)); 1277 } 1278 return abis; 1279 } 1280 1281 /** Returns the host machine abis. */ 1282 @VisibleForTesting getHostAbis()1283 protected Set<String> getHostAbis() { 1284 return AbiUtils.getHostAbi(); 1285 } 1286 1287 /** Returns the abi requested with the option -a or --abi. */ getRequestedAbi()1288 public final String getRequestedAbi() { 1289 return mAbiName; 1290 } 1291 1292 /** Getter used to validate the proper Guice injection. */ 1293 @VisibleForTesting getInjector()1294 final Injector getInjector() { 1295 return mInjector; 1296 } 1297 1298 /** Sets reboot-before-test to true. */ enableRebootBeforeTest()1299 public final void enableRebootBeforeTest() { 1300 mRebootBeforeTest = true; 1301 } 1302 1303 /** 1304 * Apply the metadata filter to the config and see if the config should run. 1305 * 1306 * @param config The {@link IConfiguration} being evaluated. 1307 * @param include the metadata include filter 1308 * @param exclude the metadata exclude filter 1309 * @return True if the module should run, false otherwise. 1310 */ 1311 @VisibleForTesting filterByConfigMetadata( IConfiguration config, MultiMap<String, String> include, MultiMap<String, String> exclude)1312 protected boolean filterByConfigMetadata( 1313 IConfiguration config, 1314 MultiMap<String, String> include, 1315 MultiMap<String, String> exclude) { 1316 MultiMap<String, String> metadata = config.getConfigurationDescription().getAllMetaData(); 1317 boolean shouldInclude = false; 1318 for (String key : include.keySet()) { 1319 Set<String> filters = new HashSet<>(include.get(key)); 1320 if (metadata.containsKey(key)) { 1321 filters.retainAll(metadata.get(key)); 1322 if (!filters.isEmpty()) { 1323 // inclusion filter is not empty and there's at least one matching inclusion 1324 // rule so there's no need to match other inclusion rules 1325 shouldInclude = true; 1326 break; 1327 } 1328 } 1329 } 1330 if (!include.isEmpty() && !shouldInclude) { 1331 // if inclusion filter is not empty and we didn't find a match, the module will not be 1332 // included 1333 return false; 1334 } 1335 // Now evaluate exclusion rules, this ordering also means that exclusion rules may override 1336 // inclusion rules: a config already matched for inclusion may still be excluded if matching 1337 // rules exist 1338 for (String key : exclude.keySet()) { 1339 Set<String> filters = new HashSet<>(exclude.get(key)); 1340 if (metadata.containsKey(key)) { 1341 filters.retainAll(metadata.get(key)); 1342 if (!filters.isEmpty()) { 1343 // we found at least one matching exclusion rules, so we are excluding this 1344 // this module 1345 return false; 1346 } 1347 } 1348 } 1349 // we've matched at least one inclusion rule (if there's any) AND we didn't match any of the 1350 // exclusion rules (if there's any) 1351 return true; 1352 } 1353 1354 /** 1355 * Filter out the preparers that were not whitelisted. This is useful for collect-tests-only 1356 * where some preparers are not needed to dry run through the invocation. 1357 * 1358 * @param config the {@link IConfiguration} considered for filtering. 1359 * @param preparerWhiteList the current preparer whitelist. 1360 */ 1361 @VisibleForTesting filterPreparers(IConfiguration config, Set<String> preparerWhiteList)1362 void filterPreparers(IConfiguration config, Set<String> preparerWhiteList) { 1363 // If no filters was provided, skip the filtering. 1364 if (preparerWhiteList.isEmpty()) { 1365 return; 1366 } 1367 for (IDeviceConfiguration deviceConfig : config.getDeviceConfig()) { 1368 List<ITargetPreparer> preparers = new ArrayList<>(deviceConfig.getTargetPreparers()); 1369 for (ITargetPreparer prep : preparers) { 1370 if (!preparerWhiteList.contains(prep.getClass().getName())) { 1371 deviceConfig.getTargetPreparers().remove(prep); 1372 } 1373 } 1374 } 1375 } 1376 1377 /** 1378 * Apply the Runner whitelist filtering, removing any runner that was not whitelisted. If a 1379 * configuration has several runners, some might be removed and the config will still run. 1380 * 1381 * @param config The {@link IConfiguration} being evaluated. 1382 * @param allowedRunners The current runner whitelist. 1383 * @return True if the configuration module is allowed to run, false otherwise. 1384 */ 1385 @VisibleForTesting filterByRunnerType(IConfiguration config, Set<String> allowedRunners)1386 protected boolean filterByRunnerType(IConfiguration config, Set<String> allowedRunners) { 1387 // If no filters are provided, simply run everything. 1388 if (allowedRunners.isEmpty()) { 1389 return true; 1390 } 1391 Iterator<IRemoteTest> iterator = config.getTests().iterator(); 1392 while (iterator.hasNext()) { 1393 IRemoteTest test = iterator.next(); 1394 if (!allowedRunners.contains(test.getClass().getName())) { 1395 CLog.d( 1396 "Runner '%s' in module '%s' was skipped by the runner whitelist: '%s'.", 1397 test.getClass().getName(), config.getName(), allowedRunners); 1398 iterator.remove(); 1399 } 1400 } 1401 1402 if (config.getTests().isEmpty()) { 1403 CLog.d("Module %s does not have any more tests, skipping it.", config.getName()); 1404 return false; 1405 } 1406 return true; 1407 } 1408 disableAutoRetryTimeReporting()1409 void disableAutoRetryTimeReporting() { 1410 mDisableAutoRetryTimeReporting = true; 1411 } 1412 1413 @VisibleForTesting setModuleInProgress(ModuleDefinition moduleInProgress)1414 void setModuleInProgress(ModuleDefinition moduleInProgress) { 1415 mModuleInProgress = moduleInProgress; 1416 } 1417 } 1418