1 /* 2 * Copyright (C) 2016 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package com.android.tradefed.testtype.suite; 17 18 import com.android.ddmlib.Log.LogLevel; 19 import com.android.tradefed.build.BuildRetrievalError; 20 import com.android.tradefed.build.IBuildInfo; 21 import com.android.tradefed.config.Configuration; 22 import com.android.tradefed.config.ConfigurationDescriptor; 23 import com.android.tradefed.config.ConfigurationException; 24 import com.android.tradefed.config.DeviceConfigurationHolder; 25 import com.android.tradefed.config.DynamicRemoteFileResolver; 26 import com.android.tradefed.config.IConfiguration; 27 import com.android.tradefed.config.IConfigurationReceiver; 28 import com.android.tradefed.device.DeviceNotAvailableException; 29 import com.android.tradefed.device.ITestDevice; 30 import com.android.tradefed.device.ITestDevice.RecoveryMode; 31 import com.android.tradefed.device.StubDevice; 32 import com.android.tradefed.device.metric.IMetricCollector; 33 import com.android.tradefed.device.metric.LogcatOnFailureCollector; 34 import com.android.tradefed.device.metric.ScreenshotOnFailureCollector; 35 import com.android.tradefed.error.IHarnessException; 36 import com.android.tradefed.invoker.IInvocationContext; 37 import com.android.tradefed.invoker.InvocationContext; 38 import com.android.tradefed.invoker.TestInformation; 39 import com.android.tradefed.invoker.logger.CurrentInvocation; 40 import com.android.tradefed.invoker.logger.InvocationMetricLogger; 41 import com.android.tradefed.invoker.logger.InvocationMetricLogger.InvocationMetricKey; 42 import com.android.tradefed.invoker.logger.TfObjectTracker; 43 import com.android.tradefed.invoker.shard.token.TokenProperty; 44 import com.android.tradefed.log.ILogRegistry.EventType; 45 import com.android.tradefed.log.ITestLogger; 46 import com.android.tradefed.log.LogRegistry; 47 import com.android.tradefed.log.LogUtil.CLog; 48 import com.android.tradefed.metrics.proto.MetricMeasurement.Metric; 49 import com.android.tradefed.result.FailureDescription; 50 import com.android.tradefed.result.ILogSaver; 51 import com.android.tradefed.result.ILogSaverListener; 52 import com.android.tradefed.result.ITestInvocationListener; 53 import com.android.tradefed.result.ITestLoggerReceiver; 54 import com.android.tradefed.result.LogFile; 55 import com.android.tradefed.result.MultiFailureDescription; 56 import com.android.tradefed.result.ResultForwarder; 57 import com.android.tradefed.result.TestDescription; 58 import com.android.tradefed.result.TestResult; 59 import com.android.tradefed.result.TestRunResult; 60 import com.android.tradefed.result.error.ErrorIdentifier; 61 import com.android.tradefed.result.error.InfraErrorIdentifier; 62 import com.android.tradefed.result.proto.TestRecordProto.FailureStatus; 63 import com.android.tradefed.retry.IRetryDecision; 64 import com.android.tradefed.retry.RetryStatistics; 65 import com.android.tradefed.suite.checker.ISystemStatusCheckerReceiver; 66 import com.android.tradefed.targetprep.BuildError; 67 import com.android.tradefed.targetprep.ITargetPreparer; 68 import com.android.tradefed.targetprep.TargetSetupError; 69 import com.android.tradefed.targetprep.multi.IMultiTargetPreparer; 70 import com.android.tradefed.testtype.IBuildReceiver; 71 import com.android.tradefed.testtype.IDeviceTest; 72 import com.android.tradefed.testtype.IInvocationContextReceiver; 73 import com.android.tradefed.testtype.IRemoteTest; 74 import com.android.tradefed.testtype.IRuntimeHintProvider; 75 import com.android.tradefed.testtype.ITestCollector; 76 import com.android.tradefed.testtype.suite.module.BaseModuleController; 77 import com.android.tradefed.testtype.suite.module.IModuleController.RunStrategy; 78 import com.android.tradefed.util.MultiMap; 79 import com.android.tradefed.util.StreamUtil; 80 import com.android.tradefed.util.proto.TfMetricProtoUtil; 81 82 import com.google.common.annotations.VisibleForTesting; 83 84 import java.util.ArrayList; 85 import java.util.Collection; 86 import java.util.Collections; 87 import java.util.HashMap; 88 import java.util.HashSet; 89 import java.util.List; 90 import java.util.ListIterator; 91 import java.util.Map; 92 import java.util.Map.Entry; 93 import java.util.Set; 94 95 /** 96 * Container for the test run configuration. This class is an helper to prepare and run the tests. 97 */ 98 public class ModuleDefinition implements Comparable<ModuleDefinition>, ITestCollector { 99 100 /** key names used for saving module info into {@link IInvocationContext} */ 101 /** 102 * Module name is the base name associated with the module, usually coming from the Xml TF 103 * config file the module was loaded from. 104 */ 105 public static final String MODULE_NAME = "module-name"; 106 public static final String MODULE_ABI = "module-abi"; 107 public static final String MODULE_PARAMETERIZATION = "module-param"; 108 /** 109 * Module ID the name that will be used to identify uniquely the module during testRunStart. It 110 * will usually be a combination of MODULE_ABI + MODULE_NAME. 111 */ 112 public static final String MODULE_ID = "module-id"; 113 114 public static final String MODULE_CONTROLLER = "module_controller"; 115 116 public static final String PREPARATION_TIME = "PREP_TIME"; 117 public static final String TEAR_DOWN_TIME = "TEARDOWN_TIME"; 118 public static final String TEST_TIME = "TEST_TIME"; 119 public static final String RETRY_TIME = "MODULE_RETRY_TIME"; 120 public static final String RETRY_SUCCESS_COUNT = "MODULE_RETRY_SUCCESS"; 121 public static final String RETRY_FAIL_COUNT = "MODULE_RETRY_FAILED"; 122 123 private final IInvocationContext mModuleInvocationContext; 124 private final IConfiguration mModuleConfiguration; 125 private IConfiguration mInternalTestConfiguration; 126 private IConfiguration mInternalTargetPreparerConfiguration; 127 private ILogSaver mLogSaver; 128 129 private final String mId; 130 private Collection<IRemoteTest> mTests = null; 131 private Map<String, List<ITargetPreparer>> mPreparersPerDevice = null; 132 133 private List<IMultiTargetPreparer> mMultiPreparers = new ArrayList<>(); 134 private IBuildInfo mBuild; 135 private ITestDevice mDevice; 136 private List<IMetricCollector> mRunMetricCollectors = new ArrayList<>(); 137 private boolean mCollectTestsOnly = false; 138 139 private List<TestRunResult> mTestsResults = new ArrayList<>(); 140 private List<ModuleListener> mRunListenersResults = new ArrayList<>(); 141 private int mExpectedTests = 0; 142 private boolean mIsFailedModule = false; 143 144 // Tracking of preparers performance 145 private long mElapsedPreparation = 0l; 146 private long mElapsedTearDown = 0l; 147 148 private long mStartTestTime = 0l; 149 private Long mStartModuleRunDate = null; 150 151 // Tracking of retry performance 152 private List<RetryStatistics> mRetryStats = new ArrayList<>(); 153 private boolean mDisableAutoRetryTimeReporting = false; 154 155 private boolean mMergeAttempts = true; 156 private IRetryDecision mRetryDecision; 157 158 // Token during sharding 159 private Set<TokenProperty> mRequiredTokens = new HashSet<>(); 160 161 private boolean mEnableDynamicDownload = false; 162 163 /** 164 * Constructor 165 * 166 * @param name unique name of the test configuration. 167 * @param tests list of {@link IRemoteTest} that needs to run. 168 * @param preparersPerDevice list of {@link ITargetPreparer} to be used to setup the device. 169 * @param moduleConfig the {@link IConfiguration} of the underlying module config. 170 */ ModuleDefinition( String name, Collection<IRemoteTest> tests, Map<String, List<ITargetPreparer>> preparersPerDevice, List<IMultiTargetPreparer> multiPreparers, IConfiguration moduleConfig)171 public ModuleDefinition( 172 String name, 173 Collection<IRemoteTest> tests, 174 Map<String, List<ITargetPreparer>> preparersPerDevice, 175 List<IMultiTargetPreparer> multiPreparers, 176 IConfiguration moduleConfig) { 177 mId = name; 178 mTests = tests; 179 mModuleConfiguration = moduleConfig; 180 ConfigurationDescriptor configDescriptor = moduleConfig.getConfigurationDescription(); 181 mModuleInvocationContext = new InvocationContext(); 182 mModuleInvocationContext.setConfigurationDescriptor(configDescriptor.clone()); 183 184 // If available in the suite, add the abi name 185 if (configDescriptor.getAbi() != null) { 186 mModuleInvocationContext.addInvocationAttribute( 187 MODULE_ABI, configDescriptor.getAbi().getName()); 188 } 189 if (configDescriptor.getModuleName() != null) { 190 mModuleInvocationContext.addInvocationAttribute( 191 MODULE_NAME, configDescriptor.getModuleName()); 192 } 193 String parameterization = 194 configDescriptor 195 .getAllMetaData() 196 .getUniqueMap() 197 .get(ConfigurationDescriptor.ACTIVE_PARAMETER_KEY); 198 if (parameterization != null) { 199 mModuleInvocationContext.addInvocationAttribute( 200 MODULE_PARAMETERIZATION, parameterization); 201 } 202 // If there is no specific abi, module-id should be module-name 203 mModuleInvocationContext.addInvocationAttribute(MODULE_ID, mId); 204 205 mMultiPreparers.addAll(multiPreparers); 206 mPreparersPerDevice = preparersPerDevice; 207 208 // Get the tokens of the module 209 List<String> tokens = configDescriptor.getMetaData(ITestSuite.TOKEN_KEY); 210 if (tokens != null) { 211 for (String token : tokens) { 212 mRequiredTokens.add(TokenProperty.valueOf(token.toUpperCase())); 213 } 214 } 215 } 216 217 /** 218 * Returns the next {@link IRemoteTest} from the list of tests. The list of tests of a module 219 * may be shared with another one in case of sharding. 220 */ poll()221 IRemoteTest poll() { 222 synchronized (mTests) { 223 if (mTests.isEmpty()) { 224 return null; 225 } 226 IRemoteTest test = mTests.iterator().next(); 227 mTests.remove(test); 228 return test; 229 } 230 } 231 232 /** 233 * Add some {@link IRemoteTest} to be executed as part of the module. Used when merging two 234 * modules. 235 */ addTests(List<IRemoteTest> test)236 void addTests(List<IRemoteTest> test) { 237 synchronized (mTests) { 238 mTests.addAll(test); 239 } 240 } 241 242 /** Returns the current number of {@link IRemoteTest} waiting to be executed. */ numTests()243 public int numTests() { 244 synchronized (mTests) { 245 return mTests.size(); 246 } 247 } 248 249 /** 250 * Return True if the Module still has {@link IRemoteTest} to run in its pool. False otherwise. 251 */ hasTests()252 protected boolean hasTests() { 253 synchronized (mTests) { 254 return mTests.isEmpty(); 255 } 256 } 257 258 /** Return the unique module name. */ getId()259 public String getId() { 260 return mId; 261 } 262 263 /** 264 * {@inheritDoc} 265 */ 266 @Override compareTo(ModuleDefinition moduleDef)267 public int compareTo(ModuleDefinition moduleDef) { 268 return getId().compareTo(moduleDef.getId()); 269 } 270 271 /** 272 * Inject the {@link IBuildInfo} to be used during the tests. 273 */ setBuild(IBuildInfo build)274 public void setBuild(IBuildInfo build) { 275 mBuild = build; 276 } 277 278 /** 279 * Inject the {@link ITestDevice} to be used during the tests. 280 */ setDevice(ITestDevice device)281 public void setDevice(ITestDevice device) { 282 mDevice = device; 283 } 284 285 /** Inject the List of {@link IMetricCollector} to be used by the module. */ setMetricCollectors(List<IMetricCollector> collectors)286 public void setMetricCollectors(List<IMetricCollector> collectors) { 287 if (collectors == null) { 288 return; 289 } 290 mRunMetricCollectors.addAll(collectors); 291 } 292 293 /** Pass the invocation log saver to the module so it can use it if necessary. */ setLogSaver(ILogSaver logSaver)294 public void setLogSaver(ILogSaver logSaver) { 295 mLogSaver = logSaver; 296 } 297 298 /** 299 * Run all the {@link IRemoteTest} contained in the module and use all the preparers before and 300 * after to setup and clean the device. 301 * 302 * @param listener the {@link ITestInvocationListener} where to report results. 303 * @throws DeviceNotAvailableException in case of device going offline. 304 */ run(TestInformation moduleInfo, ITestInvocationListener listener)305 public final void run(TestInformation moduleInfo, ITestInvocationListener listener) 306 throws DeviceNotAvailableException { 307 run(moduleInfo, listener, null, null); 308 } 309 310 /** 311 * Run all the {@link IRemoteTest} contained in the module and use all the preparers before and 312 * after to setup and clean the device. 313 * 314 * @param listener the {@link ITestInvocationListener} where to report results. 315 * @param moduleLevelListeners The list of listeners at the module level. 316 * @param failureListener a particular listener to collect logs on testFail. Can be null. 317 * @throws DeviceNotAvailableException in case of device going offline. 318 */ run( TestInformation moduleInfo, ITestInvocationListener listener, List<ITestInvocationListener> moduleLevelListeners, TestFailureListener failureListener)319 public final void run( 320 TestInformation moduleInfo, 321 ITestInvocationListener listener, 322 List<ITestInvocationListener> moduleLevelListeners, 323 TestFailureListener failureListener) 324 throws DeviceNotAvailableException { 325 run(moduleInfo, listener, moduleLevelListeners, failureListener, 1); 326 } 327 328 /** 329 * Run all the {@link IRemoteTest} contained in the module and use all the preparers before and 330 * after to setup and clean the device. 331 * 332 * @param moduleInfo the {@link TestInformation} for the module. 333 * @param listener the {@link ITestInvocationListener} where to report results. 334 * @param moduleLevelListeners The list of listeners at the module level. 335 * @param failureListener a particular listener to collect logs on testFail. Can be null. 336 * @param maxRunLimit the max number of runs for each testcase. 337 * @throws DeviceNotAvailableException in case of device going offline. 338 */ run( TestInformation moduleInfo, ITestInvocationListener listener, List<ITestInvocationListener> moduleLevelListeners, TestFailureListener failureListener, int maxRunLimit)339 public final void run( 340 TestInformation moduleInfo, 341 ITestInvocationListener listener, 342 List<ITestInvocationListener> moduleLevelListeners, 343 TestFailureListener failureListener, 344 int maxRunLimit) 345 throws DeviceNotAvailableException { 346 mStartModuleRunDate = System.currentTimeMillis(); 347 // Load extra configuration for the module from module_controller 348 // TODO: make module_controller a full TF object 349 boolean skipTestCases = false; 350 RunStrategy rs = applyConfigurationControl(failureListener); 351 if (RunStrategy.FULL_MODULE_BYPASS.equals(rs)) { 352 CLog.d("module_controller applied and module %s should not run.", getId()); 353 return; 354 } else if (RunStrategy.SKIP_MODULE_TESTCASES.equals(rs)) { 355 CLog.d("All tests cases for %s will be marked skipped.", getId()); 356 skipTestCases = true; 357 } 358 359 CLog.logAndDisplay(LogLevel.DEBUG, "Running module %s", getId()); 360 // Exception generated during setUp or run of the tests 361 Throwable preparationException = null; 362 DeviceNotAvailableException runException = null; 363 // Resolve dynamic files except for the IRemoteTest ones 364 preparationException = invokeRemoteDynamic(moduleInfo.getDevice(), mModuleConfiguration); 365 366 if (preparationException == null) { 367 mInternalTargetPreparerConfiguration = 368 new Configuration("tmp-download", "tmp-download"); 369 mInternalTargetPreparerConfiguration 370 .getCommandOptions() 371 .getDynamicDownloadArgs() 372 .putAll(mModuleConfiguration.getCommandOptions().getDynamicDownloadArgs()); 373 for (String device : mPreparersPerDevice.keySet()) { 374 mInternalTargetPreparerConfiguration.setDeviceConfig( 375 new DeviceConfigurationHolder(device)); 376 for (ITargetPreparer preparer : mPreparersPerDevice.get(device)) { 377 try { 378 mInternalTargetPreparerConfiguration 379 .getDeviceConfigByName(device) 380 .addSpecificConfig(preparer); 381 } catch (ConfigurationException e) { 382 // Shouldn't happen; 383 throw new RuntimeException(e); 384 } 385 } 386 } 387 mInternalTargetPreparerConfiguration.setMultiTargetPreparers(mMultiPreparers); 388 preparationException = 389 invokeRemoteDynamic( 390 moduleInfo.getDevice(), mInternalTargetPreparerConfiguration); 391 } 392 // Setup 393 long prepStartTime = getCurrentTime(); 394 if (preparationException == null) { 395 preparationException = runTargetPreparation(moduleInfo, listener); 396 } 397 // Skip multi-preparation if preparation already failed. 398 if (preparationException == null) { 399 for (IMultiTargetPreparer multiPreparer : mMultiPreparers) { 400 preparationException = runMultiPreparerSetup(multiPreparer, moduleInfo, listener); 401 if (preparationException != null) { 402 mIsFailedModule = true; 403 CLog.e("Some preparation step failed. failing the module %s", getId()); 404 break; 405 } 406 } 407 } 408 mElapsedPreparation = getCurrentTime() - prepStartTime; 409 // Run the tests 410 try { 411 if (preparationException != null) { 412 reportSetupFailure(preparationException, listener, moduleLevelListeners); 413 return; 414 } 415 mStartTestTime = getCurrentTime(); 416 while (true) { 417 IRemoteTest test = poll(); 418 if (test == null) { 419 return; 420 } 421 TfObjectTracker.countWithParents(test.getClass()); 422 if (test instanceof IBuildReceiver) { 423 ((IBuildReceiver) test).setBuild(mBuild); 424 } 425 if (test instanceof IDeviceTest) { 426 ((IDeviceTest) test).setDevice(mDevice); 427 } 428 if (test instanceof IInvocationContextReceiver) { 429 ((IInvocationContextReceiver) test) 430 .setInvocationContext(mModuleInvocationContext); 431 } 432 mInternalTestConfiguration = new Configuration("tmp-download", "tmp-download"); 433 mInternalTestConfiguration 434 .getCommandOptions() 435 .getDynamicDownloadArgs() 436 .putAll(mModuleConfiguration.getCommandOptions().getDynamicDownloadArgs()); 437 // We do it before the official set, otherwise the IConfiguration will not be the 438 // right one. 439 mInternalTestConfiguration.setTest(test); 440 if (test instanceof IConfigurationReceiver) { 441 ((IConfigurationReceiver) test).setConfiguration(mModuleConfiguration); 442 } 443 if (test instanceof ISystemStatusCheckerReceiver) { 444 // We do not pass down Status checker because they are already running at the 445 // top level suite. 446 ((ISystemStatusCheckerReceiver) test).setSystemStatusChecker(new ArrayList<>()); 447 } 448 if (test instanceof ITestCollector) { 449 if (skipTestCases) { 450 mCollectTestsOnly = true; 451 } 452 ((ITestCollector) test).setCollectTestsOnly(mCollectTestsOnly); 453 } 454 GranularRetriableTestWrapper retriableTest = 455 prepareGranularRetriableWrapper( 456 test, 457 listener, 458 failureListener, 459 moduleLevelListeners, 460 skipTestCases, 461 maxRunLimit); 462 retriableTest.setCollectTestsOnly(mCollectTestsOnly); 463 // Resolve the dynamic options for that one test. 464 preparationException = 465 invokeRemoteDynamic(moduleInfo.getDevice(), mInternalTestConfiguration); 466 if (preparationException != null) { 467 reportSetupFailure(preparationException, listener, moduleLevelListeners); 468 return; 469 } 470 try { 471 retriableTest.run(moduleInfo, listener); 472 } catch (DeviceNotAvailableException dnae) { 473 runException = dnae; 474 // We do special logging of some information in Context of the module for easier 475 // debugging. 476 CLog.e( 477 "Module %s threw a DeviceNotAvailableException on device %s during " 478 + "test %s", 479 getId(), mDevice.getSerialNumber(), test.getClass()); 480 CLog.e(dnae); 481 // log an events 482 logDeviceEvent( 483 EventType.MODULE_DEVICE_NOT_AVAILABLE, 484 mDevice.getSerialNumber(), 485 dnae, 486 getId()); 487 throw dnae; 488 } finally { 489 mInternalTestConfiguration.cleanConfigurationData(); 490 mInternalTestConfiguration = null; 491 if (mMergeAttempts) { 492 // A single module can generate several test runs 493 mTestsResults.addAll(retriableTest.getFinalTestRunResults()); 494 } else { 495 // Keep track of each listener for attempts 496 mRunListenersResults.add(retriableTest.getResultListener()); 497 } 498 499 mExpectedTests += retriableTest.getExpectedTestsCount(); 500 // Get information about retry 501 if (mRetryDecision != null) { 502 RetryStatistics res = mRetryDecision.getRetryStatistics(); 503 if (res != null) { 504 addRetryTime(res.mRetryTime); 505 mRetryStats.add(res); 506 } 507 } 508 } 509 // After the run, if the test failed (even after retry the final result passed) has 510 // failed, capture a bugreport. 511 if (retriableTest.getResultListener().hasLastAttemptFailed()) { 512 captureBugreport( 513 listener, 514 getId(), 515 retriableTest 516 .getResultListener() 517 .getCurrentRunResults() 518 .getRunFailureDescription()); 519 } 520 } 521 } finally { 522 // Clean target preparers dynamic files. 523 if (mInternalTargetPreparerConfiguration != null) { 524 mInternalTargetPreparerConfiguration.cleanConfigurationData(); 525 mInternalTargetPreparerConfiguration = null; 526 } 527 long cleanStartTime = getCurrentTime(); 528 RuntimeException tearDownException = null; 529 try { 530 Throwable exception = (runException != null) ? runException : preparationException; 531 // Tear down 532 runTearDown(moduleInfo, exception); 533 } catch (DeviceNotAvailableException dnae) { 534 CLog.e( 535 "Module %s failed during tearDown with: %s", 536 getId(), StreamUtil.getStackTrace(dnae)); 537 throw dnae; 538 } catch (RuntimeException e) { 539 CLog.e("Exception while running tearDown:"); 540 CLog.e(e); 541 tearDownException = e; 542 } finally { 543 if (failureListener != null) { 544 failureListener.join(); 545 } 546 mElapsedTearDown = getCurrentTime() - cleanStartTime; 547 // finalize results 548 if (preparationException == null) { 549 mModuleConfiguration.cleanConfigurationData(); 550 if (mMergeAttempts) { 551 reportFinalResults( 552 listener, mExpectedTests, mTestsResults, null, tearDownException); 553 } else { 554 // Push the attempts one by one 555 for (int i = 0; i < maxRunLimit; i++) { 556 // Get all the results for the attempt 557 List<TestRunResult> runResultList = new ArrayList<TestRunResult>(); 558 int expectedCount = 0; 559 for (ModuleListener attemptListener : mRunListenersResults) { 560 for (String runName : attemptListener.getTestRunNames()) { 561 TestRunResult run = 562 attemptListener.getTestRunAtAttempt(runName, i); 563 if (run != null) { 564 runResultList.add(run); 565 expectedCount += run.getExpectedTestCount(); 566 } 567 } 568 } 569 570 if (!runResultList.isEmpty()) { 571 reportFinalResults( 572 listener, 573 expectedCount, 574 runResultList, 575 i, 576 tearDownException); 577 } else { 578 CLog.d("No results to be forwarded for attempt %s.", i); 579 } 580 } 581 } 582 } 583 } 584 } 585 } 586 587 /** 588 * Create a wrapper class for the {@link IRemoteTest} which has built-in logic to schedule 589 * multiple test runs for the same module, and have the ability to run testcases at a more 590 * granular level (a subset of testcases in the module). 591 * 592 * @param test the {@link IRemoteTest} that is being wrapped. 593 * @param failureListener a particular listener to collect logs on testFail. Can be null. 594 * @param skipTestCases A run strategy when SKIP_MODULE_TESTCASES is defined. 595 * @param maxRunLimit a rate-limiter on testcases retrying times. 596 */ 597 @VisibleForTesting prepareGranularRetriableWrapper( IRemoteTest test, ITestInvocationListener listener, TestFailureListener failureListener, List<ITestInvocationListener> moduleLevelListeners, boolean skipTestCases, int maxRunLimit)598 GranularRetriableTestWrapper prepareGranularRetriableWrapper( 599 IRemoteTest test, 600 ITestInvocationListener listener, 601 TestFailureListener failureListener, 602 List<ITestInvocationListener> moduleLevelListeners, 603 boolean skipTestCases, 604 int maxRunLimit) { 605 GranularRetriableTestWrapper retriableTest = 606 new GranularRetriableTestWrapper( 607 test, listener, failureListener, moduleLevelListeners, maxRunLimit); 608 retriableTest.setModuleId(getId()); 609 retriableTest.setMarkTestsSkipped(skipTestCases); 610 retriableTest.setMetricCollectors(mRunMetricCollectors); 611 retriableTest.setModuleConfig(mModuleConfiguration); 612 retriableTest.setInvocationContext(mModuleInvocationContext); 613 retriableTest.setLogSaver(mLogSaver); 614 retriableTest.setRetryDecision(mRetryDecision); 615 return retriableTest; 616 } 617 captureBugreport( ITestLogger listener, String moduleId, FailureDescription failure)618 private void captureBugreport( 619 ITestLogger listener, String moduleId, FailureDescription failure) { 620 FailureStatus status = failure.getFailureStatus(); 621 if (!FailureStatus.LOST_SYSTEM_UNDER_TEST.equals(status) 622 && !FailureStatus.SYSTEM_UNDER_TEST_CRASHED.equals(status)) { 623 return; 624 } 625 for (ITestDevice device : mModuleInvocationContext.getDevices()) { 626 if (device.getIDevice() instanceof StubDevice) { 627 continue; 628 } 629 device.logBugreport( 630 String.format( 631 "module-%s-failure-%s-bugreport", moduleId, device.getSerialNumber()), 632 listener); 633 } 634 } 635 636 /** Helper to log the device events. */ logDeviceEvent(EventType event, String serial, Throwable t, String moduleId)637 private void logDeviceEvent(EventType event, String serial, Throwable t, String moduleId) { 638 Map<String, String> args = new HashMap<>(); 639 args.put("serial", serial); 640 args.put("trace", StreamUtil.getStackTrace(t)); 641 args.put("module-id", moduleId); 642 LogRegistry.getLogRegistry().logEvent(LogLevel.DEBUG, event, args); 643 } 644 645 /** Finalize results to report them all and count if there are missing tests. */ reportFinalResults( ITestInvocationListener listener, int totalExpectedTests, List<TestRunResult> listResults, Integer attempt, RuntimeException tearDownException)646 private void reportFinalResults( 647 ITestInvocationListener listener, 648 int totalExpectedTests, 649 List<TestRunResult> listResults, 650 Integer attempt, 651 RuntimeException tearDownException) { 652 long elapsedTime = 0l; 653 HashMap<String, Metric> metricsProto = new HashMap<>(); 654 if (attempt != null) { 655 long startTime = 656 listResults.isEmpty() ? mStartTestTime : listResults.get(0).getStartTime(); 657 listener.testRunStarted(getId(), totalExpectedTests, attempt, startTime); 658 } else { 659 listener.testRunStarted(getId(), totalExpectedTests, 0, mStartTestTime); 660 } 661 int numResults = 0; 662 MultiMap<String, LogFile> aggLogFiles = new MultiMap<>(); 663 List<FailureDescription> runFailureMessages = new ArrayList<>(); 664 for (TestRunResult runResult : listResults) { 665 numResults += runResult.getTestResults().size(); 666 forwardTestResults(runResult.getTestResults(), listener); 667 if (runResult.isRunFailure()) { 668 runFailureMessages.add(runResult.getRunFailureDescription()); 669 } 670 elapsedTime += runResult.getElapsedTime(); 671 // put metrics from the tests 672 metricsProto.putAll(runResult.getRunProtoMetrics()); 673 aggLogFiles.putAll(runResult.getRunLoggedFiles()); 674 } 675 // put metrics from the preparation 676 metricsProto.put( 677 PREPARATION_TIME, 678 TfMetricProtoUtil.createSingleValue(mElapsedPreparation, "milliseconds")); 679 metricsProto.put( 680 TEAR_DOWN_TIME, 681 TfMetricProtoUtil.createSingleValue(mElapsedTearDown, "milliseconds")); 682 metricsProto.put( 683 TEST_TIME, TfMetricProtoUtil.createSingleValue(elapsedTime, "milliseconds")); 684 // Report all the retry informations 685 if (!mRetryStats.isEmpty()) { 686 RetryStatistics agg = RetryStatistics.aggregateStatistics(mRetryStats); 687 metricsProto.put( 688 RETRY_TIME, 689 TfMetricProtoUtil.createSingleValue(agg.mRetryTime, "milliseconds")); 690 metricsProto.put( 691 RETRY_SUCCESS_COUNT, 692 TfMetricProtoUtil.createSingleValue(agg.mRetrySuccess, "")); 693 metricsProto.put( 694 RETRY_FAIL_COUNT, TfMetricProtoUtil.createSingleValue(agg.mRetryFailure, "")); 695 } 696 697 // Only report the mismatch if there were no error during the run. 698 if (runFailureMessages.isEmpty() && totalExpectedTests != numResults) { 699 String error = 700 String.format( 701 "Module %s only ran %d out of %d expected tests.", 702 getId(), numResults, totalExpectedTests); 703 FailureDescription mismatch = 704 FailureDescription.create(error) 705 .setFailureStatus(FailureStatus.TEST_FAILURE) 706 .setErrorIdentifier(InfraErrorIdentifier.EXPECTED_TESTS_MISMATCH); 707 runFailureMessages.add(mismatch); 708 CLog.e(error); 709 } 710 711 if (tearDownException != null) { 712 FailureDescription failure = 713 CurrentInvocation.createFailure( 714 StreamUtil.getStackTrace(tearDownException), null) 715 .setCause(tearDownException); 716 runFailureMessages.add(failure); 717 } 718 // If there is any errors report them all at once 719 if (!runFailureMessages.isEmpty()) { 720 if (runFailureMessages.size() == 1) { 721 listener.testRunFailed(runFailureMessages.get(0)); 722 } else { 723 listener.testRunFailed(new MultiFailureDescription(runFailureMessages)); 724 } 725 mIsFailedModule = true; 726 } 727 728 // Provide a strong association of the run to its logs. 729 for (String key : aggLogFiles.keySet()) { 730 for (LogFile logFile : aggLogFiles.get(key)) { 731 if (listener instanceof ILogSaverListener) { 732 ((ILogSaverListener) listener).logAssociation(key, logFile); 733 } 734 } 735 } 736 // Allow each attempt to have its own start/end time 737 if (attempt != null) { 738 listener.testRunEnded(elapsedTime, metricsProto); 739 } else { 740 listener.testRunEnded(getCurrentTime() - mStartTestTime, metricsProto); 741 } 742 } 743 forwardTestResults( Map<TestDescription, TestResult> testResults, ITestInvocationListener listener)744 private void forwardTestResults( 745 Map<TestDescription, TestResult> testResults, ITestInvocationListener listener) { 746 for (Map.Entry<TestDescription, TestResult> testEntry : testResults.entrySet()) { 747 listener.testStarted(testEntry.getKey(), testEntry.getValue().getStartTime()); 748 switch (testEntry.getValue().getStatus()) { 749 case FAILURE: 750 listener.testFailed(testEntry.getKey(), testEntry.getValue().getStackTrace()); 751 break; 752 case ASSUMPTION_FAILURE: 753 listener.testAssumptionFailure( 754 testEntry.getKey(), testEntry.getValue().getStackTrace()); 755 break; 756 case IGNORED: 757 listener.testIgnored(testEntry.getKey()); 758 break; 759 case INCOMPLETE: 760 listener.testFailed( 761 testEntry.getKey(), "Test did not complete due to exception."); 762 break; 763 default: 764 break; 765 } 766 // Provide a strong association of the test to its logs. 767 for (Entry<String, LogFile> logFile : 768 testEntry.getValue().getLoggedFiles().entrySet()) { 769 if (listener instanceof ILogSaverListener) { 770 ((ILogSaverListener) listener) 771 .logAssociation(logFile.getKey(), logFile.getValue()); 772 } 773 } 774 listener.testEnded( 775 testEntry.getKey(), 776 testEntry.getValue().getEndTime(), 777 testEntry.getValue().getProtoMetrics()); 778 } 779 } 780 781 /** Run all the prepare steps. */ runPreparerSetup( TestInformation moduleInfo, ITargetPreparer preparer, ITestLogger logger, int deviceIndex)782 private Throwable runPreparerSetup( 783 TestInformation moduleInfo, 784 ITargetPreparer preparer, 785 ITestLogger logger, 786 int deviceIndex) { 787 if (preparer.isDisabled()) { 788 // If disabled skip completely. 789 return null; 790 } 791 TfObjectTracker.countWithParents(preparer.getClass()); 792 CLog.d("Running setup preparer: %s", preparer.getClass().getSimpleName()); 793 try { 794 // set the logger in case they need it. 795 if (preparer instanceof ITestLoggerReceiver) { 796 ((ITestLoggerReceiver) preparer).setTestLogger(logger); 797 } 798 if (preparer instanceof IInvocationContextReceiver) { 799 ((IInvocationContextReceiver) preparer) 800 .setInvocationContext(mModuleInvocationContext); 801 } 802 moduleInfo.setActiveDeviceIndex(deviceIndex); 803 preparer.setUp(moduleInfo); 804 return null; 805 } catch (BuildError 806 | TargetSetupError 807 | DeviceNotAvailableException 808 | RuntimeException 809 | AssertionError e) { 810 // We catch all the TargetPreparer possible exception + RuntimeException to avoid 811 // specific issues + AssertionError since it's widely used in tests and doesn't notify 812 // something very wrong with the harness. 813 CLog.e("Unexpected Exception from preparer: %s", preparer.getClass().getName()); 814 CLog.e(e); 815 return e; 816 } finally { 817 moduleInfo.setActiveDeviceIndex(0); 818 } 819 } 820 821 /** Run all multi target preparer step. */ runMultiPreparerSetup( IMultiTargetPreparer preparer, TestInformation moduleInfo, ITestLogger logger)822 private Throwable runMultiPreparerSetup( 823 IMultiTargetPreparer preparer, TestInformation moduleInfo, ITestLogger logger) { 824 if (preparer.isDisabled()) { 825 // If disabled skip completely. 826 return null; 827 } 828 TfObjectTracker.countWithParents(preparer.getClass()); 829 CLog.d("Running setup multi preparer: %s", preparer.getClass().getSimpleName()); 830 try { 831 // set the logger in case they need it. 832 if (preparer instanceof ITestLoggerReceiver) { 833 ((ITestLoggerReceiver) preparer).setTestLogger(logger); 834 } 835 if (preparer instanceof IInvocationContextReceiver) { 836 ((IInvocationContextReceiver) preparer) 837 .setInvocationContext(mModuleInvocationContext); 838 } 839 preparer.setUp(moduleInfo); 840 return null; 841 } catch (BuildError 842 | TargetSetupError 843 | DeviceNotAvailableException 844 | RuntimeException 845 | AssertionError e) { 846 // We catch all the MultiTargetPreparer possible exception + RuntimeException to avoid 847 // specific issues + AssertionError since it's widely used in tests and doesn't notify 848 // something very wrong with the harness. 849 CLog.e("Unexpected Exception from preparer: %s", preparer.getClass().getName()); 850 CLog.e(e); 851 return e; 852 } 853 } 854 855 /** Run all the tear down steps from preparers. */ runTearDown(TestInformation moduleInfo, Throwable exception)856 private void runTearDown(TestInformation moduleInfo, Throwable exception) 857 throws DeviceNotAvailableException { 858 // Tear down 859 List<IMultiTargetPreparer> cleanerList = new ArrayList<>(mMultiPreparers); 860 Collections.reverse(cleanerList); 861 for (IMultiTargetPreparer multiCleaner : cleanerList) { 862 if (multiCleaner.isDisabled() || multiCleaner.isTearDownDisabled()) { 863 // If disabled skip completely. 864 continue; 865 } 866 CLog.d("Running teardown multi cleaner: %s", multiCleaner.getClass().getSimpleName()); 867 multiCleaner.tearDown(moduleInfo, exception); 868 } 869 870 for (int i = 0; i < mModuleInvocationContext.getDeviceConfigNames().size(); i++) { 871 String deviceName = mModuleInvocationContext.getDeviceConfigNames().get(i); 872 ITestDevice device = mModuleInvocationContext.getDevice(deviceName); 873 if (i >= mPreparersPerDevice.size()) { 874 CLog.d( 875 "Main configuration has more devices than the module configuration. '%s' " 876 + "will not run any tear down.", 877 deviceName); 878 continue; 879 } 880 List<ITargetPreparer> preparers = mPreparersPerDevice.get(deviceName); 881 if (preparers == null) { 882 CLog.w( 883 "Module configuration devices mismatch the main configuration " 884 + "(Missing device '%s'), resolving preparers by index.", 885 deviceName); 886 String key = new ArrayList<>(mPreparersPerDevice.keySet()).get(i); 887 preparers = mPreparersPerDevice.get(key); 888 } 889 ListIterator<ITargetPreparer> itr = preparers.listIterator(preparers.size()); 890 while (itr.hasPrevious()) { 891 ITargetPreparer preparer = itr.previous(); 892 // do not call the cleaner if it was disabled 893 if (preparer.isDisabled() || preparer.isTearDownDisabled()) { 894 CLog.d("%s has been disabled. skipping.", preparer); 895 continue; 896 } 897 898 RecoveryMode origMode = null; 899 try { 900 // If an exception was generated in setup with a DNAE do not attempt any 901 // recovery again in case we hit the device not available again. 902 if (exception != null && exception instanceof DeviceNotAvailableException) { 903 origMode = device.getRecoveryMode(); 904 device.setRecoveryMode(RecoveryMode.NONE); 905 } 906 moduleInfo.setActiveDeviceIndex(i); 907 preparer.tearDown(moduleInfo, exception); 908 } finally { 909 moduleInfo.setActiveDeviceIndex(0); 910 if (origMode != null) { 911 device.setRecoveryMode(origMode); 912 } 913 } 914 } 915 } 916 } 917 918 /** Returns the current time. */ getCurrentTime()919 private long getCurrentTime() { 920 return System.currentTimeMillis(); 921 } 922 923 @Override setCollectTestsOnly(boolean collectTestsOnly)924 public void setCollectTestsOnly(boolean collectTestsOnly) { 925 mCollectTestsOnly = collectTestsOnly; 926 } 927 928 /** Sets whether or not we should merge results. */ setMergeAttemps(boolean mergeAttempts)929 public final void setMergeAttemps(boolean mergeAttempts) { 930 mMergeAttempts = mergeAttempts; 931 } 932 933 /** Sets the {@link IRetryDecision} to be used for intra-module retry. */ setRetryDecision(IRetryDecision decision)934 public final void setRetryDecision(IRetryDecision decision) { 935 mRetryDecision = decision; 936 // Carry the retry decision to the module configuration 937 mModuleConfiguration.setRetryDecision(decision); 938 } 939 940 /** Returns a list of tests that ran in this module. */ getTestsResults()941 List<TestRunResult> getTestsResults() { 942 return mTestsResults; 943 } 944 945 /** Returns the number of tests that was expected to be run */ getNumExpectedTests()946 int getNumExpectedTests() { 947 return mExpectedTests; 948 } 949 950 /** Returns True if a testRunFailure has been called on the module * */ hasModuleFailed()951 public boolean hasModuleFailed() { 952 return mIsFailedModule; 953 } 954 getRequiredTokens()955 public Set<TokenProperty> getRequiredTokens() { 956 return mRequiredTokens; 957 } 958 959 /** {@inheritDoc} */ 960 @Override toString()961 public String toString() { 962 return getId(); 963 } 964 965 /** Returns the approximate time to run all the tests in the module. */ getRuntimeHint()966 public long getRuntimeHint() { 967 long hint = 0l; 968 for (IRemoteTest test : mTests) { 969 if (test instanceof IRuntimeHintProvider) { 970 hint += ((IRuntimeHintProvider) test).getRuntimeHint(); 971 } else { 972 hint += 60000; 973 } 974 } 975 return hint; 976 } 977 978 /** Returns the list of {@link IRemoteTest} defined for this module. */ 979 @VisibleForTesting getTests()980 List<IRemoteTest> getTests() { 981 return new ArrayList<>(mTests); 982 } 983 984 /** Returns the list of {@link ITargetPreparer} associated with the given device name */ 985 @VisibleForTesting getTargetPreparerForDevice(String deviceName)986 List<ITargetPreparer> getTargetPreparerForDevice(String deviceName) { 987 return mPreparersPerDevice.get(deviceName); 988 } 989 990 /** 991 * When running unit tests for ModuleDefinition we don't want to unnecessarily report some auto 992 * retry times. 993 */ 994 @VisibleForTesting disableAutoRetryReportingTime()995 void disableAutoRetryReportingTime() { 996 mDisableAutoRetryTimeReporting = true; 997 } 998 999 /** Returns the {@link IInvocationContext} associated with the module. */ getModuleInvocationContext()1000 public IInvocationContext getModuleInvocationContext() { 1001 return mModuleInvocationContext; 1002 } 1003 1004 /** Report completely not executed modules. */ reportNotExecuted(ITestInvocationListener listener, String message)1005 public final void reportNotExecuted(ITestInvocationListener listener, String message) { 1006 if (mStartModuleRunDate == null) { 1007 listener.testModuleStarted(getModuleInvocationContext()); 1008 } 1009 listener.testRunStarted(getId(), 0, 0, System.currentTimeMillis()); 1010 FailureDescription description = 1011 FailureDescription.create(message).setFailureStatus(FailureStatus.NOT_EXECUTED); 1012 listener.testRunFailed(description); 1013 listener.testRunEnded(0, new HashMap<String, Metric>()); 1014 listener.testModuleEnded(); 1015 } 1016 1017 /** Whether or not to enable dynamic download at module level. */ setEnableDynamicDownload(boolean enableDynamicDownload)1018 public void setEnableDynamicDownload(boolean enableDynamicDownload) { 1019 mEnableDynamicDownload = enableDynamicDownload; 1020 } 1021 addDynamicDownloadArgs(Map<String, String> extraArgs)1022 public void addDynamicDownloadArgs(Map<String, String> extraArgs) { 1023 mModuleConfiguration.getCommandOptions().getDynamicDownloadArgs().putAll(extraArgs); 1024 } 1025 1026 /** 1027 * Allow to load a module_controller object to tune how should a particular module run. 1028 * 1029 * @param failureListener The {@link TestFailureListener} taking actions on tests failures. 1030 * @return The strategy to use to run the tests. 1031 */ applyConfigurationControl(TestFailureListener failureListener)1032 private RunStrategy applyConfigurationControl(TestFailureListener failureListener) { 1033 Object ctrlObject = mModuleConfiguration.getConfigurationObject(MODULE_CONTROLLER); 1034 if (ctrlObject != null && ctrlObject instanceof BaseModuleController) { 1035 BaseModuleController controller = (BaseModuleController) ctrlObject; 1036 // module_controller can also control the log collection for the one module 1037 if (failureListener != null) { 1038 failureListener.applyModuleConfiguration(controller.shouldCaptureBugreport()); 1039 } 1040 if (!controller.shouldCaptureLogcat()) { 1041 mRunMetricCollectors.removeIf(c -> (c instanceof LogcatOnFailureCollector)); 1042 } 1043 if (!controller.shouldCaptureScreenshot()) { 1044 mRunMetricCollectors.removeIf(c -> (c instanceof ScreenshotOnFailureCollector)); 1045 } 1046 return controller.shouldRunModule(mModuleInvocationContext); 1047 } 1048 return RunStrategy.RUN; 1049 } 1050 addRetryTime(long retryTimeMs)1051 private void addRetryTime(long retryTimeMs) { 1052 if (retryTimeMs <= 0 || mDisableAutoRetryTimeReporting) { 1053 return; 1054 } 1055 InvocationMetricLogger.addInvocationMetrics( 1056 InvocationMetricKey.AUTO_RETRY_TIME, retryTimeMs); 1057 } 1058 runTargetPreparation(TestInformation moduleInfo, ITestLogger logger)1059 private Throwable runTargetPreparation(TestInformation moduleInfo, ITestLogger logger) { 1060 Throwable preparationException = null; 1061 for (int i = 0; i < mModuleInvocationContext.getDeviceConfigNames().size(); i++) { 1062 String deviceName = mModuleInvocationContext.getDeviceConfigNames().get(i); 1063 if (i >= mPreparersPerDevice.size()) { 1064 CLog.d( 1065 "Main configuration has more devices than the module configuration. '%s' " 1066 + "will not run any preparation.", 1067 deviceName); 1068 continue; 1069 } 1070 List<ITargetPreparer> preparers = mPreparersPerDevice.get(deviceName); 1071 if (preparers == null) { 1072 CLog.w( 1073 "Module configuration devices mismatch the main configuration " 1074 + "(Missing device '%s'), resolving preparers by index.", 1075 deviceName); 1076 String key = new ArrayList<>(mPreparersPerDevice.keySet()).get(i); 1077 preparers = mPreparersPerDevice.get(key); 1078 } 1079 for (ITargetPreparer preparer : preparers) { 1080 preparationException = runPreparerSetup(moduleInfo, preparer, logger, i); 1081 if (preparationException != null) { 1082 mIsFailedModule = true; 1083 CLog.e("Some preparation step failed. failing the module %s", getId()); 1084 // If one device errored out, we skip the remaining devices. 1085 return preparationException; 1086 } 1087 } 1088 } 1089 return null; 1090 } 1091 1092 /** 1093 * Handle calling the {@link IConfiguration#resolveDynamicOptions(DynamicRemoteFileResolver)}. 1094 */ invokeRemoteDynamic(ITestDevice device, IConfiguration moduleConfiguration)1095 private Exception invokeRemoteDynamic(ITestDevice device, IConfiguration moduleConfiguration) { 1096 if (!mEnableDynamicDownload) { 1097 return null; 1098 } 1099 // TODO: Add elapsed time tracking 1100 try { 1101 CLog.d("Attempting to resolve dynamic files from %s", getId()); 1102 DynamicRemoteFileResolver resolver = new DynamicRemoteFileResolver(); 1103 resolver.setDevice(device); 1104 resolver.addExtraArgs(moduleConfiguration.getCommandOptions().getDynamicDownloadArgs()); 1105 moduleConfiguration.resolveDynamicOptions(resolver); 1106 return null; 1107 } catch (RuntimeException | ConfigurationException | BuildRetrievalError e) { 1108 mIsFailedModule = true; 1109 return e; 1110 } 1111 } 1112 1113 /** Report a setup exception as a run failure and notify all the listeners. */ reportSetupFailure( Throwable setupException, ITestInvocationListener invocListener, List<ITestInvocationListener> moduleListeners)1114 private void reportSetupFailure( 1115 Throwable setupException, 1116 ITestInvocationListener invocListener, 1117 List<ITestInvocationListener> moduleListeners) 1118 throws DeviceNotAvailableException { 1119 List<ITestInvocationListener> allListeners = new ArrayList<>(); 1120 allListeners.add(invocListener); 1121 if (moduleListeners != null) { 1122 allListeners.addAll(moduleListeners); 1123 } 1124 // Report the early module failures to the moduleListeners too in order for them 1125 // to know about it. 1126 ITestInvocationListener forwarder = new ResultForwarder(allListeners); 1127 // For reporting purpose we create a failure placeholder with the error stack 1128 // similar to InitializationError of JUnit. 1129 forwarder.testRunStarted(getId(), 1, 0, System.currentTimeMillis()); 1130 FailureDescription failureDescription = 1131 CurrentInvocation.createFailure(StreamUtil.getStackTrace(setupException), null); 1132 if (setupException instanceof IHarnessException 1133 && ((IHarnessException) setupException).getErrorId() != null) { 1134 ErrorIdentifier id = ((IHarnessException) setupException).getErrorId(); 1135 failureDescription.setErrorIdentifier(id); 1136 failureDescription.setFailureStatus(id.status()); 1137 failureDescription.setOrigin(((IHarnessException) setupException).getOrigin()); 1138 } else if (setupException instanceof RuntimeException) { 1139 // TODO: switch to customer_issue 1140 failureDescription.setFailureStatus(FailureStatus.UNSET); 1141 failureDescription.setErrorIdentifier( 1142 InfraErrorIdentifier.MODULE_SETUP_RUNTIME_EXCEPTION); 1143 } else { 1144 failureDescription.setFailureStatus(FailureStatus.UNSET); 1145 } 1146 failureDescription.setCause(setupException); 1147 forwarder.testRunFailed(failureDescription); 1148 HashMap<String, Metric> metricsProto = new HashMap<>(); 1149 metricsProto.put(TEST_TIME, TfMetricProtoUtil.createSingleValue(0L, "milliseconds")); 1150 forwarder.testRunEnded(0, metricsProto); 1151 // If it was a not available exception rethrow it to signal the new device state. 1152 if (setupException instanceof DeviceNotAvailableException) { 1153 throw (DeviceNotAvailableException) setupException; 1154 } 1155 } 1156 } 1157