1 /* 2 * Copyright (C) 2016 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package com.android.tradefed.testtype; 17 18 import com.android.tradefed.build.IBuildInfo; 19 import com.android.tradefed.build.IFolderBuildInfo; 20 import com.android.tradefed.command.CommandOptions; 21 import com.android.tradefed.config.GlobalConfiguration; 22 import com.android.tradefed.config.IConfiguration; 23 import com.android.tradefed.config.IConfigurationReceiver; 24 import com.android.tradefed.config.Option; 25 import com.android.tradefed.device.DeviceNotAvailableException; 26 import com.android.tradefed.invoker.IInvocationContext; 27 import com.android.tradefed.invoker.TestInformation; 28 import com.android.tradefed.log.LogUtil.CLog; 29 import com.android.tradefed.metrics.proto.MetricMeasurement.Metric; 30 import com.android.tradefed.result.FileInputStreamSource; 31 import com.android.tradefed.result.ITestInvocationListener; 32 import com.android.tradefed.result.LogDataType; 33 import com.android.tradefed.result.TestDescription; 34 import com.android.tradefed.result.proto.StreamProtoReceiver; 35 import com.android.tradefed.result.proto.StreamProtoResultReporter; 36 import com.android.tradefed.util.CommandResult; 37 import com.android.tradefed.util.CommandStatus; 38 import com.android.tradefed.util.FileUtil; 39 import com.android.tradefed.util.IRunUtil; 40 import com.android.tradefed.util.IRunUtil.EnvPriority; 41 import com.android.tradefed.util.RunUtil; 42 import com.android.tradefed.util.StreamUtil; 43 import com.android.tradefed.util.SubprocessExceptionParser; 44 import com.android.tradefed.util.SubprocessTestResultsParser; 45 import com.android.tradefed.util.SystemUtil; 46 import com.android.tradefed.util.TimeUtil; 47 import com.android.tradefed.util.UniqueMultiMap; 48 49 import org.junit.Assert; 50 51 import java.io.File; 52 import java.io.FileOutputStream; 53 import java.io.IOException; 54 import java.util.ArrayList; 55 import java.util.Arrays; 56 import java.util.HashMap; 57 import java.util.LinkedHashSet; 58 import java.util.List; 59 import java.util.Set; 60 61 /** 62 * A {@link IRemoteTest} for running tests against a separate TF installation. 63 * 64 * <p>Launches an external java process to run the tests. Used for running the TF unit or functional 65 * tests continuously. 66 */ 67 public abstract class SubprocessTfLauncher 68 implements IBuildReceiver, IInvocationContextReceiver, IRemoteTest, IConfigurationReceiver { 69 70 /** The tag that will be passed to the TF subprocess to differentiate it */ 71 public static final String SUBPROCESS_TAG_NAME = "subprocess"; 72 73 public static final String PARENT_PROC_TAG_NAME = "parentprocess"; 74 /** Env. variable that affects adb selection. */ 75 public static final String ANDROID_SERIAL_VAR = "ANDROID_SERIAL"; 76 77 @Option(name = "max-run-time", description = 78 "The maximum time to allow for a TF test run.", isTimeVal = true) 79 private long mMaxTfRunTime = 20 * 60 * 1000; 80 81 @Option(name = "remote-debug", description = 82 "Start the TF java process in remote debug mode.") 83 private boolean mRemoteDebug = false; 84 85 @Option(name = "config-name", description = "The config that runs the TF tests") 86 private String mConfigName; 87 88 @Option( 89 name = "local-sharding-mode", 90 description = 91 "If sharding is requested, allow the launcher to run with local sharding.") 92 private boolean mLocalShardingMode = false; 93 94 @Option(name = "use-event-streaming", description = "Use a socket to receive results as they" 95 + "arrived instead of using a temporary file and parsing at the end.") 96 private boolean mEventStreaming = true; 97 98 @Option( 99 name = "use-proto-reporting", 100 description = "Use a proto result reporter for the results from the subprocess.") 101 private boolean mUseProtoReporting = false; 102 103 @Option(name = "sub-global-config", description = "The global config name to pass to the" 104 + "sub process, can be local or from jar resources. Be careful of conflicts with " 105 + "parent process.") 106 private String mGlobalConfig = null; 107 108 @Option( 109 name = "inject-invocation-data", 110 description = "Pass the invocation-data to the subprocess if enabled.") 111 private boolean mInjectInvocationData = true; 112 113 @Option(name = "ignore-test-log", description = "Only rely on logAssociation for logs.") 114 private boolean mIgnoreTestLog = true; 115 116 @Option( 117 name = "disable-stderr-test", 118 description = "Whether or not to disable the stderr validation check." 119 ) 120 private boolean mDisableStderrTest = false; 121 122 @Option( 123 name = "disable-add-opens", 124 description = "Whether or not to add the java add-opens flags" 125 ) 126 private boolean mDisableJavaOpens = false; 127 128 @Option(name = "add-opens", description = "Whether or not to add the java add-opens flags") 129 private Set<String> mAddOpens = 130 new LinkedHashSet<>( 131 Arrays.asList( 132 "java.base/java.nio", 133 "java.base/sun.reflect.annotation", 134 "java.base/java.io")); 135 136 // Temp global configuration filtered from the parent process. 137 private String mFilteredGlobalConfig = null; 138 139 private static final List<String> TRADEFED_JARS = 140 new ArrayList<>( 141 Arrays.asList( 142 // Loganalysis 143 "loganalysis.jar", 144 "loganalysis-tests.jar", 145 // Aosp Tf jars 146 "tradefed.jar", 147 "tradefed-test-framework.jar", 148 "tradefed-tests.jar", 149 // libs 150 "tools-common-prebuilt.jar", 151 // jar in older branches 152 "tf-prod-tests.jar", 153 "tf-prod-metatests.jar", 154 // Aosp contrib jars 155 "tradefed-contrib.jar", 156 "tf-contrib-tests.jar", 157 // Google Tf jars 158 "google-tf-prod-tests.jar", 159 "google-tf-prod-metatests.jar", 160 "google-tradefed.jar", 161 "google-tradefed-tests.jar", 162 // Google contrib jars 163 "google-tradefed-contrib.jar", 164 // Older jar required for coverage tests 165 "jack-jacoco-reporter.jar", 166 "emmalib.jar")); 167 168 /** Timeout to wait for the events received from subprocess to finish being processed.*/ 169 private static final long EVENT_THREAD_JOIN_TIMEOUT_MS = 30 * 1000; 170 171 protected IRunUtil mRunUtil = new RunUtil(); 172 173 protected IBuildInfo mBuildInfo = null; 174 // Temp directory to run the TF process. 175 protected File mTmpDir = null; 176 // List of command line arguments to run the TF process. 177 protected List<String> mCmdArgs = null; 178 // The absolute path to the build's root directory. 179 protected String mRootDir = null; 180 protected IConfiguration mConfig; 181 private IInvocationContext mContext; 182 183 @Override setInvocationContext(IInvocationContext invocationContext)184 public void setInvocationContext(IInvocationContext invocationContext) { 185 mContext = invocationContext; 186 } 187 188 @Override setConfiguration(IConfiguration configuration)189 public void setConfiguration(IConfiguration configuration) { 190 mConfig = configuration; 191 } 192 193 /** 194 * Set use-event-streaming. 195 * 196 * Exposed for unit testing. 197 */ setEventStreaming(boolean eventStreaming)198 protected void setEventStreaming(boolean eventStreaming) { 199 mEventStreaming = eventStreaming; 200 } 201 202 /** 203 * Set IRunUtil. 204 * 205 * Exposed for unit testing. 206 */ setRunUtil(IRunUtil runUtil)207 protected void setRunUtil(IRunUtil runUtil) { 208 mRunUtil = runUtil; 209 } 210 211 /** Returns the {@link IRunUtil} that will be used for the subprocess command. */ getRunUtil()212 protected IRunUtil getRunUtil() { 213 return mRunUtil; 214 } 215 216 /** 217 * Setup before running the test. 218 */ preRun()219 protected void preRun() { 220 Assert.assertNotNull(mBuildInfo); 221 Assert.assertNotNull(mConfigName); 222 IFolderBuildInfo tfBuild = (IFolderBuildInfo) mBuildInfo; 223 File rootDirFile = tfBuild.getRootDir(); 224 mRootDir = rootDirFile.getAbsolutePath(); 225 String jarClasspath = ""; 226 List<String> paths = new ArrayList<>(); 227 for (String jar : TRADEFED_JARS) { 228 File f = FileUtil.findFile(rootDirFile, jar); 229 if (f != null && f.exists()) { 230 paths.add(f.getAbsolutePath()); 231 } 232 } 233 jarClasspath = String.join(":", paths); 234 235 mCmdArgs = new ArrayList<String>(); 236 mCmdArgs.add(SystemUtil.getRunningJavaBinaryPath().getAbsolutePath()); 237 238 try { 239 mTmpDir = FileUtil.createTempDir("subprocess-" + tfBuild.getBuildId()); 240 mCmdArgs.add(String.format("-Djava.io.tmpdir=%s", mTmpDir.getAbsolutePath())); 241 } catch (IOException e) { 242 CLog.e(e); 243 throw new RuntimeException(e); 244 } 245 246 addJavaArguments(mCmdArgs); 247 248 if (mRemoteDebug) { 249 mCmdArgs.add("-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=10088"); 250 } 251 // This prevent the illegal reflective access warnings by allowing some packages. 252 if (!mDisableJavaOpens) { 253 for (String modulePackage : mAddOpens) { 254 mCmdArgs.add("--add-opens=" + modulePackage + "=ALL-UNNAMED"); 255 } 256 } 257 mCmdArgs.add("-cp"); 258 259 mCmdArgs.add(jarClasspath); 260 mCmdArgs.add("com.android.tradefed.command.CommandRunner"); 261 mCmdArgs.add(mConfigName); 262 263 Integer shardCount = mConfig.getCommandOptions().getShardCount(); 264 if (mLocalShardingMode && shardCount != null & shardCount > 1) { 265 mCmdArgs.add("--shard-count"); 266 mCmdArgs.add(Integer.toString(shardCount)); 267 } 268 269 // clear the TF_GLOBAL_CONFIG env, so another tradefed will not reuse the global config file 270 mRunUtil.unsetEnvVariable(GlobalConfiguration.GLOBAL_CONFIG_VARIABLE); 271 mRunUtil.unsetEnvVariable(GlobalConfiguration.GLOBAL_CONFIG_SERVER_CONFIG_VARIABLE); 272 mRunUtil.unsetEnvVariable(ANDROID_SERIAL_VAR); 273 274 if (mGlobalConfig == null) { 275 // If the global configuration is not set in option, create a filtered global 276 // configuration for subprocess to use. 277 try { 278 File filteredGlobalConfig = 279 GlobalConfiguration.getInstance().cloneConfigWithFilter(); 280 mFilteredGlobalConfig = filteredGlobalConfig.getAbsolutePath(); 281 mGlobalConfig = mFilteredGlobalConfig; 282 } catch (IOException e) { 283 CLog.e("Failed to create filtered global configuration"); 284 CLog.e(e); 285 } 286 } 287 if (mGlobalConfig != null) { 288 // We allow overriding this global config and then set it for the subprocess. 289 mRunUtil.setEnvVariablePriority(EnvPriority.SET); 290 mRunUtil.setEnvVariable(GlobalConfiguration.GLOBAL_CONFIG_VARIABLE, mGlobalConfig); 291 } 292 } 293 294 /** 295 * Allow to add extra java parameters to the subprocess invocation. 296 * 297 * @param args the current list of arguments to which we need to add the extra ones. 298 */ addJavaArguments(List<String> args)299 protected void addJavaArguments(List<String> args) {} 300 301 /** 302 * Actions to take after the TF test is finished. 303 * 304 * @param listener the original {@link ITestInvocationListener} where to report results. 305 * @param exception True if exception was raised inside the test. 306 * @param elapsedTime the time taken to run the tests. 307 */ postRun(ITestInvocationListener listener, boolean exception, long elapsedTime)308 protected void postRun(ITestInvocationListener listener, boolean exception, long elapsedTime) {} 309 310 /** Pipe to the subprocess the invocation-data so that it can use them if needed. */ addInvocationData()311 private void addInvocationData() { 312 if (!mInjectInvocationData) { 313 return; 314 } 315 UniqueMultiMap<String, String> data = mConfig.getCommandOptions().getInvocationData(); 316 for (String key : data.keySet()) { 317 for (String value : data.get(key)) { 318 mCmdArgs.add("--" + CommandOptions.INVOCATION_DATA); 319 mCmdArgs.add(key); 320 mCmdArgs.add(value); 321 } 322 } 323 // Finally add one last more to tag the subprocess 324 mCmdArgs.add("--" + CommandOptions.INVOCATION_DATA); 325 mCmdArgs.add(SUBPROCESS_TAG_NAME); 326 mCmdArgs.add("true"); 327 // Tag the parent invocation 328 mBuildInfo.addBuildAttribute(PARENT_PROC_TAG_NAME, "true"); 329 } 330 331 /** {@inheritDoc} */ 332 @Override run(TestInformation testInfo, ITestInvocationListener listener)333 public void run(TestInformation testInfo, ITestInvocationListener listener) 334 throws DeviceNotAvailableException { 335 preRun(); 336 addInvocationData(); 337 338 File stdoutFile = null; 339 File stderrFile = null; 340 File eventFile = null; 341 SubprocessTestResultsParser eventParser = null; 342 StreamProtoReceiver protoReceiver = null; 343 FileOutputStream stdout = null; 344 FileOutputStream stderr = null; 345 346 boolean exception = false; 347 long startTime = 0l; 348 long elapsedTime = -1l; 349 try { 350 stdoutFile = FileUtil.createTempFile("stdout_subprocess_", ".log"); 351 stderrFile = FileUtil.createTempFile("stderr_subprocess_", ".log"); 352 stderr = new FileOutputStream(stderrFile); 353 stdout = new FileOutputStream(stdoutFile); 354 355 if (mUseProtoReporting) { 356 protoReceiver = new StreamProtoReceiver(listener, mContext, false, false); 357 mCmdArgs.add("--" + StreamProtoResultReporter.PROTO_REPORT_PORT_OPTION); 358 mCmdArgs.add(Integer.toString(protoReceiver.getSocketServerPort())); 359 } else { 360 eventParser = new SubprocessTestResultsParser(listener, mEventStreaming, mContext); 361 if (mEventStreaming) { 362 mCmdArgs.add("--subprocess-report-port"); 363 mCmdArgs.add(Integer.toString(eventParser.getSocketServerPort())); 364 } else { 365 eventFile = FileUtil.createTempFile("event_subprocess_", ".log"); 366 mCmdArgs.add("--subprocess-report-file"); 367 mCmdArgs.add(eventFile.getAbsolutePath()); 368 } 369 eventParser.setIgnoreTestLog(mIgnoreTestLog); 370 } 371 startTime = System.currentTimeMillis(); 372 CommandResult result = mRunUtil.runTimedCmd(mMaxTfRunTime, stdout, 373 stderr, mCmdArgs.toArray(new String[0])); 374 375 if (eventParser != null) { 376 if (eventParser.getStartTime() != null) { 377 startTime = eventParser.getStartTime(); 378 } 379 elapsedTime = System.currentTimeMillis() - startTime; 380 // We possibly allow for a little more time if the thread is still processing 381 // events. 382 if (!eventParser.joinReceiver(EVENT_THREAD_JOIN_TIMEOUT_MS)) { 383 elapsedTime = -1l; 384 throw new RuntimeException( 385 String.format( 386 "Event receiver thread did not complete:" + "\n%s", 387 FileUtil.readStringFromFile(stderrFile))); 388 } 389 } else if (protoReceiver != null) { 390 if (!protoReceiver.joinReceiver(EVENT_THREAD_JOIN_TIMEOUT_MS)) { 391 elapsedTime = -1l; 392 throw new RuntimeException( 393 String.format( 394 "Event receiver thread did not complete:" + "\n%s", 395 FileUtil.readStringFromFile(stderrFile))); 396 } 397 } 398 if (result.getStatus().equals(CommandStatus.SUCCESS)) { 399 CLog.d("Successfully ran TF tests for build %s", mBuildInfo.getBuildId()); 400 testCleanStdErr(stderrFile, listener); 401 } else { 402 CLog.w("Failed ran TF tests for build %s, status %s", 403 mBuildInfo.getBuildId(), result.getStatus()); 404 CLog.v( 405 "TF tests output:\nstdout:\n%s\nstderr:\n%s", 406 result.getStdout(), result.getStderr()); 407 exception = true; 408 String errMessage = null; 409 if (result.getStatus().equals(CommandStatus.TIMED_OUT)) { 410 errMessage = String.format("Timeout after %s", 411 TimeUtil.formatElapsedTime(mMaxTfRunTime)); 412 throw new RuntimeException( 413 String.format( 414 "%s Tests subprocess failed due to:\n%s\n", 415 mConfigName, errMessage)); 416 } else { 417 SubprocessExceptionParser.handleStderrException(result); 418 } 419 } 420 } catch (IOException e) { 421 exception = true; 422 throw new RuntimeException(e); 423 } finally { 424 StreamUtil.close(stdout); 425 StreamUtil.close(stderr); 426 logAndCleanFile(stdoutFile, listener); 427 logAndCleanFile(stderrFile, listener); 428 if (eventFile != null) { 429 eventParser.parseFile(eventFile); 430 logAndCleanFile(eventFile, listener); 431 } 432 StreamUtil.close(eventParser); 433 StreamUtil.close(protoReceiver); 434 435 postRun(listener, exception, elapsedTime); 436 437 if (mTmpDir != null) { 438 FileUtil.recursiveDelete(mTmpDir); 439 } 440 441 if (mFilteredGlobalConfig != null) { 442 FileUtil.deleteFile(new File(mFilteredGlobalConfig)); 443 } 444 } 445 } 446 447 /** 448 * Log the content of given file to listener, then remove the file. 449 * 450 * @param fileToExport the {@link File} pointing to the file to log. 451 * @param listener the {@link ITestInvocationListener} where to report the test. 452 */ logAndCleanFile(File fileToExport, ITestInvocationListener listener)453 private void logAndCleanFile(File fileToExport, ITestInvocationListener listener) { 454 if (fileToExport == null) 455 return; 456 457 try (FileInputStreamSource inputStream = new FileInputStreamSource(fileToExport)) { 458 listener.testLog(fileToExport.getName(), LogDataType.TEXT, inputStream); 459 } 460 FileUtil.deleteFile(fileToExport); 461 } 462 463 /** 464 * {@inheritDoc} 465 */ 466 @Override setBuild(IBuildInfo buildInfo)467 public void setBuild(IBuildInfo buildInfo) { 468 mBuildInfo = buildInfo; 469 } 470 471 /** 472 * Extra test to ensure no abnormal logging is made to stderr when all the tests pass. 473 * 474 * @param stdErrFile the stderr log file of the subprocess. 475 * @param listener the {@link ITestInvocationListener} where to report the test. 476 */ testCleanStdErr(File stdErrFile, ITestInvocationListener listener)477 private void testCleanStdErr(File stdErrFile, ITestInvocationListener listener) 478 throws IOException { 479 if (mDisableStderrTest) { 480 return; 481 } 482 listener.testRunStarted("StdErr", 1); 483 TestDescription tid = new TestDescription("stderr-test", "checkIsEmpty"); 484 listener.testStarted(tid); 485 if (!FileUtil.readStringFromFile(stdErrFile).isEmpty()) { 486 String trace = 487 String.format( 488 "Found some output in stderr:\n%s", 489 FileUtil.readStringFromFile(stdErrFile)); 490 listener.testFailed(tid, trace); 491 } 492 listener.testEnded(tid, new HashMap<String, Metric>()); 493 listener.testRunEnded(0, new HashMap<String, Metric>()); 494 } 495 } 496