1 /* 2 * Copyright (C) 2017 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package com.android.tradefed.sandbox; 17 18 import com.android.annotations.VisibleForTesting; 19 import com.android.tradefed.command.CommandOptions; 20 import com.android.tradefed.config.Configuration; 21 import com.android.tradefed.config.ConfigurationException; 22 import com.android.tradefed.config.ConfigurationFactory; 23 import com.android.tradefed.config.GlobalConfiguration; 24 import com.android.tradefed.config.IConfiguration; 25 import com.android.tradefed.config.IConfigurationFactory; 26 import com.android.tradefed.config.IGlobalConfiguration; 27 import com.android.tradefed.invoker.IInvocationContext; 28 import com.android.tradefed.invoker.InvocationContext; 29 import com.android.tradefed.invoker.logger.CurrentInvocation; 30 import com.android.tradefed.invoker.logger.CurrentInvocation.InvocationInfo; 31 import com.android.tradefed.invoker.proto.InvocationContext.Context; 32 import com.android.tradefed.log.ITestLogger; 33 import com.android.tradefed.log.LogUtil.CLog; 34 import com.android.tradefed.result.FileInputStreamSource; 35 import com.android.tradefed.result.ITestInvocationListener; 36 import com.android.tradefed.result.InputStreamSource; 37 import com.android.tradefed.result.LogDataType; 38 import com.android.tradefed.result.proto.StreamProtoReceiver; 39 import com.android.tradefed.result.proto.StreamProtoResultReporter; 40 import com.android.tradefed.sandbox.SandboxConfigDump.DumpCmd; 41 import com.android.tradefed.util.CommandResult; 42 import com.android.tradefed.util.CommandStatus; 43 import com.android.tradefed.util.FileUtil; 44 import com.android.tradefed.util.IRunUtil; 45 import com.android.tradefed.util.PrettyPrintDelimiter; 46 import com.android.tradefed.util.QuotationAwareTokenizer; 47 import com.android.tradefed.util.RunUtil; 48 import com.android.tradefed.util.StreamUtil; 49 import com.android.tradefed.util.SubprocessTestResultsParser; 50 import com.android.tradefed.util.SystemUtil; 51 import com.android.tradefed.util.keystore.IKeyStoreClient; 52 53 import java.io.File; 54 import java.io.FileOutputStream; 55 import java.io.IOException; 56 import java.io.OutputStream; 57 import java.io.PrintWriter; 58 import java.lang.reflect.InvocationTargetException; 59 import java.lang.reflect.Method; 60 import java.util.ArrayList; 61 import java.util.HashSet; 62 import java.util.List; 63 import java.util.Map.Entry; 64 import java.util.Set; 65 66 /** 67 * Sandbox container that can run a Trade Federation invocation. TODO: Allow Options to be passed to 68 * the sandbox. 69 */ 70 public class TradefedSandbox implements ISandbox { 71 72 private static final String SANDBOX_PREFIX = "sandbox-"; 73 74 private File mStdoutFile = null; 75 private File mStderrFile = null; 76 private OutputStream mStdout = null; 77 private FileOutputStream mStderr = null; 78 private File mHeapDump = null; 79 80 private File mSandboxTmpFolder = null; 81 private File mRootFolder = null; 82 private File mGlobalConfig = null; 83 private File mSerializedContext = null; 84 private File mSerializedConfiguration = null; 85 86 private SubprocessTestResultsParser mEventParser = null; 87 private StreamProtoReceiver mProtoReceiver = null; 88 89 private IRunUtil mRunUtil; 90 private boolean mCollectStdout = true; 91 92 @Override run(IConfiguration config, ITestLogger logger)93 public CommandResult run(IConfiguration config, ITestLogger logger) throws Throwable { 94 List<String> mCmdArgs = new ArrayList<>(); 95 mCmdArgs.add(SystemUtil.getRunningJavaBinaryPath().getAbsolutePath()); 96 mCmdArgs.add(String.format("-Djava.io.tmpdir=%s", mSandboxTmpFolder.getAbsolutePath())); 97 mCmdArgs.add(String.format("-DTF_JAR_DIR=%s", mRootFolder.getAbsolutePath())); 98 // Setup heap dump collection 99 try { 100 mHeapDump = FileUtil.createTempDir("heap-dump", getWorkFolder()); 101 mCmdArgs.add("-XX:+HeapDumpOnOutOfMemoryError"); 102 mCmdArgs.add(String.format("-XX:HeapDumpPath=%s", mHeapDump.getAbsolutePath())); 103 } catch (IOException e) { 104 CLog.e(e); 105 } 106 mCmdArgs.addAll(getSandboxOptions(config).getJavaOptions()); 107 mCmdArgs.add("-cp"); 108 mCmdArgs.add(createClasspath(mRootFolder)); 109 mCmdArgs.add(TradefedSandboxRunner.class.getCanonicalName()); 110 mCmdArgs.add(mSerializedContext.getAbsolutePath()); 111 mCmdArgs.add(mSerializedConfiguration.getAbsolutePath()); 112 if (mProtoReceiver != null) { 113 mCmdArgs.add("--" + StreamProtoResultReporter.PROTO_REPORT_PORT_OPTION); 114 mCmdArgs.add(Integer.toString(mProtoReceiver.getSocketServerPort())); 115 } else { 116 mCmdArgs.add("--subprocess-report-port"); 117 mCmdArgs.add(Integer.toString(mEventParser.getSocketServerPort())); 118 } 119 if (config.getCommandOptions().shouldUseSandboxTestMode()) { 120 // In test mode, re-add the --use-sandbox to trigger a sandbox run again in the process 121 mCmdArgs.add("--" + CommandOptions.USE_SANDBOX); 122 } 123 124 long timeout = config.getCommandOptions().getInvocationTimeout(); 125 // Allow interruption, subprocess should handle signals itself 126 mRunUtil.allowInterrupt(true); 127 CommandResult result = null; 128 RuntimeException interruptedException = null; 129 try { 130 result = 131 mRunUtil.runTimedCmd( 132 timeout, mStdout, mStderr, mCmdArgs.toArray(new String[0])); 133 } catch (RuntimeException interrupted) { 134 CLog.e("Sandbox runtimedCmd threw an exception"); 135 CLog.e(interrupted); 136 interruptedException = interrupted; 137 result = new CommandResult(CommandStatus.EXCEPTION); 138 result.setStdout(StreamUtil.getStackTrace(interrupted)); 139 } 140 141 boolean failedStatus = false; 142 String stderrText; 143 try { 144 stderrText = FileUtil.readStringFromFile(mStderrFile); 145 } catch (IOException e) { 146 stderrText = "Could not read the stderr output from process."; 147 } 148 if (!CommandStatus.SUCCESS.equals(result.getStatus())) { 149 failedStatus = true; 150 result.setStderr(stderrText); 151 } 152 153 try { 154 boolean joinResult = false; 155 long waitTime = getSandboxOptions(config).getWaitForEventsTimeout(); 156 if (mProtoReceiver != null) { 157 joinResult = mProtoReceiver.joinReceiver(waitTime); 158 } else { 159 joinResult = mEventParser.joinReceiver(waitTime); 160 } 161 if (interruptedException != null) { 162 throw interruptedException; 163 } 164 if (!joinResult) { 165 if (!failedStatus) { 166 result.setStatus(CommandStatus.EXCEPTION); 167 } 168 result.setStderr( 169 String.format("Event receiver thread did not complete.:\n%s", stderrText)); 170 } 171 PrettyPrintDelimiter.printStageDelimiter( 172 String.format( 173 "Execution of the tests occurred in the sandbox, you can find its logs " 174 + "under the name pattern '%s*'", 175 SANDBOX_PREFIX)); 176 } finally { 177 if (mProtoReceiver != null) { 178 mProtoReceiver.completeModuleEvents(); 179 } 180 // Log the configuration used to run 181 try (InputStreamSource configFile = 182 new FileInputStreamSource(mSerializedConfiguration)) { 183 logger.testLog("sandbox-config", LogDataType.XML, configFile); 184 } 185 try (InputStreamSource contextFile = new FileInputStreamSource(mSerializedContext)) { 186 logger.testLog("sandbox-context", LogDataType.PB, contextFile); 187 } 188 // Log stdout and stderr 189 if (mStdoutFile != null) { 190 try (InputStreamSource sourceStdOut = new FileInputStreamSource(mStdoutFile)) { 191 logger.testLog("sandbox-stdout", LogDataType.TEXT, sourceStdOut); 192 } 193 } 194 try (InputStreamSource sourceStdErr = new FileInputStreamSource(mStderrFile)) { 195 logger.testLog("sandbox-stderr", LogDataType.TEXT, sourceStdErr); 196 } 197 // Collect heap dump if any 198 logAndCleanHeapDump(mHeapDump, logger); 199 mHeapDump = null; 200 } 201 202 return result; 203 } 204 205 @Override prepareEnvironment( IInvocationContext context, IConfiguration config, ITestInvocationListener listener)206 public Exception prepareEnvironment( 207 IInvocationContext context, IConfiguration config, ITestInvocationListener listener) { 208 // Check for local sharding, avoid redirecting several stdout (from each shards) to the 209 // sandbox stdout as it creates a lot of I/O to the same output. 210 if (config.getCommandOptions().getShardCount() != null 211 && config.getCommandOptions().getShardIndex() == null) { 212 mCollectStdout = false; 213 } 214 // Create our temp directories. 215 try { 216 if (mCollectStdout) { 217 mStdoutFile = 218 FileUtil.createTempFile("stdout_subprocess_", ".log", getWorkFolder()); 219 mStdout = new FileOutputStream(mStdoutFile); 220 } else { 221 mStdout = 222 new OutputStream() { 223 @Override 224 public void write(int b) throws IOException { 225 // Ignore stdout 226 } 227 }; 228 } 229 230 mStderrFile = FileUtil.createTempFile("stderr_subprocess_", ".log", getWorkFolder()); 231 mStderr = new FileOutputStream(mStderrFile); 232 233 mSandboxTmpFolder = FileUtil.createTempDir("tradefed-container", getWorkFolder()); 234 } catch (IOException e) { 235 return e; 236 } 237 // Unset the current global environment 238 mRunUtil = createRunUtil(); 239 mRunUtil.unsetEnvVariable(GlobalConfiguration.GLOBAL_CONFIG_VARIABLE); 240 mRunUtil.unsetEnvVariable(GlobalConfiguration.GLOBAL_CONFIG_SERVER_CONFIG_VARIABLE); 241 // TODO: add handling of setting and creating the subprocess global configuration 242 if (getSandboxOptions(config).shouldEnableDebugThread()) { 243 mRunUtil.setEnvVariable(TradefedSandboxRunner.DEBUG_THREAD_KEY, "true"); 244 } 245 for (Entry<String, String> envEntry : 246 getSandboxOptions(config).getEnvVariables().entrySet()) { 247 mRunUtil.setEnvVariable(envEntry.getKey(), envEntry.getValue()); 248 } 249 250 try { 251 mRootFolder = 252 getTradefedSandboxEnvironment( 253 context, 254 config, 255 QuotationAwareTokenizer.tokenizeLine( 256 config.getCommandLine(), 257 /** no logging */ 258 false)); 259 } catch (ConfigurationException e) { 260 return e; 261 } 262 263 PrettyPrintDelimiter.printStageDelimiter("Sandbox Configuration Preparation"); 264 // Prepare the configuration 265 Exception res = prepareConfiguration(context, config, listener); 266 if (res != null) { 267 return res; 268 } 269 // Prepare the context 270 try { 271 mSerializedContext = prepareContext(context, config); 272 } catch (IOException e) { 273 return e; 274 } 275 276 return null; 277 } 278 279 @Override tearDown()280 public void tearDown() { 281 StreamUtil.close(mEventParser); 282 StreamUtil.close(mProtoReceiver); 283 StreamUtil.close(mStdout); 284 StreamUtil.close(mStderr); 285 FileUtil.deleteFile(mStdoutFile); 286 FileUtil.deleteFile(mStderrFile); 287 FileUtil.recursiveDelete(mSandboxTmpFolder); 288 FileUtil.deleteFile(mSerializedContext); 289 FileUtil.deleteFile(mSerializedConfiguration); 290 FileUtil.deleteFile(mGlobalConfig); 291 } 292 293 @Override getTradefedSandboxEnvironment( IInvocationContext context, IConfiguration nonVersionedConfig, String[] args)294 public File getTradefedSandboxEnvironment( 295 IInvocationContext context, IConfiguration nonVersionedConfig, String[] args) 296 throws ConfigurationException { 297 SandboxOptions options = getSandboxOptions(nonVersionedConfig); 298 // Check that we have no args conflicts. 299 if (options.getSandboxTfDirectory() != null && options.getSandboxBuildId() != null) { 300 throw new ConfigurationException( 301 String.format( 302 "Sandbox options %s and %s cannot be set at the same time", 303 SandboxOptions.TF_LOCATION, SandboxOptions.SANDBOX_BUILD_ID)); 304 } 305 306 if (options.getSandboxTfDirectory() != null) { 307 return options.getSandboxTfDirectory(); 308 } 309 String tfDir = System.getProperty("TF_JAR_DIR"); 310 if (tfDir == null || tfDir.isEmpty()) { 311 throw new ConfigurationException( 312 "Could not read TF_JAR_DIR to get current Tradefed instance."); 313 } 314 return new File(tfDir); 315 } 316 317 /** 318 * Create a classpath based on the environment and the working directory returned by {@link 319 * #getTradefedSandboxEnvironment(IInvocationContext, IConfiguration, String[])}. 320 * 321 * @param workingDir the current working directory for the sandbox. 322 * @return The classpath to be use. 323 */ 324 @Override createClasspath(File workingDir)325 public String createClasspath(File workingDir) throws ConfigurationException { 326 // Get the classpath property. 327 String classpathStr = System.getProperty("java.class.path"); 328 if (classpathStr == null) { 329 throw new ConfigurationException( 330 "Could not find the classpath property: java.class.path"); 331 } 332 return classpathStr; 333 } 334 335 /** 336 * Prepare the {@link IConfiguration} that will be passed to the subprocess and will drive the 337 * container execution. 338 * 339 * @param context The current {@link IInvocationContext}. 340 * @param config the {@link IConfiguration} to be prepared. 341 * @param listener The current invocation {@link ITestInvocationListener}. 342 * @return an Exception if anything went wrong, null otherwise. 343 */ prepareConfiguration( IInvocationContext context, IConfiguration config, ITestInvocationListener listener)344 protected Exception prepareConfiguration( 345 IInvocationContext context, IConfiguration config, ITestInvocationListener listener) { 346 try { 347 // TODO: switch reporting of parent and subprocess to proto 348 String commandLine = config.getCommandLine(); 349 if (getSandboxOptions(config).shouldUseProtoReporter()) { 350 mProtoReceiver = 351 new StreamProtoReceiver(listener, context, false, false, SANDBOX_PREFIX); 352 // Force the child to the same mode as the parent. 353 commandLine = commandLine + " --" + SandboxOptions.USE_PROTO_REPORTER; 354 } else { 355 mEventParser = new SubprocessTestResultsParser(listener, true, context); 356 commandLine = commandLine + " --no-" + SandboxOptions.USE_PROTO_REPORTER; 357 } 358 String[] args = 359 QuotationAwareTokenizer.tokenizeLine(commandLine, /* No Logging */ false); 360 mGlobalConfig = dumpGlobalConfig(config, new HashSet<>()); 361 try (InputStreamSource source = new FileInputStreamSource(mGlobalConfig)) { 362 listener.testLog("sandbox-global-config", LogDataType.XML, source); 363 } 364 DumpCmd mode = DumpCmd.RUN_CONFIG; 365 if (config.getCommandOptions().shouldUseSandboxTestMode()) { 366 mode = DumpCmd.TEST_MODE; 367 } 368 369 try { 370 mSerializedConfiguration = 371 SandboxConfigUtil.dumpConfigForVersion( 372 createClasspath(mRootFolder), mRunUtil, args, mode, mGlobalConfig); 373 } catch (SandboxConfigurationException e) { 374 // TODO: Improve our detection of that scenario 375 CLog.e(e); 376 CLog.e("%s", args[0]); 377 if (e.getMessage().contains(String.format("Can not find local config %s", args[0])) 378 || e.getMessage() 379 .contains( 380 String.format( 381 "Could not find configuration '%s'", args[0]))) { 382 CLog.w( 383 "Child version doesn't contains '%s'. Attempting to backfill missing parent configuration.", 384 args[0]); 385 File parentConfig = handleChildMissingConfig(args); 386 if (parentConfig != null) { 387 try (InputStreamSource source = new FileInputStreamSource(parentConfig)) { 388 listener.testLog("sandbox-parent-config", LogDataType.XML, source); 389 } 390 try { 391 mSerializedConfiguration = 392 SandboxConfigUtil.dumpConfigForVersion( 393 createClasspath(mRootFolder), 394 mRunUtil, 395 new String[] {parentConfig.getAbsolutePath()}, 396 mode, 397 mGlobalConfig); 398 } finally { 399 FileUtil.deleteFile(parentConfig); 400 } 401 return null; 402 } 403 } 404 throw e; 405 } 406 // Turn off some of the invocation level options that would be duplicated in the 407 // child sandbox subprocess. 408 config.getCommandOptions().setBugreportOnInvocationEnded(false); 409 config.getCommandOptions().setBugreportzOnInvocationEnded(false); 410 } catch (IOException | ConfigurationException e) { 411 StreamUtil.close(mEventParser); 412 StreamUtil.close(mProtoReceiver); 413 return e; 414 } 415 return null; 416 } 417 418 @VisibleForTesting createRunUtil()419 IRunUtil createRunUtil() { 420 return new RunUtil(); 421 } 422 423 /** 424 * Prepare and serialize the {@link IInvocationContext}. 425 * 426 * @param context the {@link IInvocationContext} to be prepared. 427 * @param config The {@link IConfiguration} of the sandbox. 428 * @return the serialized {@link IInvocationContext}. 429 * @throws IOException 430 */ prepareContext(IInvocationContext context, IConfiguration config)431 protected File prepareContext(IInvocationContext context, IConfiguration config) 432 throws IOException { 433 // In test mode we need to keep the context unlocked for the next layer. 434 if (config.getCommandOptions().shouldUseSandboxTestMode()) { 435 try { 436 Method unlock = InvocationContext.class.getDeclaredMethod("unlock"); 437 unlock.setAccessible(true); 438 unlock.invoke(context); 439 unlock.setAccessible(false); 440 } catch (NoSuchMethodException 441 | SecurityException 442 | IllegalAccessException 443 | IllegalArgumentException 444 | InvocationTargetException e) { 445 throw new IOException("Couldn't unlock the context.", e); 446 } 447 } 448 File protoFile = 449 FileUtil.createTempFile( 450 "context-proto", "." + LogDataType.PB.getFileExt(), mSandboxTmpFolder); 451 Context contextProto = context.toProto(); 452 contextProto.writeDelimitedTo(new FileOutputStream(protoFile)); 453 return protoFile; 454 } 455 456 /** Dump the global configuration filtered from some objects. */ dumpGlobalConfig(IConfiguration config, Set<String> exclusionPatterns)457 protected File dumpGlobalConfig(IConfiguration config, Set<String> exclusionPatterns) 458 throws IOException, ConfigurationException { 459 SandboxOptions options = getSandboxOptions(config); 460 if (options.getChildGlobalConfig() != null) { 461 IConfigurationFactory factory = ConfigurationFactory.getInstance(); 462 IGlobalConfiguration globalConfig = 463 factory.createGlobalConfigurationFromArgs( 464 new String[] {options.getChildGlobalConfig()}, new ArrayList<>()); 465 CLog.d( 466 "Using %s directly as global config without filtering", 467 options.getChildGlobalConfig()); 468 return globalConfig.cloneConfigWithFilter(); 469 } 470 return SandboxConfigUtil.dumpFilteredGlobalConfig(exclusionPatterns); 471 } 472 473 /** {@inheritDoc} */ 474 @Override createThinLauncherConfig( String[] args, IKeyStoreClient keyStoreClient, IRunUtil runUtil, File globalConfig)475 public IConfiguration createThinLauncherConfig( 476 String[] args, IKeyStoreClient keyStoreClient, IRunUtil runUtil, File globalConfig) { 477 // Default thin launcher cannot do anything, since this sandbox uses the same version as 478 // the parent version. 479 return null; 480 } 481 getSandboxOptions(IConfiguration config)482 private SandboxOptions getSandboxOptions(IConfiguration config) { 483 return (SandboxOptions) 484 config.getConfigurationObject(Configuration.SANBOX_OPTIONS_TYPE_NAME); 485 } 486 handleChildMissingConfig(String[] args)487 private File handleChildMissingConfig(String[] args) { 488 IConfiguration parentConfig = null; 489 File tmpParentConfig = null; 490 PrintWriter pw = null; 491 try { 492 tmpParentConfig = FileUtil.createTempFile("parent-config", ".xml", mSandboxTmpFolder); 493 pw = new PrintWriter(tmpParentConfig); 494 parentConfig = ConfigurationFactory.getInstance().createConfigurationFromArgs(args); 495 // Do not print deprecated options to avoid compatibility issues, and do not print 496 // unchanged options. 497 parentConfig.dumpXml(pw, new ArrayList<>(), false, false); 498 return tmpParentConfig; 499 } catch (ConfigurationException | IOException e) { 500 CLog.e("Parent doesn't understand the command either:"); 501 CLog.e(e); 502 FileUtil.deleteFile(tmpParentConfig); 503 return null; 504 } finally { 505 StreamUtil.close(pw); 506 } 507 } 508 logAndCleanHeapDump(File heapDumpDir, ITestLogger logger)509 private void logAndCleanHeapDump(File heapDumpDir, ITestLogger logger) { 510 try { 511 if (heapDumpDir != null && heapDumpDir.listFiles().length != 0) { 512 for (File f : heapDumpDir.listFiles()) { 513 FileInputStreamSource fileInput = new FileInputStreamSource(f); 514 logger.testLog(f.getName(), LogDataType.HPROF, fileInput); 515 StreamUtil.cancel(fileInput); 516 } 517 } 518 } finally { 519 FileUtil.recursiveDelete(heapDumpDir); 520 } 521 } 522 getWorkFolder()523 private File getWorkFolder() { 524 File workfolder = CurrentInvocation.getInfo(InvocationInfo.WORK_FOLDER); 525 if (workfolder == null || !workfolder.exists()) { 526 return null; 527 } 528 return workfolder; 529 } 530 } 531