1 /* 2 * Copyright (C) 2018 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package com.android.tradefed.device.cloud; 17 18 import com.android.tradefed.build.IBuildInfo; 19 import com.android.tradefed.command.remote.DeviceDescriptor; 20 import com.android.tradefed.device.TestDeviceOptions; 21 import com.android.tradefed.device.cloud.AcloudConfigParser.AcloudKeys; 22 import com.android.tradefed.device.cloud.GceAvdInfo.GceStatus; 23 import com.android.tradefed.log.ITestLogger; 24 import com.android.tradefed.log.LogUtil.CLog; 25 import com.android.tradefed.result.ByteArrayInputStreamSource; 26 import com.android.tradefed.result.FileInputStreamSource; 27 import com.android.tradefed.result.InputStreamSource; 28 import com.android.tradefed.result.LogDataType; 29 import com.android.tradefed.result.error.DeviceErrorIdentifier; 30 import com.android.tradefed.targetprep.TargetSetupError; 31 import com.android.tradefed.util.ArrayUtil; 32 import com.android.tradefed.util.CommandResult; 33 import com.android.tradefed.util.CommandStatus; 34 import com.android.tradefed.util.FileUtil; 35 import com.android.tradefed.util.GoogleApiClientUtil; 36 import com.android.tradefed.util.IRunUtil; 37 import com.android.tradefed.util.RunUtil; 38 39 import com.google.api.client.auth.oauth2.Credential; 40 import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport; 41 import com.google.api.client.json.JsonFactory; 42 import com.google.api.client.json.jackson2.JacksonFactory; 43 import com.google.api.services.compute.Compute; 44 import com.google.api.services.compute.Compute.Instances.GetSerialPortOutput; 45 import com.google.api.services.compute.ComputeScopes; 46 import com.google.api.services.compute.model.SerialPortOutput; 47 import com.google.common.annotations.VisibleForTesting; 48 import com.google.common.net.HostAndPort; 49 50 import java.io.File; 51 import java.io.IOException; 52 import java.lang.ProcessBuilder.Redirect; 53 import java.security.GeneralSecurityException; 54 import java.time.Duration; 55 import java.util.Arrays; 56 import java.util.List; 57 import java.util.regex.Matcher; 58 import java.util.regex.Pattern; 59 60 /** Helper that manages the GCE calls to start/stop and collect logs from GCE. */ 61 public class GceManager { 62 public static final String GCE_INSTANCE_NAME_KEY = "gce-instance-name"; 63 public static final String GCE_INSTANCE_CLEANED_KEY = "gce-instance-clean-called"; 64 65 private static final long BUGREPORT_TIMEOUT = 15 * 60 * 1000L; 66 private static final long REMOTE_FILE_OP_TIMEOUT = 10 * 60 * 1000L; 67 private static final Pattern BUGREPORTZ_RESPONSE_PATTERN = Pattern.compile("(OK:)(.*)"); 68 private static final JsonFactory JSON_FACTORY = JacksonFactory.getDefaultInstance(); 69 private static final List<String> SCOPES = Arrays.asList(ComputeScopes.COMPUTE_READONLY); 70 71 private DeviceDescriptor mDeviceDescriptor; 72 private TestDeviceOptions mDeviceOptions; 73 private IBuildInfo mBuildInfo; 74 75 private String mGceInstanceName = null; 76 private String mGceHost = null; 77 private GceAvdInfo mGceAvdInfo = null; 78 79 /** 80 * Ctor 81 * 82 * @param deviceDesc The {@link DeviceDescriptor} that will be associated with the GCE device. 83 * @param deviceOptions A {@link TestDeviceOptions} associated with the device. 84 * @param buildInfo A {@link IBuildInfo} describing the gce build to start. 85 */ GceManager( DeviceDescriptor deviceDesc, TestDeviceOptions deviceOptions, IBuildInfo buildInfo)86 public GceManager( 87 DeviceDescriptor deviceDesc, TestDeviceOptions deviceOptions, IBuildInfo buildInfo) { 88 mDeviceDescriptor = deviceDesc; 89 mDeviceOptions = deviceOptions; 90 mBuildInfo = buildInfo; 91 92 if (!deviceOptions.allowGceCmdTimeoutOverride()) { 93 return; 94 } 95 int index = deviceOptions.getGceDriverParams().lastIndexOf("--boot-timeout"); 96 if (index != -1 && deviceOptions.getGceDriverParams().size() > index + 1) { 97 String driverTimeoutStringSec = deviceOptions.getGceDriverParams().get(index + 1); 98 try { 99 // Add some extra time on top of Acloud: acloud boot the device then we expect 100 // the Tradefed online check to take a bit of time, use 3min as a safe overhead 101 long driverTimeoutMs = 102 Long.parseLong(driverTimeoutStringSec) * 1000 + 3 * 60 * 1000; 103 long gceCmdTimeoutMs = deviceOptions.getGceCmdTimeout(); 104 deviceOptions.setGceCmdTimeout(driverTimeoutMs); 105 CLog.i( 106 "Replacing --gce-boot-timeout %s by --boot-timeout %s.", 107 gceCmdTimeoutMs, driverTimeoutMs); 108 } catch (NumberFormatException e) { 109 CLog.e(e); 110 } 111 } 112 } 113 114 /** @deprecated Use other constructors, we keep this temporarily for backward compatibility. */ 115 @Deprecated GceManager( DeviceDescriptor deviceDesc, TestDeviceOptions deviceOptions, IBuildInfo buildInfo, List<IBuildInfo> testResourceBuildInfos)116 public GceManager( 117 DeviceDescriptor deviceDesc, 118 TestDeviceOptions deviceOptions, 119 IBuildInfo buildInfo, 120 List<IBuildInfo> testResourceBuildInfos) { 121 this(deviceDesc, deviceOptions, buildInfo); 122 } 123 124 /** 125 * Ctor, variation that can be used to provide the GCE instance name to use directly. 126 * 127 * @param deviceDesc The {@link DeviceDescriptor} that will be associated with the GCE device. 128 * @param deviceOptions A {@link TestDeviceOptions} associated with the device 129 * @param buildInfo A {@link IBuildInfo} describing the gce build to start. 130 * @param gceInstanceName The instance name to use. 131 * @param gceHost The host name or ip of the instance to use. 132 */ GceManager( DeviceDescriptor deviceDesc, TestDeviceOptions deviceOptions, IBuildInfo buildInfo, String gceInstanceName, String gceHost)133 public GceManager( 134 DeviceDescriptor deviceDesc, 135 TestDeviceOptions deviceOptions, 136 IBuildInfo buildInfo, 137 String gceInstanceName, 138 String gceHost) { 139 this(deviceDesc, deviceOptions, buildInfo); 140 mGceInstanceName = gceInstanceName; 141 mGceHost = gceHost; 142 } 143 startGce()144 public GceAvdInfo startGce() throws TargetSetupError { 145 return startGce(null); 146 } 147 148 /** 149 * Attempt to start a gce instance 150 * 151 * @return a {@link GceAvdInfo} describing the GCE instance. Could be a BOOT_FAIL instance. 152 * @throws TargetSetupError 153 */ startGce(String ipDevice)154 public GceAvdInfo startGce(String ipDevice) throws TargetSetupError { 155 mGceAvdInfo = null; 156 // For debugging purposes bypass. 157 if (mGceHost != null && mGceInstanceName != null) { 158 mGceAvdInfo = 159 new GceAvdInfo( 160 mGceInstanceName, 161 HostAndPort.fromString(mGceHost) 162 .withDefaultPort(mDeviceOptions.getRemoteAdbPort())); 163 return mGceAvdInfo; 164 } 165 // Add extra args. 166 File reportFile = null; 167 try { 168 reportFile = FileUtil.createTempFile("gce_avd_driver", ".json"); 169 List<String> gceArgs = buildGceCmd(reportFile, mBuildInfo, ipDevice); 170 171 long driverTimeoutMs = getTestDeviceOptions().getGceCmdTimeout(); 172 if (!getTestDeviceOptions().allowGceCmdTimeoutOverride()) { 173 long driverTimeoutSec = 174 Duration.ofMillis(driverTimeoutMs - 3 * 60 * 1000).toSeconds(); 175 // --boot-timeout takes a value in seconds 176 gceArgs.add("--boot-timeout"); 177 gceArgs.add(Long.toString(driverTimeoutSec)); 178 driverTimeoutMs = driverTimeoutSec * 1000; 179 } 180 181 CLog.i("Launching GCE with %s", gceArgs.toString()); 182 CommandResult cmd = 183 getRunUtil() 184 .runTimedCmd( 185 getTestDeviceOptions().getGceCmdTimeout(), 186 gceArgs.toArray(new String[gceArgs.size()])); 187 CLog.i("GCE driver stderr: %s", cmd.getStderr()); 188 String instanceName = extractInstanceName(cmd.getStderr()); 189 if (instanceName != null) { 190 mBuildInfo.addBuildAttribute(GCE_INSTANCE_NAME_KEY, instanceName); 191 } else { 192 CLog.w("Could not extract an instance name for the gce device."); 193 } 194 if (CommandStatus.TIMED_OUT.equals(cmd.getStatus())) { 195 String errors = 196 String.format( 197 "acloud errors: timeout after %dms, acloud did not return", 198 driverTimeoutMs); 199 if (instanceName != null) { 200 // If we managed to parse the instance name, report the boot failure so it 201 // can be shutdown. 202 mGceAvdInfo = new GceAvdInfo(instanceName, null, errors, GceStatus.BOOT_FAIL); 203 return mGceAvdInfo; 204 } 205 throw new TargetSetupError(errors, mDeviceDescriptor); 206 } else if (!CommandStatus.SUCCESS.equals(cmd.getStatus())) { 207 CLog.w("Error when booting the Gce instance, reading output of gce driver"); 208 mGceAvdInfo = 209 GceAvdInfo.parseGceInfoFromFile( 210 reportFile, mDeviceDescriptor, mDeviceOptions.getRemoteAdbPort()); 211 String errors = ""; 212 if (mGceAvdInfo != null) { 213 // We always return the GceAvdInfo describing the instance when possible 214 // The caller can decide actions to be taken. 215 return mGceAvdInfo; 216 } else { 217 errors = 218 "Could not get a valid instance name, check the gce driver's output." 219 + "The instance may not have booted up at all."; 220 CLog.e(errors); 221 throw new TargetSetupError( 222 String.format("acloud errors: %s", errors), 223 mDeviceDescriptor, 224 DeviceErrorIdentifier.FAILED_TO_LAUNCH_GCE); 225 } 226 } 227 mGceAvdInfo = 228 GceAvdInfo.parseGceInfoFromFile( 229 reportFile, mDeviceDescriptor, mDeviceOptions.getRemoteAdbPort()); 230 return mGceAvdInfo; 231 } catch (IOException e) { 232 throw new TargetSetupError("failed to create log file", e, mDeviceDescriptor); 233 } finally { 234 FileUtil.deleteFile(reportFile); 235 } 236 } 237 238 /** 239 * Retrieve the instance name from the gce boot logs. Search for the 'name': 'gce-<name>' 240 * pattern to extract the name of it. We extract from the logs instead of result file because on 241 * gce boot failure, the attempted instance name won't show in json. 242 */ extractInstanceName(String bootupLogs)243 protected String extractInstanceName(String bootupLogs) { 244 if (bootupLogs != null) { 245 final String pattern = "'name': u?'(((gce-)|(ins-))(.*?))'"; 246 Pattern namePattern = Pattern.compile(pattern); 247 Matcher matcher = namePattern.matcher(bootupLogs); 248 if (matcher.find()) { 249 return matcher.group(1); 250 } 251 } 252 return null; 253 } 254 255 /** Build and return the command to launch GCE. Exposed for testing. */ buildGceCmd(File reportFile, IBuildInfo b, String ipDevice)256 protected List<String> buildGceCmd(File reportFile, IBuildInfo b, String ipDevice) { 257 File avdDriverFile = getTestDeviceOptions().getAvdDriverBinary(); 258 if (!avdDriverFile.exists()) { 259 throw new RuntimeException( 260 String.format( 261 "Could not find the Acloud driver at %s", 262 avdDriverFile.getAbsolutePath())); 263 } 264 if (!avdDriverFile.canExecute()) { 265 // Set the executable bit if needed 266 FileUtil.chmodGroupRWX(avdDriverFile); 267 } 268 List<String> gceArgs = ArrayUtil.list(avdDriverFile.getAbsolutePath()); 269 gceArgs.add( 270 TestDeviceOptions.getCreateCommandByInstanceType( 271 getTestDeviceOptions().getInstanceType())); 272 // Handle the build id related params 273 List<String> gceDriverParams = getTestDeviceOptions().getGceDriverParams(); 274 275 if (TestDeviceOptions.InstanceType.CHEEPS.equals( 276 getTestDeviceOptions().getInstanceType())) { 277 gceArgs.add("--avd-type"); 278 gceArgs.add("cheeps"); 279 280 if (getTestDeviceOptions().getCrosUser() != null 281 && getTestDeviceOptions().getCrosPassword() != null) { 282 gceArgs.add("--user"); 283 gceArgs.add(getTestDeviceOptions().getCrosUser()); 284 gceArgs.add("--password"); 285 gceArgs.add(getTestDeviceOptions().getCrosPassword()); 286 } 287 } 288 289 // If args passed by gce-driver-param do not contain build_id or branch, 290 // use build_id and branch from device BuildInfo 291 if (!gceDriverParams.contains("--build_id") && !gceDriverParams.contains("--branch")) { 292 gceArgs.add("--build_target"); 293 if (b.getBuildAttributes().containsKey("build_target")) { 294 // If BuildInfo contains the attribute for a build target, use that. 295 gceArgs.add(b.getBuildAttributes().get("build_target")); 296 } else { 297 gceArgs.add(b.getBuildFlavor()); 298 } 299 gceArgs.add("--branch"); 300 gceArgs.add(b.getBuildBranch()); 301 gceArgs.add("--build_id"); 302 gceArgs.add(b.getBuildId()); 303 } 304 // Add additional args passed by gce-driver-param. 305 gceArgs.addAll(gceDriverParams); 306 // Get extra params by instance type 307 gceArgs.addAll( 308 TestDeviceOptions.getExtraParamsByInstanceType( 309 getTestDeviceOptions().getInstanceType(), 310 getTestDeviceOptions().getBaseImage())); 311 if (ipDevice == null) { 312 gceArgs.add("--config_file"); 313 gceArgs.add(getAvdConfigFile().getAbsolutePath()); 314 if (getTestDeviceOptions().getServiceAccountJsonKeyFile() != null) { 315 gceArgs.add("--service_account_json_private_key_path"); 316 gceArgs.add( 317 getTestDeviceOptions().getServiceAccountJsonKeyFile().getAbsolutePath()); 318 } 319 } else { 320 gceArgs.add("--host"); 321 gceArgs.add(ipDevice); 322 } 323 gceArgs.add("--report_file"); 324 gceArgs.add(reportFile.getAbsolutePath()); 325 switch (getTestDeviceOptions().getGceDriverLogLevel()) { 326 case DEBUG: 327 gceArgs.add("-v"); 328 break; 329 case VERBOSE: 330 gceArgs.add("-vv"); 331 break; 332 default: 333 break; 334 } 335 if (getTestDeviceOptions().getGceAccount() != null) { 336 gceArgs.add("--email"); 337 gceArgs.add(getTestDeviceOptions().getGceAccount()); 338 } 339 // Do not pass flags --logcat_file and --serial_log_file to collect logcat and serial logs. 340 341 return gceArgs; 342 } 343 344 /** 345 * Shutdown the Gce instance associated with the {@link #startGce()}. 346 * 347 * @return returns true if gce shutdown was requested as non-blocking. 348 */ shutdownGce()349 public boolean shutdownGce() { 350 if (!getTestDeviceOptions().getAvdDriverBinary().canExecute()) { 351 mGceAvdInfo = null; 352 throw new RuntimeException( 353 String.format( 354 "GCE launcher %s is invalid", 355 getTestDeviceOptions().getAvdDriverBinary())); 356 } 357 String instanceName = null; 358 boolean notFromGceAvd = false; 359 if (mGceAvdInfo != null) { 360 instanceName = mGceAvdInfo.instanceName(); 361 } 362 if (instanceName == null) { 363 instanceName = mBuildInfo.getBuildAttributes().get(GCE_INSTANCE_NAME_KEY); 364 notFromGceAvd = true; 365 } 366 if (instanceName == null) { 367 CLog.d("No instance to shutdown."); 368 return false; 369 } 370 try { 371 boolean res = AcloudShutdown(getTestDeviceOptions(), getRunUtil(), instanceName); 372 // Be more lenient if instance name was not reported officially and we still attempt 373 // to clean it. 374 if (res || notFromGceAvd) { 375 mBuildInfo.addBuildAttribute(GCE_INSTANCE_CLEANED_KEY, "true"); 376 } 377 return res; 378 } finally { 379 mGceAvdInfo = null; 380 } 381 } 382 383 /** 384 * Actual Acloud run to shutdown the virtual device. 385 * 386 * @param options The {@link TestDeviceOptions} for the Acloud options 387 * @param runUtil The {@link IRunUtil} to run Acloud 388 * @param instanceName The instance to shutdown. 389 * @return True if successful 390 */ AcloudShutdown( TestDeviceOptions options, IRunUtil runUtil, String instanceName)391 public static boolean AcloudShutdown( 392 TestDeviceOptions options, IRunUtil runUtil, String instanceName) { 393 List<String> gceArgs = ArrayUtil.list(options.getAvdDriverBinary().getAbsolutePath()); 394 gceArgs.add("delete"); 395 // Add extra args. 396 File f = null; 397 File config = null; 398 try { 399 config = FileUtil.createTempFile(options.getAvdConfigFile().getName(), "config"); 400 gceArgs.add("--instance_names"); 401 gceArgs.add(instanceName); 402 gceArgs.add("--config_file"); 403 // Copy the config in case it comes from a dynamic file. In order to ensure Acloud has 404 // the file until it's done with it. 405 FileUtil.copyFile(options.getAvdConfigFile(), config); 406 gceArgs.add(config.getAbsolutePath()); 407 if (options.getServiceAccountJsonKeyFile() != null) { 408 gceArgs.add("--service_account_json_private_key_path"); 409 gceArgs.add(options.getServiceAccountJsonKeyFile().getAbsolutePath()); 410 } 411 f = FileUtil.createTempFile("gce_avd_driver", ".json"); 412 gceArgs.add("--report_file"); 413 gceArgs.add(f.getAbsolutePath()); 414 CLog.i("Tear down of GCE with %s", gceArgs.toString()); 415 if (options.waitForGceTearDown()) { 416 CommandResult cmd = 417 runUtil.runTimedCmd( 418 options.getGceCmdTimeout(), 419 gceArgs.toArray(new String[gceArgs.size()])); 420 FileUtil.deleteFile(config); 421 if (!CommandStatus.SUCCESS.equals(cmd.getStatus())) { 422 CLog.w( 423 "Failed to tear down GCE %s with the following arg: %s." 424 + "\nstdout:%s\nstderr:%s", 425 instanceName, gceArgs, cmd.getStdout(), cmd.getStderr()); 426 return false; 427 } 428 } else { 429 // Discard the output so the process is not linked to the parent and doesn't die 430 // if the JVM exit. 431 Process p = runUtil.runCmdInBackground(Redirect.DISCARD, gceArgs); 432 AcloudDeleteCleaner cleaner = new AcloudDeleteCleaner(p, config); 433 cleaner.start(); 434 } 435 } catch (IOException | RuntimeException e) { 436 CLog.e("failed to create log file for GCE Teardown"); 437 CLog.e(e); 438 FileUtil.deleteFile(config); 439 return false; 440 } finally { 441 FileUtil.deleteFile(f); 442 } 443 return true; 444 } 445 446 /** 447 * Get a bugreportz from the device using ssh to avoid any adb connection potential issue. 448 * 449 * @param gceAvd The {@link GceAvdInfo} that describe the device. 450 * @param options a {@link TestDeviceOptions} describing the device options to be used for the 451 * GCE device. 452 * @param runUtil a {@link IRunUtil} to execute commands. 453 * @return A file pointing to the zip bugreport, or null if an issue occurred. 454 * @throws IOException 455 */ getBugreportzWithSsh( GceAvdInfo gceAvd, TestDeviceOptions options, IRunUtil runUtil)456 public static File getBugreportzWithSsh( 457 GceAvdInfo gceAvd, TestDeviceOptions options, IRunUtil runUtil) throws IOException { 458 String output = remoteSshCommandExec(gceAvd, options, runUtil, "bugreportz"); 459 Matcher match = BUGREPORTZ_RESPONSE_PATTERN.matcher(output); 460 if (!match.find()) { 461 CLog.e("Something went wrong during bugreportz collection: '%s'", output); 462 return null; 463 } 464 String remoteFilePath = match.group(2); 465 File localTmpFile = FileUtil.createTempFile("bugreport-ssh", ".zip"); 466 if (!RemoteFileUtil.fetchRemoteFile( 467 gceAvd, options, runUtil, REMOTE_FILE_OP_TIMEOUT, remoteFilePath, localTmpFile)) { 468 FileUtil.deleteFile(localTmpFile); 469 return null; 470 } 471 return localTmpFile; 472 } 473 474 /** 475 * Get a bugreport via ssh for a nested instance. This requires requesting the adb in the nested 476 * virtual instance. 477 * 478 * @param gceAvd The {@link GceAvdInfo} that describe the device. 479 * @param options a {@link TestDeviceOptions} describing the device options to be used for the 480 * GCE device. 481 * @param runUtil a {@link IRunUtil} to execute commands. 482 * @return A file pointing to the zip bugreport, or null if an issue occurred. 483 * @throws IOException 484 */ getNestedDeviceSshBugreportz( GceAvdInfo gceAvd, TestDeviceOptions options, IRunUtil runUtil)485 public static File getNestedDeviceSshBugreportz( 486 GceAvdInfo gceAvd, TestDeviceOptions options, IRunUtil runUtil) throws IOException { 487 if (gceAvd == null || gceAvd.hostAndPort() == null) { 488 return null; 489 } 490 String output = 491 remoteSshCommandExec( 492 gceAvd, 493 options, 494 runUtil, 495 "./bin/adb", 496 "wait-for-device", 497 "shell", 498 "bugreportz"); 499 Matcher match = BUGREPORTZ_RESPONSE_PATTERN.matcher(output); 500 if (!match.find()) { 501 CLog.e("Something went wrong during bugreportz collection: '%s'", output); 502 return null; 503 } 504 String deviceFilePath = match.group(2); 505 String pullOutput = 506 remoteSshCommandExec(gceAvd, options, runUtil, "./bin/adb", "pull", deviceFilePath); 507 CLog.d(pullOutput); 508 String remoteFilePath = "./" + new File(deviceFilePath).getName(); 509 File localTmpFile = FileUtil.createTempFile("bugreport-ssh", ".zip"); 510 if (!RemoteFileUtil.fetchRemoteFile( 511 gceAvd, options, runUtil, REMOTE_FILE_OP_TIMEOUT, remoteFilePath, localTmpFile)) { 512 FileUtil.deleteFile(localTmpFile); 513 return null; 514 } 515 return localTmpFile; 516 } 517 518 /** 519 * Fetch a remote file from a nested instance and log it. 520 * 521 * @param logger The {@link ITestLogger} where to log the file. 522 * @param gceAvd The {@link GceAvdInfo} that describe the device. 523 * @param options a {@link TestDeviceOptions} describing the device options to be used for the 524 * GCE device. 525 * @param runUtil a {@link IRunUtil} to execute commands. 526 * @param remoteFilePath The remote path where to find the file. 527 * @param type the {@link LogDataType} of the logged file. 528 */ logNestedRemoteFile( ITestLogger logger, GceAvdInfo gceAvd, TestDeviceOptions options, IRunUtil runUtil, String remoteFilePath, LogDataType type)529 public static void logNestedRemoteFile( 530 ITestLogger logger, 531 GceAvdInfo gceAvd, 532 TestDeviceOptions options, 533 IRunUtil runUtil, 534 String remoteFilePath, 535 LogDataType type) { 536 logNestedRemoteFile(logger, gceAvd, options, runUtil, remoteFilePath, type, null); 537 } 538 539 /** 540 * Fetch a remote file from a nested instance and log it. 541 * 542 * @param logger The {@link ITestLogger} where to log the file. 543 * @param gceAvd The {@link GceAvdInfo} that describe the device. 544 * @param options a {@link TestDeviceOptions} describing the device options to be used for the 545 * GCE device. 546 * @param runUtil a {@link IRunUtil} to execute commands. 547 * @param remoteFilePath The remote path where to find the file. 548 * @param type the {@link LogDataType} of the logged file. 549 * @param baseName The base name to use to log the file. If null the actual file name will be 550 * used. 551 */ logNestedRemoteFile( ITestLogger logger, GceAvdInfo gceAvd, TestDeviceOptions options, IRunUtil runUtil, String remoteFilePath, LogDataType type, String baseName)552 public static void logNestedRemoteFile( 553 ITestLogger logger, 554 GceAvdInfo gceAvd, 555 TestDeviceOptions options, 556 IRunUtil runUtil, 557 String remoteFilePath, 558 LogDataType type, 559 String baseName) { 560 File remoteFile = 561 RemoteFileUtil.fetchRemoteFile( 562 gceAvd, options, runUtil, REMOTE_FILE_OP_TIMEOUT, remoteFilePath); 563 if (remoteFile != null) { 564 // If we happened to fetch a directory, log all the subfiles 565 logFile(remoteFile, baseName, logger, type); 566 } 567 } 568 logFile( File remoteFile, String baseName, ITestLogger logger, LogDataType type)569 private static void logFile( 570 File remoteFile, String baseName, ITestLogger logger, LogDataType type) { 571 if (remoteFile.isDirectory()) { 572 for (File f : remoteFile.listFiles()) { 573 logFile(f, null, logger, type); 574 } 575 } else { 576 try (InputStreamSource remoteFileStream = new FileInputStreamSource(remoteFile, true)) { 577 String name = baseName; 578 if (name == null) { 579 name = remoteFile.getName(); 580 } 581 logger.testLog(name, type, remoteFileStream); 582 } 583 } 584 } 585 586 /** 587 * Execute the remote command via ssh on an instance. 588 * 589 * @param gceAvd The {@link GceAvdInfo} that describe the device. 590 * @param options a {@link TestDeviceOptions} describing the device options to be used for the 591 * GCE device. 592 * @param runUtil a {@link IRunUtil} to execute commands. 593 * @param timeoutMs The timeout in millisecond for the command. 0 means no timeout. 594 * @param command The remote command to execute. 595 * @return {@link CommandResult} containing the result of the execution. 596 */ remoteSshCommandExecution( GceAvdInfo gceAvd, TestDeviceOptions options, IRunUtil runUtil, long timeoutMs, String... command)597 public static CommandResult remoteSshCommandExecution( 598 GceAvdInfo gceAvd, 599 TestDeviceOptions options, 600 IRunUtil runUtil, 601 long timeoutMs, 602 String... command) { 603 return RemoteSshUtil.remoteSshCommandExec(gceAvd, options, runUtil, timeoutMs, command); 604 } 605 remoteSshCommandExec( GceAvdInfo gceAvd, TestDeviceOptions options, IRunUtil runUtil, String... command)606 private static String remoteSshCommandExec( 607 GceAvdInfo gceAvd, TestDeviceOptions options, IRunUtil runUtil, String... command) { 608 CommandResult res = 609 remoteSshCommandExecution(gceAvd, options, runUtil, BUGREPORT_TIMEOUT, command); 610 // We attempt to get a clean output from our command 611 String output = res.getStdout().trim(); 612 if (!CommandStatus.SUCCESS.equals(res.getStatus())) { 613 CLog.e("issue when attempting to execute '%s':", Arrays.asList(command)); 614 CLog.e("Stderr: %s", res.getStderr()); 615 } else if (output.isEmpty()) { 616 CLog.e("Stdout from '%s' was empty", Arrays.asList(command)); 617 CLog.e("Stderr: %s", res.getStderr()); 618 } 619 return output; 620 } 621 622 /** 623 * Reads the current content of the Gce Avd instance serial log. 624 * 625 * @param infos The {@link GceAvdInfo} describing the instance. 626 * @param avdConfigFile the avd config file 627 * @param jsonKeyFile the service account json key file. 628 * @param runUtil a {@link IRunUtil} to execute commands. 629 * @return The serial log output or null if something goes wrong. 630 */ getInstanceSerialLog( GceAvdInfo infos, File avdConfigFile, File jsonKeyFile, IRunUtil runUtil)631 public static String getInstanceSerialLog( 632 GceAvdInfo infos, File avdConfigFile, File jsonKeyFile, IRunUtil runUtil) { 633 AcloudConfigParser config = AcloudConfigParser.parseConfig(avdConfigFile); 634 if (config == null) { 635 CLog.e("Failed to parse our acloud config."); 636 return null; 637 } 638 if (infos == null) { 639 return null; 640 } 641 try { 642 Credential credential = createCredential(config, jsonKeyFile); 643 String project = config.getValueForKey(AcloudKeys.PROJECT); 644 String zone = config.getValueForKey(AcloudKeys.ZONE); 645 String instanceName = infos.instanceName(); 646 Compute compute = 647 new Compute.Builder( 648 GoogleNetHttpTransport.newTrustedTransport(), 649 JSON_FACTORY, 650 null) 651 .setApplicationName(project) 652 .setHttpRequestInitializer(credential) 653 .build(); 654 GetSerialPortOutput outputPort = 655 compute.instances().getSerialPortOutput(project, zone, instanceName); 656 SerialPortOutput output = outputPort.execute(); 657 return output.getContents(); 658 } catch (GeneralSecurityException | IOException e) { 659 CLog.e(e); 660 return null; 661 } 662 } 663 createCredential(AcloudConfigParser config, File jsonKeyFile)664 private static Credential createCredential(AcloudConfigParser config, File jsonKeyFile) 665 throws GeneralSecurityException, IOException { 666 if (jsonKeyFile != null) { 667 return GoogleApiClientUtil.createCredentialFromJsonKeyFile(jsonKeyFile, SCOPES); 668 } else if (config.getValueForKey(AcloudKeys.SERVICE_ACCOUNT_JSON_PRIVATE_KEY) != null) { 669 jsonKeyFile = 670 new File(config.getValueForKey(AcloudKeys.SERVICE_ACCOUNT_JSON_PRIVATE_KEY)); 671 return GoogleApiClientUtil.createCredentialFromJsonKeyFile(jsonKeyFile, SCOPES); 672 } else { 673 String serviceAccount = config.getValueForKey(AcloudKeys.SERVICE_ACCOUNT_NAME); 674 String serviceKey = config.getValueForKey(AcloudKeys.SERVICE_ACCOUNT_PRIVATE_KEY); 675 return GoogleApiClientUtil.createCredentialFromP12File( 676 serviceAccount, new File(serviceKey), SCOPES); 677 } 678 } 679 cleanUp()680 public void cleanUp() { 681 // Clean up logs file if any was created. 682 } 683 684 /** Returns the instance of the {@link IRunUtil}. */ 685 @VisibleForTesting getRunUtil()686 IRunUtil getRunUtil() { 687 return RunUtil.getDefault(); 688 } 689 690 /** 691 * Log the serial output of a device described by {@link GceAvdInfo}. 692 * 693 * @param infos The {@link GceAvdInfo} describing the instance. 694 * @param logger The {@link ITestLogger} where to log the serial log. 695 */ logSerialOutput(GceAvdInfo infos, ITestLogger logger)696 public void logSerialOutput(GceAvdInfo infos, ITestLogger logger) { 697 String output = 698 GceManager.getInstanceSerialLog( 699 infos, 700 getAvdConfigFile(), 701 getTestDeviceOptions().getServiceAccountJsonKeyFile(), 702 getRunUtil()); 703 if (output == null) { 704 CLog.w("Failed to collect the instance serial logs."); 705 return; 706 } 707 try (ByteArrayInputStreamSource source = 708 new ByteArrayInputStreamSource(output.getBytes())) { 709 logger.testLog("gce_full_serial_log", LogDataType.TEXT, source); 710 } 711 } 712 713 /** Log the information related to the stable host image used. */ logStableHostImageInfos(IBuildInfo build)714 public void logStableHostImageInfos(IBuildInfo build) { 715 AcloudConfigParser config = AcloudConfigParser.parseConfig(getAvdConfigFile()); 716 if (config == null) { 717 CLog.e("Failed to parse our acloud config."); 718 return; 719 } 720 if (build == null) { 721 return; 722 } 723 if (config.getValueForKey(AcloudKeys.STABLE_HOST_IMAGE_NAME) != null) { 724 build.addBuildAttribute( 725 AcloudKeys.STABLE_HOST_IMAGE_NAME.toString(), 726 config.getValueForKey(AcloudKeys.STABLE_HOST_IMAGE_NAME)); 727 } 728 if (config.getValueForKey(AcloudKeys.STABLE_HOST_IMAGE_PROJECT) != null) { 729 build.addBuildAttribute( 730 AcloudKeys.STABLE_HOST_IMAGE_PROJECT.toString(), 731 config.getValueForKey(AcloudKeys.STABLE_HOST_IMAGE_PROJECT)); 732 } 733 } 734 735 /** 736 * Returns the {@link TestDeviceOptions} associated with the device that the gce manager was 737 * initialized with. 738 */ getTestDeviceOptions()739 private TestDeviceOptions getTestDeviceOptions() { 740 return mDeviceOptions; 741 } 742 743 @VisibleForTesting getAvdConfigFile()744 File getAvdConfigFile() { 745 return getTestDeviceOptions().getAvdConfigFile(); 746 } 747 748 /** 749 * Thread that helps cleaning the copied config when the process is done. This ensures acloud is 750 * not missing its config until its done. 751 */ 752 private static class AcloudDeleteCleaner extends Thread { 753 private Process mProcess; 754 private File mConfigFile; 755 AcloudDeleteCleaner(Process p, File config)756 public AcloudDeleteCleaner(Process p, File config) { 757 setDaemon(true); 758 setName("acloud-delete-cleaner"); 759 mProcess = p; 760 mConfigFile = config; 761 } 762 763 @Override run()764 public void run() { 765 try { 766 mProcess.waitFor(); 767 } catch (InterruptedException e) { 768 CLog.e(e); 769 } 770 FileUtil.deleteFile(mConfigFile); 771 } 772 } 773 } 774