1# Copyright 2016 - The Android Open Source Project 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14"""Common Utilities.""" 15# pylint: disable=too-many-lines 16from __future__ import print_function 17 18import base64 19import binascii 20import collections 21import errno 22import getpass 23import grp 24import logging 25import os 26import platform 27import shlex 28import shutil 29import signal 30import struct 31import socket 32import subprocess 33import sys 34import tarfile 35import tempfile 36import time 37import uuid 38import webbrowser 39import zipfile 40 41import six 42 43from acloud import errors 44from acloud.internal import constants 45 46 47logger = logging.getLogger(__name__) 48 49SSH_KEYGEN_CMD = ["ssh-keygen", "-t", "rsa", "-b", "4096"] 50SSH_KEYGEN_PUB_CMD = ["ssh-keygen", "-y"] 51SSH_ARGS = ["-o", "UserKnownHostsFile=/dev/null", 52 "-o", "StrictHostKeyChecking=no"] 53SSH_CMD = ["ssh"] + SSH_ARGS 54SCP_CMD = ["scp"] + SSH_ARGS 55GET_BUILD_VAR_CMD = ["build/soong/soong_ui.bash", "--dumpvar-mode"] 56DEFAULT_RETRY_BACKOFF_FACTOR = 1 57DEFAULT_SLEEP_MULTIPLIER = 0 58 59_SSH_TUNNEL_ARGS = ( 60 "-i %(rsa_key_file)s -o UserKnownHostsFile=/dev/null " 61 "-o StrictHostKeyChecking=no " 62 "%(port_mapping)s" 63 "-N -f -l %(ssh_user)s %(ip_addr)s") 64PORT_MAPPING = "-L %(local_port)d:127.0.0.1:%(target_port)d " 65_RELEASE_PORT_CMD = "kill $(lsof -t -i :%d)" 66_WEBRTC_TARGET_PORT = 8443 67WEBRTC_PORTS_MAPPING = [{"local": constants.WEBRTC_LOCAL_PORT, 68 "target": _WEBRTC_TARGET_PORT}, 69 {"local": 15550, "target": 15550}, 70 {"local": 15551, "target": 15551}] 71_ADB_CONNECT_ARGS = "connect 127.0.0.1:%(adb_port)d" 72# Store the ports that vnc/adb are forwarded to, both are integers. 73ForwardedPorts = collections.namedtuple("ForwardedPorts", [constants.VNC_PORT, 74 constants.ADB_PORT]) 75AVD_PORT_DICT = { 76 constants.TYPE_GCE: ForwardedPorts(constants.GCE_VNC_PORT, 77 constants.GCE_ADB_PORT), 78 constants.TYPE_CF: ForwardedPorts(constants.CF_VNC_PORT, 79 constants.CF_ADB_PORT), 80 constants.TYPE_GF: ForwardedPorts(constants.GF_VNC_PORT, 81 constants.GF_ADB_PORT), 82 constants.TYPE_CHEEPS: ForwardedPorts(constants.CHEEPS_VNC_PORT, 83 constants.CHEEPS_ADB_PORT), 84 constants.TYPE_FVP: ForwardedPorts(None, constants.FVP_ADB_PORT), 85} 86 87_VNC_BIN = "ssvnc" 88_CMD_KILL = ["pkill", "-9", "-f"] 89_CMD_SG = "sg " 90_CMD_START_VNC = "%(bin)s vnc://127.0.0.1:%(port)d" 91_CMD_INSTALL_SSVNC = "sudo apt-get --assume-yes install ssvnc" 92_ENV_DISPLAY = "DISPLAY" 93_SSVNC_ENV_VARS = {"SSVNC_NO_ENC_WARN": "1", "SSVNC_SCALE": "auto", "VNCVIEWER_X11CURSOR": "1"} 94_DEFAULT_DISPLAY_SCALE = 1.0 95_DIST_DIR = "DIST_DIR" 96 97# For webrtc 98_WEBRTC_URL = "https://%(webrtc_ip)s:%(webrtc_port)d/?use_tcp=true" 99 100_CONFIRM_CONTINUE = ("In order to display the screen to the AVD, we'll need to " 101 "install a vnc client (ssvnc). \nWould you like acloud to " 102 "install it for you? (%s) \nPress 'y' to continue or " 103 "anything else to abort it[y/N]: ") % _CMD_INSTALL_SSVNC 104_EvaluatedResult = collections.namedtuple("EvaluatedResult", 105 ["is_result_ok", "result_message"]) 106# dict of supported system and their distributions. 107_SUPPORTED_SYSTEMS_AND_DISTS = {"Linux": ["Ubuntu", "ubuntu", "Debian", "debian"]} 108_DEFAULT_TIMEOUT_ERR = "Function did not complete within %d secs." 109_SSVNC_VIEWER_PATTERN = "vnc://127.0.0.1:%(vnc_port)d" 110 111 112class TempDir(object): 113 """A context manager that ceates a temporary directory. 114 115 Attributes: 116 path: The path of the temporary directory. 117 """ 118 119 def __init__(self): 120 self.path = tempfile.mkdtemp() 121 os.chmod(self.path, 0o700) 122 logger.debug("Created temporary dir %s", self.path) 123 124 def __enter__(self): 125 """Enter.""" 126 return self.path 127 128 def __exit__(self, exc_type, exc_value, traceback): 129 """Exit. 130 131 Args: 132 exc_type: Exception type raised within the context manager. 133 None if no execption is raised. 134 exc_value: Exception instance raised within the context manager. 135 None if no execption is raised. 136 traceback: Traceback for exeception that is raised within 137 the context manager. 138 None if no execption is raised. 139 Raises: 140 EnvironmentError or OSError when failed to delete temp directory. 141 """ 142 try: 143 if self.path: 144 shutil.rmtree(self.path) 145 logger.debug("Deleted temporary dir %s", self.path) 146 except EnvironmentError as e: 147 # Ignore error if there is no exception raised 148 # within the with-clause and the EnvironementError is 149 # about problem that directory or file does not exist. 150 if not exc_type and e.errno != errno.ENOENT: 151 raise 152 except Exception as e: # pylint: disable=W0703 153 if exc_type: 154 logger.error( 155 "Encountered error while deleting %s: %s", 156 self.path, 157 str(e), 158 exc_info=True) 159 else: 160 raise 161 162 163def RetryOnException(retry_checker, 164 max_retries, 165 sleep_multiplier=0, 166 retry_backoff_factor=1): 167 """Decorater which retries the function call if |retry_checker| returns true. 168 169 Args: 170 retry_checker: A callback function which should take an exception instance 171 and return True if functor(*args, **kwargs) should be retried 172 when such exception is raised, and return False if it should 173 not be retried. 174 max_retries: Maximum number of retries allowed. 175 sleep_multiplier: Will sleep sleep_multiplier * attempt_count seconds if 176 retry_backoff_factor is 1. Will sleep 177 sleep_multiplier * ( 178 retry_backoff_factor ** (attempt_count - 1)) 179 if retry_backoff_factor != 1. 180 retry_backoff_factor: See explanation of sleep_multiplier. 181 182 Returns: 183 The function wrapper. 184 """ 185 186 def _Wrapper(func): 187 def _FunctionWrapper(*args, **kwargs): 188 return Retry(retry_checker, max_retries, func, sleep_multiplier, 189 retry_backoff_factor, *args, **kwargs) 190 191 return _FunctionWrapper 192 193 return _Wrapper 194 195 196def Retry(retry_checker, max_retries, functor, sleep_multiplier, 197 retry_backoff_factor, *args, **kwargs): 198 """Conditionally retry a function. 199 200 Args: 201 retry_checker: A callback function which should take an exception instance 202 and return True if functor(*args, **kwargs) should be retried 203 when such exception is raised, and return False if it should 204 not be retried. 205 max_retries: Maximum number of retries allowed. 206 functor: The function to call, will call functor(*args, **kwargs). 207 sleep_multiplier: Will sleep sleep_multiplier * attempt_count seconds if 208 retry_backoff_factor is 1. Will sleep 209 sleep_multiplier * ( 210 retry_backoff_factor ** (attempt_count - 1)) 211 if retry_backoff_factor != 1. 212 retry_backoff_factor: See explanation of sleep_multiplier. 213 *args: Arguments to pass to the functor. 214 **kwargs: Key-val based arguments to pass to the functor. 215 216 Returns: 217 The return value of the functor. 218 219 Raises: 220 Exception: The exception that functor(*args, **kwargs) throws. 221 """ 222 attempt_count = 0 223 while attempt_count <= max_retries: 224 try: 225 attempt_count += 1 226 return_value = functor(*args, **kwargs) 227 return return_value 228 except Exception as e: # pylint: disable=W0703 229 if retry_checker(e) and attempt_count <= max_retries: 230 if retry_backoff_factor != 1: 231 sleep = sleep_multiplier * (retry_backoff_factor** 232 (attempt_count - 1)) 233 else: 234 sleep = sleep_multiplier * attempt_count 235 time.sleep(sleep) 236 else: 237 raise 238 239 240def RetryExceptionType(exception_types, max_retries, functor, *args, **kwargs): 241 """Retry exception if it is one of the given types. 242 243 Args: 244 exception_types: A tuple of exception types, e.g. (ValueError, KeyError) 245 max_retries: Max number of retries allowed. 246 functor: The function to call. Will be retried if exception is raised and 247 the exception is one of the exception_types. 248 *args: Arguments to pass to Retry function. 249 **kwargs: Key-val based arguments to pass to Retry functions. 250 251 Returns: 252 The value returned by calling functor. 253 """ 254 return Retry(lambda e: isinstance(e, exception_types), max_retries, 255 functor, *args, **kwargs) 256 257 258def PollAndWait(func, expected_return, timeout_exception, timeout_secs, 259 sleep_interval_secs, *args, **kwargs): 260 """Call a function until the function returns expected value or times out. 261 262 Args: 263 func: Function to call. 264 expected_return: The expected return value. 265 timeout_exception: Exception to raise when it hits timeout. 266 timeout_secs: Timeout seconds. 267 If 0 or less than zero, the function will run once and 268 we will not wait on it. 269 sleep_interval_secs: Time to sleep between two attemps. 270 *args: list of args to pass to func. 271 **kwargs: dictionary of keyword based args to pass to func. 272 273 Raises: 274 timeout_exception: if the run of function times out. 275 """ 276 # TODO(fdeng): Currently this method does not kill 277 # |func|, if |func| takes longer than |timeout_secs|. 278 # We can use a more robust version from chromite. 279 start = time.time() 280 while True: 281 return_value = func(*args, **kwargs) 282 if return_value == expected_return: 283 return 284 elif time.time() - start > timeout_secs: 285 raise timeout_exception 286 else: 287 if sleep_interval_secs > 0: 288 time.sleep(sleep_interval_secs) 289 290 291def GenerateUniqueName(prefix=None, suffix=None): 292 """Generate a random unique name using uuid4. 293 294 Args: 295 prefix: String, desired prefix to prepend to the generated name. 296 suffix: String, desired suffix to append to the generated name. 297 298 Returns: 299 String, a random name. 300 """ 301 name = uuid.uuid4().hex 302 if prefix: 303 name = "-".join([prefix, name]) 304 if suffix: 305 name = "-".join([name, suffix]) 306 return name 307 308 309def MakeTarFile(src_dict, dest): 310 """Archive files in tar.gz format to a file named as |dest|. 311 312 Args: 313 src_dict: A dictionary that maps a path to be archived 314 to the corresponding name that appears in the archive. 315 dest: String, path to output file, e.g. /tmp/myfile.tar.gz 316 """ 317 logger.info("Compressing %s into %s.", src_dict.keys(), dest) 318 with tarfile.open(dest, "w:gz") as tar: 319 for src, arcname in six.iteritems(src_dict): 320 tar.add(src, arcname=arcname) 321 322def CreateSshKeyPairIfNotExist(private_key_path, public_key_path): 323 """Create the ssh key pair if they don't exist. 324 325 Case1. If the private key doesn't exist, we will create both the public key 326 and the private key. 327 Case2. If the private key exists but public key doesn't, we will create the 328 public key by using the private key. 329 Case3. If the public key exists but the private key doesn't, we will create 330 a new private key and overwrite the public key. 331 332 Args: 333 private_key_path: Path to the private key file. 334 e.g. ~/.ssh/acloud_rsa 335 public_key_path: Path to the public key file. 336 e.g. ~/.ssh/acloud_rsa.pub 337 338 Raises: 339 error.DriverError: If failed to create the key pair. 340 """ 341 public_key_path = os.path.expanduser(public_key_path) 342 private_key_path = os.path.expanduser(private_key_path) 343 public_key_exist = os.path.exists(public_key_path) 344 private_key_exist = os.path.exists(private_key_path) 345 if public_key_exist and private_key_exist: 346 logger.debug( 347 "The ssh private key (%s) and public key (%s) already exist," 348 "will not automatically create the key pairs.", private_key_path, 349 public_key_path) 350 return 351 key_folder = os.path.dirname(private_key_path) 352 if not os.path.exists(key_folder): 353 os.makedirs(key_folder) 354 try: 355 if private_key_exist: 356 cmd = SSH_KEYGEN_PUB_CMD + ["-f", private_key_path] 357 with open(public_key_path, 'w') as outfile: 358 stream_content = CheckOutput(cmd) 359 outfile.write( 360 stream_content.rstrip('\n') + " " + getpass.getuser()) 361 logger.info( 362 "The ssh public key (%s) do not exist, " 363 "automatically creating public key, calling: %s", 364 public_key_path, " ".join(cmd)) 365 else: 366 cmd = SSH_KEYGEN_CMD + [ 367 "-C", getpass.getuser(), "-f", private_key_path 368 ] 369 logger.info( 370 "Creating public key from private key (%s) via cmd: %s", 371 private_key_path, " ".join(cmd)) 372 subprocess.check_call(cmd, stdout=sys.stderr, stderr=sys.stdout) 373 except subprocess.CalledProcessError as e: 374 raise errors.DriverError("Failed to create ssh key pair: %s" % str(e)) 375 except OSError as e: 376 raise errors.DriverError( 377 "Failed to create ssh key pair, please make sure " 378 "'ssh-keygen' is installed: %s" % str(e)) 379 380 # By default ssh-keygen will create a public key file 381 # by append .pub to the private key file name. Rename it 382 # to what's requested by public_key_path. 383 default_pub_key_path = "%s.pub" % private_key_path 384 try: 385 if default_pub_key_path != public_key_path: 386 os.rename(default_pub_key_path, public_key_path) 387 except OSError as e: 388 raise errors.DriverError( 389 "Failed to rename %s to %s: %s" % (default_pub_key_path, 390 public_key_path, str(e))) 391 392 logger.info("Created ssh private key (%s) and public key (%s)", 393 private_key_path, public_key_path) 394 395 396def VerifyRsaPubKey(rsa): 397 """Verify the format of rsa public key. 398 399 Args: 400 rsa: content of rsa public key. It should follow the format of 401 ssh-rsa AAAAB3NzaC1yc2EA.... test@test.com 402 403 Raises: 404 DriverError if the format is not correct. 405 """ 406 if not rsa or not all(ord(c) < 128 for c in rsa): 407 raise errors.DriverError( 408 "rsa key is empty or contains non-ascii character: %s" % rsa) 409 410 elements = rsa.split() 411 if len(elements) != 3: 412 raise errors.DriverError("rsa key is invalid, wrong format: %s" % rsa) 413 414 key_type, data, _ = elements 415 try: 416 binary_data = base64.decodestring(six.b(data)) 417 # number of bytes of int type 418 int_length = 4 419 # binary_data is like "7ssh-key..." in a binary format. 420 # The first 4 bytes should represent 7, which should be 421 # the length of the following string "ssh-key". 422 # And the next 7 bytes should be string "ssh-key". 423 # We will verify that the rsa conforms to this format. 424 # ">I" in the following line means "big-endian unsigned integer". 425 type_length = struct.unpack(">I", binary_data[:int_length])[0] 426 if binary_data[int_length:int_length + type_length] != six.b(key_type): 427 raise errors.DriverError("rsa key is invalid: %s" % rsa) 428 except (struct.error, binascii.Error) as e: 429 raise errors.DriverError( 430 "rsa key is invalid: %s, error: %s" % (rsa, str(e))) 431 432 433def Decompress(sourcefile, dest=None): 434 """Decompress .zip or .tar.gz. 435 436 Args: 437 sourcefile: A string, a source file path to decompress. 438 dest: A string, a folder path as decompress destination. 439 440 Raises: 441 errors.UnsupportedCompressionFileType: Not supported extension. 442 """ 443 logger.info("Start to decompress %s!", sourcefile) 444 dest_path = dest if dest else "." 445 if sourcefile.endswith(".tar.gz"): 446 with tarfile.open(sourcefile, "r:gz") as compressor: 447 compressor.extractall(dest_path) 448 elif sourcefile.endswith(".zip"): 449 with zipfile.ZipFile(sourcefile, 'r') as compressor: 450 compressor.extractall(dest_path) 451 else: 452 raise errors.UnsupportedCompressionFileType( 453 "Sorry, we could only support compression file type " 454 "for zip or tar.gz.") 455 456 457# pylint: disable=old-style-class,no-init 458class TextColors: 459 """A class that defines common color ANSI code.""" 460 461 HEADER = "\033[95m" 462 OKBLUE = "\033[94m" 463 OKGREEN = "\033[92m" 464 WARNING = "\033[33m" 465 FAIL = "\033[91m" 466 ENDC = "\033[0m" 467 BOLD = "\033[1m" 468 UNDERLINE = "\033[4m" 469 470 471def PrintColorString(message, colors=TextColors.OKBLUE, **kwargs): 472 """A helper function to print out colored text. 473 474 Use print function "print(message, end="")" to show message in one line. 475 Example code: 476 DisplayMessages("Creating GCE instance...", end="") 477 # Job execute 20s 478 DisplayMessages("Done! (20s)") 479 Display: 480 Creating GCE instance... 481 # After job finished, messages update as following: 482 Creating GCE instance...Done! (20s) 483 484 Args: 485 message: String, the message text. 486 colors: String, color code. 487 **kwargs: dictionary of keyword based args to pass to func. 488 """ 489 print(colors + message + TextColors.ENDC, **kwargs) 490 sys.stdout.flush() 491 492 493def InteractWithQuestion(question, colors=TextColors.WARNING): 494 """A helper function to define the common way to run interactive cmd. 495 496 Args: 497 question: String, the question to ask user. 498 colors: String, color code. 499 500 Returns: 501 String, input from user. 502 """ 503 return str(six.moves.input(colors + question + TextColors.ENDC).strip()) 504 505 506def GetUserAnswerYes(question): 507 """Ask user about acloud setup question. 508 509 Args: 510 question: String of question for user. Enter is equivalent to pressing 511 n. We should hint user with upper case N surrounded in square 512 brackets. 513 Ex: "Are you sure to change bucket name[y/N]:" 514 515 Returns: 516 Boolean, True if answer is "Yes", False otherwise. 517 """ 518 answer = InteractWithQuestion(question) 519 return answer.lower() in constants.USER_ANSWER_YES 520 521 522class BatchHttpRequestExecutor(object): 523 """A helper class that executes requests in batch with retry. 524 525 This executor executes http requests in a batch and retry 526 those that have failed. It iteratively updates the dictionary 527 self._final_results with latest results, which can be retrieved 528 via GetResults. 529 """ 530 531 def __init__(self, 532 execute_once_functor, 533 requests, 534 retry_http_codes=None, 535 max_retry=None, 536 sleep=None, 537 backoff_factor=None, 538 other_retriable_errors=None): 539 """Initializes the executor. 540 541 Args: 542 execute_once_functor: A function that execute requests in batch once. 543 It should return a dictionary like 544 {request_id: (response, exception)} 545 requests: A dictionary where key is request id picked by caller, 546 and value is a apiclient.http.HttpRequest. 547 retry_http_codes: A list of http codes to retry. 548 max_retry: See utils.Retry. 549 sleep: See utils.Retry. 550 backoff_factor: See utils.Retry. 551 other_retriable_errors: A tuple of error types that should be retried 552 other than errors.HttpError. 553 """ 554 self._execute_once_functor = execute_once_functor 555 self._requests = requests 556 # A dictionary that maps request id to pending request. 557 self._pending_requests = {} 558 # A dictionary that maps request id to a tuple (response, exception). 559 self._final_results = {} 560 self._retry_http_codes = retry_http_codes 561 self._max_retry = max_retry 562 self._sleep = sleep 563 self._backoff_factor = backoff_factor 564 self._other_retriable_errors = other_retriable_errors 565 566 def _ShoudRetry(self, exception): 567 """Check if an exception is retriable. 568 569 Args: 570 exception: An exception instance. 571 """ 572 if isinstance(exception, self._other_retriable_errors): 573 return True 574 575 if (isinstance(exception, errors.HttpError) 576 and exception.code in self._retry_http_codes): 577 return True 578 return False 579 580 def _ExecuteOnce(self): 581 """Executes pending requests and update it with failed, retriable ones. 582 583 Raises: 584 HasRetriableRequestsError: if some requests fail and are retriable. 585 """ 586 results = self._execute_once_functor(self._pending_requests) 587 # Update final_results with latest results. 588 self._final_results.update(results) 589 # Clear pending_requests 590 self._pending_requests.clear() 591 for request_id, result in six.iteritems(results): 592 exception = result[1] 593 if exception is not None and self._ShoudRetry(exception): 594 # If this is a retriable exception, put it in pending_requests 595 self._pending_requests[request_id] = self._requests[request_id] 596 if self._pending_requests: 597 # If there is still retriable requests pending, raise an error 598 # so that Retry will retry this function with pending_requests. 599 raise errors.HasRetriableRequestsError( 600 "Retriable errors: %s" % 601 [str(results[rid][1]) for rid in self._pending_requests]) 602 603 def Execute(self): 604 """Executes the requests and retry if necessary. 605 606 Will populate self._final_results. 607 """ 608 609 def _ShouldRetryHandler(exc): 610 """Check if |exc| is a retriable exception. 611 612 Args: 613 exc: An exception. 614 615 Returns: 616 True if exception is of type HasRetriableRequestsError; False otherwise. 617 """ 618 should_retry = isinstance(exc, errors.HasRetriableRequestsError) 619 if should_retry: 620 logger.info("Will retry failed requests.", exc_info=True) 621 logger.info("%s", exc) 622 return should_retry 623 624 try: 625 self._pending_requests = self._requests.copy() 626 Retry( 627 _ShouldRetryHandler, 628 max_retries=self._max_retry, 629 functor=self._ExecuteOnce, 630 sleep_multiplier=self._sleep, 631 retry_backoff_factor=self._backoff_factor) 632 except errors.HasRetriableRequestsError: 633 logger.debug("Some requests did not succeed after retry.") 634 635 def GetResults(self): 636 """Returns final results. 637 638 Returns: 639 results, a dictionary in the following format 640 {request_id: (response, exception)} 641 request_ids are those from requests; response 642 is the http response for the request or None on error; 643 exception is an instance of DriverError or None if no error. 644 """ 645 return self._final_results 646 647 648def DefaultEvaluator(result): 649 """Default Evaluator always return result is ok. 650 651 Args: 652 result:the return value of the target function. 653 654 Returns: 655 _EvaluatedResults namedtuple. 656 """ 657 return _EvaluatedResult(is_result_ok=True, result_message=result) 658 659 660def ReportEvaluator(report): 661 """Evalute the acloud operation by the report. 662 663 Args: 664 report: acloud.public.report() object. 665 666 Returns: 667 _EvaluatedResults namedtuple. 668 """ 669 if report is None or report.errors: 670 return _EvaluatedResult(is_result_ok=False, 671 result_message=report.errors) 672 673 return _EvaluatedResult(is_result_ok=True, result_message=None) 674 675 676def BootEvaluator(boot_dict): 677 """Evaluate if the device booted successfully. 678 679 Args: 680 boot_dict: Dict of instance_name:boot error. 681 682 Returns: 683 _EvaluatedResults namedtuple. 684 """ 685 if boot_dict: 686 return _EvaluatedResult(is_result_ok=False, result_message=boot_dict) 687 return _EvaluatedResult(is_result_ok=True, result_message=None) 688 689 690class TimeExecute(object): 691 """Count the function execute time.""" 692 693 def __init__(self, function_description=None, print_before_call=True, 694 print_status=True, result_evaluator=DefaultEvaluator, 695 display_waiting_dots=True): 696 """Initializes the class. 697 698 Args: 699 function_description: String that describes function (e.g."Creating 700 Instance...") 701 print_before_call: Boolean, print the function description before 702 calling the function, default True. 703 print_status: Boolean, print the status of the function after the 704 function has completed, default True ("OK" or "Fail"). 705 result_evaluator: Func object. Pass func to evaluate result. 706 Default evaluator always report result is ok and 707 failed result will be identified only in exception 708 case. 709 display_waiting_dots: Boolean, if true print the function_description 710 followed by waiting dot. 711 """ 712 self._function_description = function_description 713 self._print_before_call = print_before_call 714 self._print_status = print_status 715 self._result_evaluator = result_evaluator 716 self._display_waiting_dots = display_waiting_dots 717 718 def __call__(self, func): 719 def DecoratorFunction(*args, **kargs): 720 """Decorator function. 721 722 Args: 723 *args: Arguments to pass to the functor. 724 **kwargs: Key-val based arguments to pass to the functor. 725 726 Raises: 727 Exception: The exception that functor(*args, **kwargs) throws. 728 """ 729 timestart = time.time() 730 if self._print_before_call: 731 waiting_dots = "..." if self._display_waiting_dots else "" 732 PrintColorString("%s %s"% (self._function_description, 733 waiting_dots), end="") 734 try: 735 result = func(*args, **kargs) 736 result_time = time.time() - timestart 737 if not self._print_before_call: 738 PrintColorString("%s (%ds)" % (self._function_description, 739 result_time), 740 TextColors.OKGREEN) 741 if self._print_status: 742 evaluated_result = self._result_evaluator(result) 743 if evaluated_result.is_result_ok: 744 PrintColorString("OK! (%ds)" % (result_time), 745 TextColors.OKGREEN) 746 else: 747 PrintColorString("Fail! (%ds)" % (result_time), 748 TextColors.FAIL) 749 PrintColorString("Error: %s" % 750 evaluated_result.result_message, 751 TextColors.FAIL) 752 return result 753 except: 754 if self._print_status: 755 PrintColorString("Fail! (%ds)" % (time.time() - timestart), 756 TextColors.FAIL) 757 raise 758 return DecoratorFunction 759 760 761def PickFreePort(): 762 """Helper to pick a free port. 763 764 Returns: 765 Integer, a free port number. 766 """ 767 tcp_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 768 tcp_socket.bind(("", 0)) 769 port = tcp_socket.getsockname()[1] 770 tcp_socket.close() 771 return port 772 773 774def CheckPortFree(port): 775 """Check the availablity of the tcp port. 776 777 Args: 778 Integer, a port number. 779 780 Raises: 781 PortOccupied: This port is not available. 782 """ 783 tcp_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 784 try: 785 tcp_socket.bind(("", port)) 786 except socket.error: 787 raise errors.PortOccupied("Port (%d) is taken, please choose another " 788 "port." % port) 789 tcp_socket.close() 790 791 792def _ExecuteCommand(cmd, args): 793 """Execute command. 794 795 Args: 796 cmd: Strings of execute binary name. 797 args: List of args to pass in with cmd. 798 799 Raises: 800 errors.NoExecuteBin: Can't find the execute bin file. 801 """ 802 bin_path = FindExecutable(cmd) 803 if not bin_path: 804 raise errors.NoExecuteCmd("unable to locate %s" % cmd) 805 command = [bin_path] + args 806 logger.debug("Running '%s'", ' '.join(command)) 807 with open(os.devnull, "w") as dev_null: 808 subprocess.check_call(command, stderr=dev_null, stdout=dev_null) 809 810 811def ReleasePort(port): 812 """Release local port. 813 814 Args: 815 port: Integer of local port number. 816 """ 817 try: 818 with open(os.devnull, "w") as dev_null: 819 subprocess.check_call(_RELEASE_PORT_CMD % port, 820 stderr=dev_null, stdout=dev_null, shell=True) 821 except subprocess.CalledProcessError: 822 logger.debug("The port %d is available.", constants.WEBRTC_LOCAL_PORT) 823 824 825def EstablishWebRTCSshTunnel(ip_addr, rsa_key_file, ssh_user, 826 extra_args_ssh_tunnel=None): 827 """Create ssh tunnels for webrtc. 828 829 # TODO(151418177): Before fully supporting webrtc feature, we establish one 830 # WebRTC tunnel at a time. so always delete the previous connection before 831 # establishing new one. 832 833 Args: 834 ip_addr: String, use to build the adb & vnc tunnel between local 835 and remote instance. 836 rsa_key_file: String, Private key file path to use when creating 837 the ssh tunnels. 838 ssh_user: String of user login into the instance. 839 extra_args_ssh_tunnel: String, extra args for ssh tunnel connection. 840 """ 841 ReleasePort(constants.WEBRTC_LOCAL_PORT) 842 try: 843 port_mapping = [PORT_MAPPING % { 844 "local_port":port["local"], 845 "target_port":port["target"]} for port in WEBRTC_PORTS_MAPPING] 846 ssh_tunnel_args = _SSH_TUNNEL_ARGS % { 847 "rsa_key_file": rsa_key_file, 848 "ssh_user": ssh_user, 849 "ip_addr": ip_addr, 850 "port_mapping":" ".join(port_mapping)} 851 ssh_tunnel_args_list = shlex.split(ssh_tunnel_args) 852 if extra_args_ssh_tunnel != None: 853 ssh_tunnel_args_list.extend(shlex.split(extra_args_ssh_tunnel)) 854 _ExecuteCommand(constants.SSH_BIN, ssh_tunnel_args_list) 855 except subprocess.CalledProcessError as e: 856 PrintColorString("\n%s\nFailed to create ssh tunnels, retry with '#acloud " 857 "reconnect'." % e, TextColors.FAIL) 858 859 860# TODO(147337696): create ssh tunnels tear down as adb and vnc. 861# pylint: disable=too-many-locals 862def AutoConnect(ip_addr, rsa_key_file, target_vnc_port, target_adb_port, 863 ssh_user, client_adb_port=None, extra_args_ssh_tunnel=None): 864 """Autoconnect to an AVD instance. 865 866 Args: 867 ip_addr: String, use to build the adb & vnc tunnel between local 868 and remote instance. 869 rsa_key_file: String, Private key file path to use when creating 870 the ssh tunnels. 871 target_vnc_port: Integer of target vnc port number. 872 target_adb_port: Integer of target adb port number. 873 ssh_user: String of user login into the instance. 874 client_adb_port: Integer, Specified adb port to establish connection. 875 extra_args_ssh_tunnel: String, extra args for ssh tunnel connection. 876 877 Returns: 878 NamedTuple of (vnc_port, adb_port) SSHTUNNEL of the connect, both are 879 integers. 880 """ 881 local_adb_port = client_adb_port or PickFreePort() 882 port_mapping = [PORT_MAPPING % { 883 "local_port":local_adb_port, 884 "target_port":target_adb_port}] 885 local_free_vnc_port = None 886 if target_vnc_port: 887 local_free_vnc_port = PickFreePort() 888 port_mapping += [PORT_MAPPING % { 889 "local_port":local_free_vnc_port, 890 "target_port":target_vnc_port}] 891 try: 892 ssh_tunnel_args = _SSH_TUNNEL_ARGS % { 893 "rsa_key_file": rsa_key_file, 894 "port_mapping": " ".join(port_mapping), 895 "ssh_user": ssh_user, 896 "ip_addr": ip_addr} 897 ssh_tunnel_args_list = shlex.split(ssh_tunnel_args) 898 if extra_args_ssh_tunnel != None: 899 ssh_tunnel_args_list.extend(shlex.split(extra_args_ssh_tunnel)) 900 _ExecuteCommand(constants.SSH_BIN, ssh_tunnel_args_list) 901 except subprocess.CalledProcessError as e: 902 PrintColorString("\n%s\nFailed to create ssh tunnels, retry with '#acloud " 903 "reconnect'." % e, TextColors.FAIL) 904 return ForwardedPorts(vnc_port=None, adb_port=None) 905 906 try: 907 adb_connect_args = _ADB_CONNECT_ARGS % {"adb_port": local_adb_port} 908 _ExecuteCommand(constants.ADB_BIN, adb_connect_args.split()) 909 except subprocess.CalledProcessError: 910 PrintColorString("Failed to adb connect, retry with " 911 "'#acloud reconnect'", TextColors.FAIL) 912 913 return ForwardedPorts(vnc_port=local_free_vnc_port, 914 adb_port=local_adb_port) 915 916 917def GetAnswerFromList(answer_list, enable_choose_all=False): 918 """Get answer from a list. 919 920 Args: 921 answer_list: list of the answers to choose from. 922 enable_choose_all: True to choose all items from answer list. 923 924 Return: 925 List holding the answer(s). 926 """ 927 print("[0] to exit.") 928 start_index = 1 929 max_choice = len(answer_list) 930 931 for num, item in enumerate(answer_list, start_index): 932 print("[%d] %s" % (num, item)) 933 if enable_choose_all: 934 max_choice += 1 935 print("[%d] for all." % max_choice) 936 937 choice = -1 938 939 while True: 940 try: 941 choice = six.moves.input("Enter your choice[0-%d]: " % max_choice) 942 choice = int(choice) 943 except ValueError: 944 print("'%s' is not a valid integer.", choice) 945 continue 946 # Filter out choices 947 if choice == 0: 948 sys.exit(constants.EXIT_BY_USER) 949 if enable_choose_all and choice == max_choice: 950 return answer_list 951 if choice < 0 or choice > max_choice: 952 print("please choose between 0 and %d" % max_choice) 953 else: 954 return [answer_list[choice-start_index]] 955 956 957def LaunchVNCFromReport(report, avd_spec, no_prompts=False): 958 """Launch vnc client according to the instances report. 959 960 Args: 961 report: Report object, that stores and generates report. 962 avd_spec: AVDSpec object that tells us what we're going to create. 963 no_prompts: Boolean, True to skip all prompts. 964 """ 965 for device in report.data.get("devices", []): 966 if device.get(constants.VNC_PORT): 967 LaunchVncClient(device.get(constants.VNC_PORT), 968 avd_width=avd_spec.hw_property["x_res"], 969 avd_height=avd_spec.hw_property["y_res"], 970 no_prompts=no_prompts) 971 else: 972 PrintColorString("No VNC port specified, skipping VNC startup.", 973 TextColors.FAIL) 974 975 976def LaunchBrowserFromReport(report): 977 """Open browser when autoconnect to webrtc according to the instances report. 978 979 Args: 980 report: Report object, that stores and generates report. 981 """ 982 PrintColorString("(This is an experimental project for webrtc, and since " 983 "the certificate is self-signed, Chrome will mark it as " 984 "an insecure website. keep going.)", 985 TextColors.WARNING) 986 987 for device in report.data.get("devices", []): 988 if device.get("ip"): 989 LaunchBrowser(constants.WEBRTC_LOCAL_HOST, 990 constants.WEBRTC_LOCAL_PORT) 991 992 993def LaunchBrowser(ip_addr, port): 994 """Launch browser to connect the webrtc AVD. 995 996 Args: 997 ip_addr: String, use to connect to webrtc AVD on the instance. 998 port: Integer, port number. 999 """ 1000 webrtc_link = _WEBRTC_URL % { 1001 "webrtc_ip": ip_addr, 1002 "webrtc_port": port} 1003 if os.environ.get(_ENV_DISPLAY, None): 1004 webbrowser.open_new_tab(webrtc_link) 1005 else: 1006 PrintColorString("Remote terminal can't support launch webbrowser.", 1007 TextColors.FAIL) 1008 PrintColorString("WebRTC AVD URL: %s "% webrtc_link) 1009 1010 1011def LaunchVncClient(port, avd_width=None, avd_height=None, no_prompts=False): 1012 """Launch ssvnc. 1013 1014 Args: 1015 port: Integer, port number. 1016 avd_width: String, the width of avd. 1017 avd_height: String, the height of avd. 1018 no_prompts: Boolean, True to skip all prompts. 1019 """ 1020 try: 1021 os.environ[_ENV_DISPLAY] 1022 except KeyError: 1023 PrintColorString("Remote terminal can't support VNC. " 1024 "Skipping VNC startup. " 1025 "VNC server is listening at 127.0.0.1:{}.".format(port), 1026 TextColors.FAIL) 1027 return 1028 1029 if IsSupportedPlatform() and not FindExecutable(_VNC_BIN): 1030 if no_prompts or GetUserAnswerYes(_CONFIRM_CONTINUE): 1031 try: 1032 PrintColorString("Installing ssvnc vnc client... ", end="") 1033 sys.stdout.flush() 1034 CheckOutput(_CMD_INSTALL_SSVNC, shell=True) 1035 PrintColorString("Done", TextColors.OKGREEN) 1036 except subprocess.CalledProcessError as cpe: 1037 PrintColorString("Failed to install ssvnc: %s" % 1038 cpe.output, TextColors.FAIL) 1039 return 1040 else: 1041 return 1042 ssvnc_env = os.environ.copy() 1043 ssvnc_env.update(_SSVNC_ENV_VARS) 1044 # Override SSVNC_SCALE 1045 if avd_width or avd_height: 1046 scale_ratio = CalculateVNCScreenRatio(avd_width, avd_height) 1047 ssvnc_env["SSVNC_SCALE"] = str(scale_ratio) 1048 logger.debug("SSVNC_SCALE:%s", scale_ratio) 1049 1050 ssvnc_args = _CMD_START_VNC % {"bin": FindExecutable(_VNC_BIN), 1051 "port": port} 1052 subprocess.Popen(ssvnc_args.split(), env=ssvnc_env) 1053 1054 1055def PrintDeviceSummary(report): 1056 """Display summary of devices. 1057 1058 -Display device details from the report instance. 1059 report example: 1060 'data': [{'devices':[{'instance_name': 'ins-f6a397-none-53363', 1061 'ip': u'35.234.10.162'}]}] 1062 -Display error message from report.error. 1063 1064 Args: 1065 report: A Report instance. 1066 """ 1067 PrintColorString("\n") 1068 PrintColorString("Device summary:") 1069 for device in report.data.get("devices", []): 1070 adb_serial = "(None)" 1071 adb_port = device.get("adb_port") 1072 if adb_port: 1073 adb_serial = constants.LOCALHOST_ADB_SERIAL % adb_port 1074 instance_name = device.get("instance_name") 1075 instance_ip = device.get("ip") 1076 instance_details = "" if not instance_name else "(%s[%s])" % ( 1077 instance_name, instance_ip) 1078 PrintColorString(" - device serial: %s %s" % (adb_serial, 1079 instance_details)) 1080 PrintColorString(" export ANDROID_SERIAL=%s" % adb_serial) 1081 1082 # TODO(b/117245508): Help user to delete instance if it got created. 1083 if report.errors: 1084 error_msg = "\n".join(report.errors) 1085 PrintColorString("Fail in:\n%s\n" % error_msg, TextColors.FAIL) 1086 1087 1088def CalculateVNCScreenRatio(avd_width, avd_height): 1089 """calculate the vnc screen scale ratio to fit into user's monitor. 1090 1091 Args: 1092 avd_width: String, the width of avd. 1093 avd_height: String, the height of avd. 1094 Return: 1095 Float, scale ratio for vnc client. 1096 """ 1097 try: 1098 import Tkinter 1099 # Some python interpreters may not be configured for Tk, just return default scale ratio. 1100 except ImportError: 1101 try: 1102 import tkinter as Tkinter 1103 except ImportError: 1104 PrintColorString( 1105 "no module named tkinter, vnc display scale were not be fit." 1106 "please run 'sudo apt-get install python3-tk' to install it.") 1107 return _DEFAULT_DISPLAY_SCALE 1108 root = Tkinter.Tk() 1109 margin = 100 # leave some space on user's monitor. 1110 screen_height = root.winfo_screenheight() - margin 1111 screen_width = root.winfo_screenwidth() - margin 1112 1113 scale_h = _DEFAULT_DISPLAY_SCALE 1114 scale_w = _DEFAULT_DISPLAY_SCALE 1115 if float(screen_height) < float(avd_height): 1116 scale_h = round(float(screen_height) / float(avd_height), 1) 1117 1118 if float(screen_width) < float(avd_width): 1119 scale_w = round(float(screen_width) / float(avd_width), 1) 1120 1121 logger.debug("scale_h: %s (screen_h: %s/avd_h: %s)," 1122 " scale_w: %s (screen_w: %s/avd_w: %s)", 1123 scale_h, screen_height, avd_height, 1124 scale_w, screen_width, avd_width) 1125 1126 # Return the larger scale-down ratio. 1127 return scale_h if scale_h < scale_w else scale_w 1128 1129 1130def IsCommandRunning(command): 1131 """Check if command is running. 1132 1133 Args: 1134 command: String of command name. 1135 1136 Returns: 1137 Boolean, True if command is running. False otherwise. 1138 """ 1139 try: 1140 with open(os.devnull, "w") as dev_null: 1141 subprocess.check_call([constants.CMD_PGREP, "-af", command], 1142 stderr=dev_null, stdout=dev_null) 1143 return True 1144 except subprocess.CalledProcessError: 1145 return False 1146 1147 1148def AddUserGroupsToCmd(cmd, user_groups): 1149 """Add the user groups to the command if necessary. 1150 1151 As part of local host setup to enable local instance support, the user is 1152 added to certain groups. For those settings to take effect systemwide 1153 requires the user to log out and log back in. In the scenario where the 1154 user has run setup and hasn't logged out, we still want them to be able to 1155 launch a local instance so add the user to the groups as part of the 1156 command to ensure success. 1157 1158 The reason using here-doc instead of '&' is all operations need to be ran in 1159 ths same pid. Here's an example cmd: 1160 $ sg kvm << EOF 1161 sg libvirt 1162 sg cvdnetwork 1163 launch_cvd --cpus 2 --x_res 1280 --y_res 720 --dpi 160 --memory_mb 4096 1164 EOF 1165 1166 Args: 1167 cmd: String of the command to prepend the user groups to. 1168 user_groups: List of user groups name.(String) 1169 1170 Returns: 1171 String of the command with the user groups prepended to it if necessary, 1172 otherwise the same existing command. 1173 """ 1174 user_group_cmd = "" 1175 if not CheckUserInGroups(user_groups): 1176 logger.debug("Need to add user groups to the command") 1177 for idx, group in enumerate(user_groups): 1178 user_group_cmd += _CMD_SG + group 1179 if idx == 0: 1180 user_group_cmd += " <<EOF\n" 1181 else: 1182 user_group_cmd += "\n" 1183 cmd += "\nEOF" 1184 user_group_cmd += cmd 1185 logger.debug("user group cmd: %s", user_group_cmd) 1186 return user_group_cmd 1187 1188 1189def CheckUserInGroups(group_name_list): 1190 """Check if the current user is in the group. 1191 1192 Args: 1193 group_name_list: The list of group name. 1194 Returns: 1195 True if current user is in all the groups. 1196 """ 1197 logger.info("Checking if user is in following groups: %s", group_name_list) 1198 current_groups = [grp.getgrgid(g).gr_name for g in os.getgroups()] 1199 all_groups_present = True 1200 for group in group_name_list: 1201 if group not in current_groups: 1202 all_groups_present = False 1203 logger.info("missing group: %s", group) 1204 return all_groups_present 1205 1206 1207def IsSupportedPlatform(print_warning=False): 1208 """Check if user's os is the supported platform. 1209 1210 platform.version() return such as '#1 SMP Debian 5.6.14-1rodete2...' 1211 and use to judge supported or not. 1212 1213 Args: 1214 print_warning: Boolean, print the unsupported warning 1215 if True. 1216 Returns: 1217 Boolean, True if user is using supported platform. 1218 """ 1219 system = platform.system() 1220 # TODO(b/161085678): After python3 fully migrated, then use distro to fix. 1221 platform_supported = False 1222 if system in _SUPPORTED_SYSTEMS_AND_DISTS: 1223 for dist in _SUPPORTED_SYSTEMS_AND_DISTS[system]: 1224 if dist in platform.version(): 1225 platform_supported = True 1226 break 1227 1228 logger.info("Updated supported system and dists: %s", 1229 _SUPPORTED_SYSTEMS_AND_DISTS) 1230 platform_supported_msg = ("%s[%s] %s supported platform" % 1231 (system, 1232 platform.version(), 1233 "is a" if platform_supported else "is not a")) 1234 if print_warning and not platform_supported: 1235 PrintColorString(platform_supported_msg, TextColors.WARNING) 1236 else: 1237 logger.info(platform_supported_msg) 1238 1239 return platform_supported 1240 1241 1242def GetDistDir(): 1243 """Return the absolute path to the dist dir.""" 1244 android_build_top = os.environ.get(constants.ENV_ANDROID_BUILD_TOP) 1245 if not android_build_top: 1246 return None 1247 dist_cmd = GET_BUILD_VAR_CMD[:] 1248 dist_cmd.append(_DIST_DIR) 1249 try: 1250 dist_dir = CheckOutput(dist_cmd, cwd=android_build_top) 1251 except subprocess.CalledProcessError: 1252 return None 1253 return os.path.join(android_build_top, dist_dir.strip()) 1254 1255 1256def CleanupProcess(pattern): 1257 """Cleanup process with pattern. 1258 1259 Args: 1260 pattern: String, string of process pattern. 1261 """ 1262 if IsCommandRunning(pattern): 1263 command_kill = _CMD_KILL + [pattern] 1264 subprocess.check_call(command_kill) 1265 1266 1267def TimeoutException(timeout_secs, timeout_error=_DEFAULT_TIMEOUT_ERR): 1268 """Decorater which function timeout setup and raise custom exception. 1269 1270 Args: 1271 timeout_secs: Number of maximum seconds of waiting time. 1272 timeout_error: String to describe timeout exception. 1273 1274 Returns: 1275 The function wrapper. 1276 """ 1277 if timeout_error == _DEFAULT_TIMEOUT_ERR: 1278 timeout_error = timeout_error % timeout_secs 1279 1280 def _Wrapper(func): 1281 # pylint: disable=unused-argument 1282 def _HandleTimeout(signum, frame): 1283 raise errors.FunctionTimeoutError(timeout_error) 1284 1285 def _FunctionWrapper(*args, **kwargs): 1286 signal.signal(signal.SIGALRM, _HandleTimeout) 1287 signal.alarm(timeout_secs) 1288 try: 1289 result = func(*args, **kwargs) 1290 finally: 1291 signal.alarm(0) 1292 return result 1293 1294 return _FunctionWrapper 1295 1296 return _Wrapper 1297 1298 1299def GetBuildEnvironmentVariable(variable_name): 1300 """Get build environment variable. 1301 1302 Args: 1303 variable_name: String of variable name. 1304 1305 Returns: 1306 String, the value of the variable. 1307 1308 Raises: 1309 errors.GetAndroidBuildEnvVarError: No environment variable found. 1310 """ 1311 try: 1312 return os.environ[variable_name] 1313 except KeyError: 1314 raise errors.GetAndroidBuildEnvVarError( 1315 "Could not get environment var: %s\n" 1316 "Try to run 'source build/envsetup.sh && lunch <target>'" 1317 % variable_name 1318 ) 1319 1320 1321# pylint: disable=no-member 1322def FindExecutable(filename): 1323 """A compatibility function to find execution file path. 1324 1325 Args: 1326 filename: String of execution filename. 1327 1328 Returns: 1329 String: execution file path. 1330 """ 1331 try: 1332 from distutils.spawn import find_executable 1333 return find_executable(filename) 1334 except ImportError: 1335 return shutil.which(filename) 1336 1337 1338def GetDictItems(namedtuple_object): 1339 """A compatibility function to access the OrdereDict object from the given namedtuple object. 1340 1341 Args: 1342 namedtuple_object: namedtuple object. 1343 1344 Returns: 1345 collections.namedtuple.__dict__.items() when using python2. 1346 collections.namedtuple._asdict().items() when using python3. 1347 """ 1348 return (namedtuple_object.__dict__.items() if six.PY2 1349 else namedtuple_object._asdict().items()) 1350 1351 1352def CleanupSSVncviewer(vnc_port): 1353 """Cleanup the old disconnected ssvnc viewer. 1354 1355 Args: 1356 vnc_port: Integer, port number of vnc. 1357 """ 1358 ssvnc_viewer_pattern = _SSVNC_VIEWER_PATTERN % {"vnc_port":vnc_port} 1359 CleanupProcess(ssvnc_viewer_pattern) 1360 1361 1362def CheckOutput(cmd, **kwargs): 1363 """Call subprocess.check_output to get output. 1364 1365 The subprocess.check_output return type is "bytes" in python 3, we have 1366 to convert bytes as string with .decode() in advance. 1367 1368 Args: 1369 cmd: String of command. 1370 **kwargs: dictionary of keyword based args to pass to func. 1371 1372 Return: 1373 String to command output. 1374 """ 1375 return subprocess.check_output(cmd, **kwargs).decode() 1376