#!/usr/bin/env python # # Copyright 2018 - The Android Open Source Project # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Gcloud setup runner.""" from __future__ import print_function import logging import os import re import subprocess import six from acloud import errors from acloud.internal.lib import utils from acloud.public import config from acloud.setup import base_task_runner from acloud.setup import google_sdk logger = logging.getLogger(__name__) # APIs that need to be enabled for GCP project. _ANDROID_BUILD_SERVICE = "androidbuildinternal.googleapis.com" _ANDROID_BUILD_MSG = ( "This service (%s) help to download images from Android Build. If it isn't " "enabled, acloud only supports local images to create AVD." % _ANDROID_BUILD_SERVICE) _COMPUTE_ENGINE_SERVICE = "compute.googleapis.com" _COMPUTE_ENGINE_MSG = ( "This service (%s) help to create instance in google cloud platform. If it " "isn't enabled, acloud can't work anymore." % _COMPUTE_ENGINE_SERVICE) _OPEN_SERVICE_FAILED_MSG = ( "\n[Open Service Failed]\n" "Service name: %(service_name)s\n" "%(service_msg)s\n") _BUILD_SERVICE_ACCOUNT = "android-build-prod@system.gserviceaccount.com" _BILLING_ENABLE_MSG = "billingEnabled: true" _DEFAULT_SSH_FOLDER = os.path.expanduser("~/.ssh") _DEFAULT_SSH_KEY = "acloud_rsa" _DEFAULT_SSH_PRIVATE_KEY = os.path.join(_DEFAULT_SSH_FOLDER, _DEFAULT_SSH_KEY) _DEFAULT_SSH_PUBLIC_KEY = os.path.join(_DEFAULT_SSH_FOLDER, _DEFAULT_SSH_KEY + ".pub") _ENV_CLOUDSDK_PYTHON = "CLOUDSDK_PYTHON" _GCLOUD_COMPONENT_ALPHA = "alpha" # Regular expression to get project/zone information. _PROJECT_RE = re.compile(r"^project = (?P.+)") _ZONE_RE = re.compile(r"^zone = (?P.+)") def UpdateConfigFile(config_path, item, value): """Update config data. Case A: config file contain this item. In config, "project = A_project". New value is B_project Set config "project = B_project". Case B: config file didn't contain this item. New value is B_project. Setup config as "project = B_project". Args: config_path: String, acloud config path. item: String, item name in config file. EX: project, zone value: String, value of item in config file. TODO(111574698): Refactor this to minimize writes to the config file. TODO(111574698): Use proto method to update config. """ write_lines = [] find_item = False write_line = item + ": \"" + value + "\"\n" if os.path.isfile(config_path): with open(config_path, "r") as cfg_file: for read_line in cfg_file.readlines(): if read_line.startswith(item + ":"): find_item = True write_lines.append(write_line) else: write_lines.append(read_line) if not find_item: write_lines.append(write_line) with open(config_path, "w") as cfg_file: cfg_file.writelines(write_lines) def SetupSSHKeys(config_path, private_key_path, public_key_path): """Setup the pair of the ssh key for acloud.config. User can use the default path: "~/.ssh/acloud_rsa". Args: config_path: String, acloud config path. private_key_path: Path to the private key file. e.g. ~/.ssh/acloud_rsa public_key_path: Path to the public key file. e.g. ~/.ssh/acloud_rsa.pub """ private_key_path = os.path.expanduser(private_key_path) if (private_key_path == "" or public_key_path == "" or private_key_path == _DEFAULT_SSH_PRIVATE_KEY): utils.CreateSshKeyPairIfNotExist(_DEFAULT_SSH_PRIVATE_KEY, _DEFAULT_SSH_PUBLIC_KEY) UpdateConfigFile(config_path, "ssh_private_key_path", _DEFAULT_SSH_PRIVATE_KEY) UpdateConfigFile(config_path, "ssh_public_key_path", _DEFAULT_SSH_PUBLIC_KEY) def _InputIsEmpty(input_string): """Check input string is empty. Tool requests user to input client ID & client secret. This basic check can detect user input is empty. Args: input_string: String, user input string. Returns: Boolean: True if input is empty, False otherwise. """ if input_string is None: return True if input_string == "": print("Please enter a non-empty value.") return True return False class GoogleSDKBins(object): """Class to run tools in the Google SDK.""" def __init__(self, google_sdk_folder): """GoogleSDKBins initialize. Args: google_sdk_folder: String, google sdk path. """ self.gcloud_command_path = os.path.join(google_sdk_folder, "gcloud") self.gsutil_command_path = os.path.join(google_sdk_folder, "gsutil") # TODO(137195528): Remove python2 environment after acloud support python3. self._env = os.environ.copy() self._env[_ENV_CLOUDSDK_PYTHON] = "python2" def RunGcloud(self, cmd, **kwargs): """Run gcloud command. Args: cmd: String list, command strings. Ex: [config], then this function call "gcloud config". **kwargs: dictionary of keyword based args to pass to func. Returns: String, return message after execute gcloud command. """ return utils.CheckOutput([self.gcloud_command_path] + cmd, env=self._env, **kwargs) def RunGsutil(self, cmd, **kwargs): """Run gsutil command. Args: cmd : String list, command strings. Ex: [list], then this function call "gsutil list". **kwargs: dictionary of keyword based args to pass to func. Returns: String, return message after execute gsutil command. """ return utils.CheckOutput([self.gsutil_command_path] + cmd, env=self._env, **kwargs) class GoogleAPIService(object): """Class to enable api service in the gcp project.""" def __init__(self, service_name, error_msg, required=False): """GoogleAPIService initialize. Args: service_name: String, name of api service. error_msg: String, show messages if api service enable failed. required: Boolean, True for service must be enabled for acloud. """ self._name = service_name self._error_msg = error_msg self._required = required def EnableService(self, gcloud_runner): """Enable api service. Args: gcloud_runner: A GcloudRunner class to run "gcloud" command. """ try: gcloud_runner.RunGcloud(["services", "enable", self._name], stderr=subprocess.STDOUT) except subprocess.CalledProcessError as error: self.ShowFailMessages(error.output) def ShowFailMessages(self, error): """Show fail messages. Show the fail messages to hint users the impact if the api service isn't enabled. Args: error: String of error message when opening api service failed. """ msg_color = (utils.TextColors.FAIL if self._required else utils.TextColors.WARNING) utils.PrintColorString( error + _OPEN_SERVICE_FAILED_MSG % { "service_name": self._name, "service_msg": self._error_msg} , msg_color) @property def name(self): """Return name.""" return self._name class GcpTaskRunner(base_task_runner.BaseTaskRunner): """Runner to setup google cloud user information.""" WELCOME_MESSAGE_TITLE = "Setup google cloud user information" WELCOME_MESSAGE = ( "This step will walk you through gcloud SDK installation." "Then configure gcloud user information." "Finally enable some gcloud API services.") def __init__(self, config_path): """Initialize parameters. Load config file to get current values. Args: config_path: String, acloud config path. """ # pylint: disable=invalid-name config_mgr = config.AcloudConfigManager(config_path) cfg = config_mgr.Load() self.config_path = config_mgr.user_config_path self.project = cfg.project self.zone = cfg.zone self.ssh_private_key_path = cfg.ssh_private_key_path self.ssh_public_key_path = cfg.ssh_public_key_path self.stable_host_image_name = cfg.stable_host_image_name self.client_id = cfg.client_id self.client_secret = cfg.client_secret self.service_account_name = cfg.service_account_name self.service_account_private_key_path = cfg.service_account_private_key_path self.service_account_json_private_key_path = cfg.service_account_json_private_key_path def ShouldRun(self): """Check if we actually need to run GCP setup. We'll only do the gcp setup if certain fields in the cfg are empty. Returns: True if reqired config fields are empty, False otherwise. """ # We need to ensure the config has the proper auth-related fields set, # so config requires just 1 of the following: # 1. client id/secret # 2. service account name/private key path # 3. service account json private key path if ((not self.client_id or not self.client_secret) and (not self.service_account_name or not self.service_account_private_key_path) and not self.service_account_json_private_key_path): return True # If a project isn't set, then we need to run setup. return not self.project def _Run(self): """Run GCP setup task.""" self._SetupGcloudInfo() SetupSSHKeys(self.config_path, self.ssh_private_key_path, self.ssh_public_key_path) def _SetupGcloudInfo(self): """Setup Gcloud user information. 1. Setup Gcloud SDK tools. 2. Setup Gcloud project. a. Setup Gcloud project and zone. b. Setup Client ID and Client secret. c. Setup Google Cloud Storage bucket. 3. Enable Gcloud API services. """ google_sdk_init = google_sdk.GoogleSDK() try: google_sdk_runner = GoogleSDKBins(google_sdk_init.GetSDKBinPath()) google_sdk_init.InstallGcloudComponent(google_sdk_runner, _GCLOUD_COMPONENT_ALPHA) self._SetupProject(google_sdk_runner) self._EnableGcloudServices(google_sdk_runner) self._CreateStableHostImage() finally: google_sdk_init.CleanUp() def _CreateStableHostImage(self): """Create the stable host image.""" # Write default stable_host_image_name with unused value. # TODO(113091773): An additional step to create the host image. if not self.stable_host_image_name: UpdateConfigFile(self.config_path, "stable_host_image_name", "") def _NeedProjectSetup(self): """Confirm project setup should run or not. If the project settings (project name and zone) are blank (either one), we'll run the project setup flow. If they are set, we'll check with the user if they want to update them. Returns: Boolean: True if we need to setup the project, False otherwise. """ user_question = ( "Your default Project/Zone settings are:\n" "project:[%s]\n" "zone:[%s]\n" "Would you like to update them?[y/N]: \n") % (self.project, self.zone) if not self.project or not self.zone: logger.info("Project or zone is empty. Start to run setup process.") return True return utils.GetUserAnswerYes(user_question) def _NeedClientIDSetup(self, project_changed): """Confirm client setup should run or not. If project changed, client ID must also have to change. So tool will force to run setup function. If client ID or client secret is empty, tool force to run setup function. If project didn't change and config hold user client ID/secret, tool would skip client ID setup. Args: project_changed: Boolean, True for project changed. Returns: Boolean: True for run setup function. """ if project_changed: logger.info("Your project changed. Start to run setup process.") return True elif not self.client_id or not self.client_secret: logger.info("Client ID or client secret is empty. Start to run setup process.") return True logger.info("Project was unchanged and client ID didn't need to changed.") return False def _SetupProject(self, gcloud_runner): """Setup gcloud project information. Setup project and zone. Setup client ID and client secret. Make sure billing account enabled in project. Args: gcloud_runner: A GcloudRunner class to run "gcloud" command. """ project_changed = False if self._NeedProjectSetup(): project_changed = self._UpdateProject(gcloud_runner) if self._NeedClientIDSetup(project_changed): self._SetupClientIDSecret() self._CheckBillingEnable(gcloud_runner) def _UpdateProject(self, gcloud_runner): """Setup gcloud project name and zone name and check project changed. Run "gcloud init" to handle gcloud project setup. Then "gcloud list" to get user settings information include "project" & "zone". Record project_changed for next setup steps. Args: gcloud_runner: A GcloudRunner class to run "gcloud" command. Returns: project_changed: True for project settings changed. """ project_changed = False gcloud_runner.RunGcloud(["init"]) gcp_config_list_out = gcloud_runner.RunGcloud(["config", "list"]) for line in gcp_config_list_out.splitlines(): project_match = _PROJECT_RE.match(line) if project_match: project = project_match.group("project") project_changed = (self.project != project) self.project = project continue zone_match = _ZONE_RE.match(line) if zone_match: self.zone = zone_match.group("zone") continue UpdateConfigFile(self.config_path, "project", self.project) UpdateConfigFile(self.config_path, "zone", self.zone) return project_changed def _SetupClientIDSecret(self): """Setup Client ID / Client Secret in config file. User can use input new values for Client ID and Client Secret. """ print("Please generate a new client ID/secret by following the instructions here:") print("https://support.google.com/cloud/answer/6158849?hl=en") # TODO: Create markdown readme instructions since the link isn't too helpful. self.client_id = None self.client_secret = None while _InputIsEmpty(self.client_id): self.client_id = str(six.moves.input("Enter Client ID: ").strip()) while _InputIsEmpty(self.client_secret): self.client_secret = str(six.moves.input("Enter Client Secret: ").strip()) UpdateConfigFile(self.config_path, "client_id", self.client_id) UpdateConfigFile(self.config_path, "client_secret", self.client_secret) def _CheckBillingEnable(self, gcloud_runner): """Check billing enabled in gcp project. The billing info get by gcloud alpha command. Here is one example: $ gcloud alpha billing projects describe project_name billingAccountName: billingAccounts/011BXX-A30XXX-9XXXX billingEnabled: true name: projects/project_name/billingInfo projectId: project_name Args: gcloud_runner: A GcloudRunner class to run "gcloud" command. Raises: NoBillingError: gcp project doesn't enable billing account. """ billing_info = gcloud_runner.RunGcloud( ["alpha", "billing", "projects", "describe", self.project]) if _BILLING_ENABLE_MSG not in billing_info: raise errors.NoBillingError( "Please set billing account to project(%s) by following the " "instructions here: " "https://cloud.google.com/billing/docs/how-to/modify-project" % self.project) @staticmethod def _EnableGcloudServices(gcloud_runner): """Enable 3 Gcloud API services. 1. Android build service 2. Compute engine service To avoid confuse user, we don't show messages for services processing messages. e.g. "Waiting for async operation operations ...." Args: gcloud_runner: A GcloudRunner class to run "gcloud" command. """ google_apis = [ GoogleAPIService(_ANDROID_BUILD_SERVICE, _ANDROID_BUILD_MSG), GoogleAPIService(_COMPUTE_ENGINE_SERVICE, _COMPUTE_ENGINE_MSG, required=True) ] enabled_services = gcloud_runner.RunGcloud( ["services", "list", "--enabled", "--format", "value(NAME)"], stderr=subprocess.STDOUT).splitlines() for service in google_apis: if service.name not in enabled_services: service.EnableService(gcloud_runner)