1# Copyright 2019 - The Android Open Source Project
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7#     http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14
15"""RemoteInstanceDeviceFactory provides basic interface to create a cuttlefish
16device factory."""
17
18import glob
19import logging
20import os
21import shutil
22import tempfile
23
24from acloud.create import create_common
25from acloud.internal import constants
26from acloud.internal.lib import auth
27from acloud.internal.lib import cvd_compute_client_multi_stage
28from acloud.internal.lib import utils
29from acloud.internal.lib import ssh
30from acloud.public.actions import gce_device_factory
31
32
33logger = logging.getLogger(__name__)
34
35class RemoteInstanceDeviceFactory(gce_device_factory.GCEDeviceFactory):
36    """A class that can produce a cuttlefish device.
37
38    Attributes:
39        avd_spec: AVDSpec object that tells us what we're going to create.
40        cfg: An AcloudConfig instance.
41        local_image_artifact: A string, path to local image.
42        cvd_host_package_artifact: A string, path to cvd host package.
43        report_internal_ip: Boolean, True for the internal ip is used when
44                            connecting from another GCE instance.
45        credentials: An oauth2client.OAuth2Credentials instance.
46        compute_client: An object of cvd_compute_client.CvdComputeClient.
47        ssh: An Ssh object.
48    """
49    def __init__(self, avd_spec, local_image_artifact=None,
50                 cvd_host_package_artifact=None):
51        super(RemoteInstanceDeviceFactory, self).__init__(avd_spec, local_image_artifact)
52        self._cvd_host_package_artifact = cvd_host_package_artifact
53
54    # pylint: disable=broad-except
55    def CreateInstance(self):
56        """Create a single configured cuttlefish device.
57
58        GCE:
59        1. Create gcp instance.
60        2. Upload local built artifacts to remote instance or fetch build on
61           remote instance.
62        3. Launch CVD.
63
64        Remote host:
65        1. Init remote host.
66        2. Download the artifacts to local and upload the artifacts to host
67        3. Launch CVD.
68
69        Returns:
70            A string, representing instance name.
71        """
72        if self._avd_spec.instance_type == constants.INSTANCE_TYPE_HOST:
73            instance = self._InitRemotehost()
74            self._ProcessRemoteHostArtifacts()
75            self._LaunchCvd(instance=instance,
76                            decompress_kernel=True,
77                            boot_timeout_secs=self._avd_spec.boot_timeout_secs)
78        else:
79            instance = self._CreateGceInstance()
80            # If instance is failed, no need to go next step.
81            if instance in self.GetFailures():
82                return instance
83            try:
84                self._ProcessArtifacts(self._avd_spec.image_source)
85                self._LaunchCvd(instance=instance,
86                                boot_timeout_secs=self._avd_spec.boot_timeout_secs)
87            except Exception as e:
88                self._SetFailures(instance, e)
89
90        return instance
91
92    def _InitRemotehost(self):
93        """Initialize remote host.
94
95        Determine the remote host instance name, and activate ssh. It need to
96        get the IP address in the common_operation. So need to pass the IP and
97        ssh to compute_client.
98
99        build_target: The format is like "aosp_cf_x86_phone". We only get info
100                      from the user build image file name. If the file name is
101                      not custom format (no "-"), we will use $TARGET_PRODUCT
102                      from environment variable as build_target.
103
104        Returns:
105            A string, representing instance name.
106        """
107        image_name = os.path.basename(
108            self._local_image_artifact) if self._local_image_artifact else ""
109        build_target = (os.environ.get(constants.ENV_BUILD_TARGET) if "-" not
110                        in image_name else image_name.split("-")[0])
111        build_id = self._USER_BUILD
112        if self._avd_spec.image_source == constants.IMAGE_SRC_REMOTE:
113            build_id = self._avd_spec.remote_image[constants.BUILD_ID]
114
115        instance = "%s-%s-%s-%s" % (constants.INSTANCE_TYPE_HOST,
116                                    self._avd_spec.remote_host,
117                                    build_id, build_target)
118        ip = ssh.IP(ip=self._avd_spec.remote_host)
119        self._ssh = ssh.Ssh(
120            ip=ip,
121            user=self._avd_spec.host_user,
122            ssh_private_key_path=(self._avd_spec.host_ssh_private_key_path or
123                                  self._cfg.ssh_private_key_path),
124            extra_args_ssh_tunnel=self._cfg.extra_args_ssh_tunnel,
125            report_internal_ip=self._report_internal_ip)
126        self._compute_client.InitRemoteHost(
127            self._ssh, ip, self._avd_spec.host_user)
128        return instance
129
130    @utils.TimeExecute(function_description="Downloading Android Build artifact")
131    def _DownloadArtifacts(self, extract_path):
132        """Download the CF image artifacts and process them.
133
134        - Download image from the Android Build system, then decompress it.
135        - Download cvd host package from the Android Build system.
136
137        Args:
138            extract_path: String, a path include extracted files.
139        """
140        cfg = self._avd_spec.cfg
141        build_id = self._avd_spec.remote_image[constants.BUILD_ID]
142        build_target = self._avd_spec.remote_image[constants.BUILD_TARGET]
143
144        # Image zip
145        remote_image = "%s-img-%s.zip" % (build_target.split('-')[0], build_id)
146        create_common.DownloadRemoteArtifact(
147            cfg, build_target, build_id, remote_image, extract_path, decompress=True)
148
149        # Cvd host package
150        create_common.DownloadRemoteArtifact(
151            cfg, build_target, build_id, constants.CVD_HOST_PACKAGE,
152            extract_path)
153
154    def _ProcessRemoteHostArtifacts(self):
155        """Process remote host artifacts.
156
157        - If images source is local, tool will upload images from local site to
158          remote host.
159        - If images source is remote, tool will download images from android
160          build to local and unzip it then upload to remote host, because there
161          is no permission to fetch build rom on the remote host.
162        """
163        if self._avd_spec.image_source == constants.IMAGE_SRC_LOCAL:
164            self._UploadArtifacts(
165                self._local_image_artifact, self._cvd_host_package_artifact,
166                self._avd_spec.local_image_dir)
167        else:
168            try:
169                artifacts_path = tempfile.mkdtemp()
170                logger.debug("Extracted path of artifacts: %s", artifacts_path)
171                self._DownloadArtifacts(artifacts_path)
172                self._UploadArtifacts(
173                    None,
174                    os.path.join(artifacts_path, constants.CVD_HOST_PACKAGE),
175                    artifacts_path)
176            finally:
177                shutil.rmtree(artifacts_path)
178
179    def _ProcessArtifacts(self, image_source):
180        """Process artifacts.
181
182        - If images source is local, tool will upload images from local site to
183          remote instance.
184        - If images source is remote, tool will download images from android
185          build to remote instance. Before download images, we have to update
186          fetch_cvd to remote instance.
187
188        Args:
189            image_source: String, the type of image source is remote or local.
190        """
191        if image_source == constants.IMAGE_SRC_LOCAL:
192            self._UploadArtifacts(self._local_image_artifact,
193                                  self._cvd_host_package_artifact,
194                                  self._avd_spec.local_image_dir)
195        elif image_source == constants.IMAGE_SRC_REMOTE:
196            self._compute_client.UpdateFetchCvd()
197            self._FetchBuild(
198                self._avd_spec.remote_image[constants.BUILD_ID],
199                self._avd_spec.remote_image[constants.BUILD_BRANCH],
200                self._avd_spec.remote_image[constants.BUILD_TARGET],
201                self._avd_spec.system_build_info[constants.BUILD_ID],
202                self._avd_spec.system_build_info[constants.BUILD_BRANCH],
203                self._avd_spec.system_build_info[constants.BUILD_TARGET],
204                self._avd_spec.kernel_build_info[constants.BUILD_ID],
205                self._avd_spec.kernel_build_info[constants.BUILD_BRANCH],
206                self._avd_spec.kernel_build_info[constants.BUILD_TARGET])
207
208    def _FetchBuild(self, build_id, branch, build_target, system_build_id,
209                    system_branch, system_build_target, kernel_build_id,
210                    kernel_branch, kernel_build_target):
211        """Download CF artifacts from android build.
212
213        Args:
214            build_branch: String, git branch name. e.g. "aosp-master"
215            build_target: String, the build target, e.g. cf_x86_phone-userdebug
216            build_id: String, build id, e.g. "2263051", "P2804227"
217            kernel_branch: Kernel branch name, e.g. "kernel-common-android-4.14"
218            kernel_build_id: Kernel build id, a string, e.g. "223051", "P280427"
219            kernel_build_target: String, Kernel build target name.
220            system_build_target: Target name for the system image,
221                                 e.g. "cf_x86_phone-userdebug"
222            system_branch: A String, branch name for the system image.
223            system_build_id: A string, build id for the system image.
224
225        """
226        self._compute_client.FetchBuild(
227            build_id, branch, build_target, system_build_id,
228            system_branch, system_build_target, kernel_build_id,
229            kernel_branch, kernel_build_target)
230
231    @utils.TimeExecute(function_description="Processing and uploading local images")
232    def _UploadArtifacts(self,
233                         local_image_zip,
234                         cvd_host_package_artifact,
235                         images_dir):
236        """Upload local images and avd local host package to instance.
237
238        There are two ways to upload local images.
239        1. Using local image zip, it would be decompressed by install_zip.sh.
240        2. Using local image directory, this directory contains all images.
241           Images are compressed/decompressed by lzop during upload process.
242
243        Args:
244            local_image_zip: String, path to zip of local images which
245                             build from 'm dist'.
246            cvd_host_package_artifact: String, path to cvd host package.
247            images_dir: String, directory of local images which build
248                        from 'm'.
249        """
250        if local_image_zip:
251            remote_cmd = ("/usr/bin/install_zip.sh . < %s" % local_image_zip)
252            logger.debug("remote_cmd:\n %s", remote_cmd)
253            self._ssh.Run(remote_cmd)
254        else:
255            # Compress image files for faster upload.
256            try:
257                images_path = os.path.join(images_dir, "required_images")
258                with open(images_path, "r") as images:
259                    artifact_files = images.read().splitlines()
260            except IOError:
261                # Older builds may not have a required_images file. In this case
262                # we fall back to *.img.
263                artifact_files = [os.path.basename(image) for image in
264                    glob.glob(os.path.join(images_dir, "*.img"))]
265            cmd = ("tar -cf - --lzop -S -C {images_dir} {artifact_files} | "
266                   "{ssh_cmd} -- tar -xf - --lzop -S".format(
267                       images_dir=images_dir,
268                       artifact_files=" ".join(artifact_files),
269                       ssh_cmd=self._ssh.GetBaseCmd(constants.SSH_BIN)))
270            logger.debug("cmd:\n %s", cmd)
271            ssh.ShellCmdWithRetry(cmd)
272
273        # host_package
274        remote_cmd = ("tar -x -z -f - < %s" % cvd_host_package_artifact)
275        logger.debug("remote_cmd:\n %s", remote_cmd)
276        self._ssh.Run(remote_cmd)
277
278    def _LaunchCvd(self, instance, decompress_kernel=None,
279                   boot_timeout_secs=None):
280        """Launch CVD.
281
282        Args:
283            instance: String, instance name.
284            boot_timeout_secs: Integer, the maximum time to wait for the
285                               command to respond.
286        """
287        kernel_build = None
288        # TODO(b/140076771) Support kernel image for local image mode.
289        if self._avd_spec.image_source == constants.IMAGE_SRC_REMOTE:
290            kernel_build = self._compute_client.GetKernelBuild(
291                self._avd_spec.kernel_build_info[constants.BUILD_ID],
292                self._avd_spec.kernel_build_info[constants.BUILD_BRANCH],
293                self._avd_spec.kernel_build_info[constants.BUILD_TARGET])
294        self._compute_client.LaunchCvd(
295            instance,
296            self._avd_spec,
297            self._cfg.extra_data_disk_size_gb,
298            kernel_build,
299            decompress_kernel,
300            boot_timeout_secs)
301
302    def GetBuildInfoDict(self):
303        """Get build info dictionary.
304
305        Returns:
306            A build info dictionary. None for local image case.
307        """
308        if self._avd_spec.image_source == constants.IMAGE_SRC_LOCAL:
309            return None
310        build_info_dict = {
311            key: val for key, val in self._avd_spec.remote_image.items() if val}
312
313        # kernel_target have default value "kernel". If user provide kernel_build_id
314        # or kernel_branch, then start to process kernel image.
315        if (self._avd_spec.kernel_build_info[constants.BUILD_ID]
316                or self._avd_spec.kernel_build_info[constants.BUILD_BRANCH]):
317            build_info_dict.update(
318                {"kernel_%s" % key: val
319                 for key, val in self._avd_spec.kernel_build_info.items() if val}
320            )
321        build_info_dict.update(
322            {"system_%s" % key: val
323             for key, val in self._avd_spec.system_build_info.items() if val}
324        )
325        return build_info_dict
326