1# Copyright 2020 Google LLC
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7#     https://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14
15"""Mounts all the projects required by a selected Build target.
16
17For details on how filesystem overlays work see the filesystem overlays
18section of the README.md.
19"""
20
21from __future__ import absolute_import
22from __future__ import division
23from __future__ import print_function
24
25import collections
26import os
27import subprocess
28import tempfile
29import xml.etree.ElementTree as ET
30from . import config
31
32BindMount = collections.namedtuple('BindMount', ['source_dir', 'readonly'])
33
34
35class BindOverlay(object):
36  """Manages filesystem overlays of Android source tree using bind mounts.
37  """
38
39  MAX_BIND_MOUNTS = 10000
40
41  def _HideDir(self, target_dir):
42    """Temporarily replace the target directory for an empty directory.
43
44    Args:
45      target_dir: A string path to the target directory.
46
47    Returns:
48      A string path to the empty directory that replaced the target directory.
49    """
50    empty_dir = tempfile.mkdtemp(prefix='empty_dir_')
51    self._AddBindMount(empty_dir, target_dir)
52    return empty_dir
53
54  def _FindBindMountConflict(self, path):
55    """Finds any path in the bind mounts that conflicts with the provided path.
56
57    Args:
58      path: A string path to be checked.
59
60    Returns:
61      A string of the conflicting path in the bind mounts.
62      None if there was no conflict found.
63    """
64    conflict_path = None
65    for bind_destination, bind_mount in self._bind_mounts.items():
66      # Check if the path is a subdir or the bind destination
67      if path == bind_destination:
68        conflict_path = bind_mount.source_dir
69        break
70      elif path.startswith(bind_destination + os.sep):
71        relative_path = os.path.relpath(path, bind_destination)
72        path_in_source = os.path.join(bind_mount.source_dir, relative_path)
73        if os.path.exists(path_in_source) and os.listdir(path_in_source):
74          # A conflicting path exists within this bind mount
75          # and it's not empty
76          conflict_path = path_in_source
77          break
78
79    return conflict_path
80
81  def _AddOverlay(self, source_dir, overlay_dir, intermediate_work_dir, skip_subdirs,
82                  allowed_projects, destination_dir, allowed_read_write):
83    """Adds a single overlay directory.
84
85    Args:
86      source_dir: A string with the path to the Android platform source.
87      overlay_dir: A string path to the overlay directory to apply.
88      intermediate_work_dir: A string path to the intermediate work directory used as the
89        base for constructing the overlay filesystem.
90      skip_subdirs: A set of string paths to skip from overlaying.
91      allowed_projects: If not None, any .git project path not in this list
92        is excluded from overlaying.
93      destination_dir: A string with the path to the source with the overlays
94        applied to it.
95      allowed_read_write: A function returns true if the path input should
96        be allowed read/write access.
97    """
98    # Traverse the overlay directory twice
99    # The first pass only process git projects
100    # The second time process all other files that are not in git projects
101
102    # We need to process all git projects first because
103    # the way we process a non-git directory will depend on if
104    # it contains a git project in a subdirectory or not.
105
106    dirs_with_git_projects = set('/')
107    for current_dir_origin, subdirs, files in os.walk(overlay_dir):
108
109      if current_dir_origin in skip_subdirs:
110        del subdirs[:]
111        continue
112
113      current_dir_relative = os.path.relpath(current_dir_origin, overlay_dir)
114      current_dir_destination = os.path.normpath(
115        os.path.join(destination_dir, current_dir_relative))
116
117      if '.git' in subdirs or '.git' in files:
118        # The current dir is a git project
119        # so just bind mount it
120        del subdirs[:]
121
122        if (not allowed_projects or
123            os.path.relpath(current_dir_origin, source_dir) in allowed_projects):
124          if allowed_read_write(current_dir_origin):
125            self._AddBindMount(current_dir_origin, current_dir_destination, False)
126          else:
127            self._AddBindMount(current_dir_origin, current_dir_destination, True)
128
129        current_dir_ancestor = current_dir_origin
130        while current_dir_ancestor and current_dir_ancestor not in dirs_with_git_projects:
131          dirs_with_git_projects.add(current_dir_ancestor)
132          current_dir_ancestor = os.path.dirname(current_dir_ancestor)
133
134    # Process all other files that are not in git projects
135    for current_dir_origin, subdirs, files in os.walk(overlay_dir):
136
137      if current_dir_origin in skip_subdirs:
138        del subdirs[:]
139        continue
140
141      if '.git' in subdirs or '.git' in files:
142        del subdirs[:]
143        continue
144
145      current_dir_relative = os.path.relpath(current_dir_origin, overlay_dir)
146      current_dir_destination = os.path.normpath(
147        os.path.join(destination_dir, current_dir_relative))
148
149      if current_dir_origin in dirs_with_git_projects:
150        # Symbolic links to subdirectories
151        # have to be copied to the intermediate work directory.
152        # We can't bind mount them because bind mounts deference
153        # symbolic links, and the build system filters out any
154        # directory symbolic links.
155        for subdir in subdirs:
156          subdir_origin = os.path.join(current_dir_origin, subdir)
157          if os.path.islink(subdir_origin):
158            if subdir_origin not in skip_subdirs:
159              subdir_destination = os.path.join(intermediate_work_dir,
160                  current_dir_relative, subdir)
161              self._CopyFile(subdir_origin, subdir_destination)
162
163        # bind each file individually then keep travesting
164        for file in files:
165          file_origin = os.path.join(current_dir_origin, file)
166          file_destination = os.path.join(current_dir_destination, file)
167          if allowed_read_write(file_origin):
168            self._AddBindMount(file_origin, file_destination, False)
169          else:
170            self._AddBindMount(file_origin, file_destination, True)
171
172      else:
173        # The current dir does not have any git projects to it can be bind
174        # mounted wholesale
175        del subdirs[:]
176        if allowed_read_write(current_dir_origin):
177          self._AddBindMount(current_dir_origin, current_dir_destination, False)
178        else:
179          self._AddBindMount(current_dir_origin, current_dir_destination, True)
180
181  def _AddArtifactDirectories(self, source_dir, destination_dir, skip_subdirs):
182    """Add directories that were not synced as workspace source.
183
184    Args:
185      source_dir: A string with the path to the Android platform source.
186      destination_dir: A string with the path to the source where the overlays
187        will be applied.
188      skip_subdirs: A set of string paths to be skipped from overlays.
189
190    Returns:
191      A list of string paths to be skipped from overlaying.
192    """
193
194    # Ensure the main out directory exists
195    main_out_dir = os.path.join(source_dir, 'out')
196    if not os.path.exists(main_out_dir):
197      os.makedirs(main_out_dir)
198
199    for subdir in os.listdir(source_dir):
200      if subdir.startswith('out'):
201        out_origin = os.path.join(source_dir, subdir)
202        if out_origin in skip_subdirs:
203          continue
204        out_destination = os.path.join(destination_dir, subdir)
205        self._AddBindMount(out_origin, out_destination, False)
206        skip_subdirs.add(out_origin)
207
208    repo_origin = os.path.join(source_dir, '.repo')
209    if os.path.exists(repo_origin):
210      repo_destination = os.path.normpath(
211        os.path.join(destination_dir, '.repo'))
212      self._AddBindMount(repo_origin, repo_destination, False)
213      skip_subdirs.add(repo_origin)
214
215    return skip_subdirs
216
217  def _AddOverlays(self, source_dir, overlay_dirs, destination_dir,
218                   skip_subdirs, allowed_projects, allowed_read_write):
219    """Add the selected overlay directories.
220
221    Args:
222      source_dir: A string with the path to the Android platform source.
223      overlay_dirs: A list of strings with the paths to the overlay
224        directory to apply.
225      destination_dir: A string with the path to the source where the overlays
226        will be applied.
227      skip_subdirs: A set of string paths to be skipped from overlays.
228      allowed_projects: If not None, any .git project path not in this list
229        is excluded from overlaying.
230      allowed_read_write: A function returns true if the path input should
231        be allowed read/write access.
232    """
233
234    # Create empty intermediate workdir
235    intermediate_work_dir = self._HideDir(destination_dir)
236    overlay_dirs.append(source_dir)
237
238    skip_subdirs = self._AddArtifactDirectories(source_dir, destination_dir,
239        skip_subdirs)
240
241
242    # Bind mount each overlay directory using a
243    # depth first traversal algorithm.
244    #
245    # The algorithm described works under the condition that the overlaid file
246    # systems do not have conflicting projects.
247    #
248    # The results of attempting to overlay two git projects on top
249    # of each other are unpredictable and may push the limits of bind mounts.
250
251    skip_subdirs.add(os.path.join(source_dir, 'overlays'))
252
253    for overlay_dir in overlay_dirs:
254      self._AddOverlay(source_dir, overlay_dir, intermediate_work_dir,
255                       skip_subdirs, allowed_projects,
256                       destination_dir, allowed_read_write)
257
258
259  def _AddBindMount(self, source_dir, destination_dir, readonly=False):
260    """Adds a bind mount for the specified directory.
261
262    Args:
263      source_dir: A string with the path of a source directory to bind.
264        It must already exist.
265      destination_dir: A string with the path ofa destination
266        directory to bind the source into. If it does not exist,
267        it will be created.
268      readonly: A flag to indicate whether this path should be bind mounted
269        with read-only access.
270    """
271    conflict_path = self._FindBindMountConflict(destination_dir)
272    if conflict_path:
273      raise ValueError("Project %s could not be overlaid at %s "
274        "because it conflicts with %s"
275        % (source_dir, destination_dir, conflict_path))
276
277    if len(self._bind_mounts) >= self.MAX_BIND_MOUNTS:
278      raise ValueError("Bind mount limit of %s reached" % self.MAX_BIND_MOUNTS)
279
280    self._bind_mounts[destination_dir] = BindMount(
281        source_dir=source_dir, readonly=readonly)
282
283  def _CopyFile(self, source_path, dest_path):
284    """Copies a file to the specified destination.
285
286    Args:
287      source_path: A string with the path of a source file to copy. It must
288        exist.
289      dest_path: A string with the path to copy the file to. It should not
290        exist.
291    """
292    dest_dir = os.path.dirname(dest_path)
293    if not os.path.exists(dest_dir):
294      os.makedirs(dest_dir)
295    subprocess.check_call(['cp', '--no-dereference', source_path, dest_path])
296
297  def GetBindMounts(self):
298    """Enumerates all bind mounts required by this Overlay.
299
300    Returns:
301      An ordered dict of BindMount objects keyed by destination path string.
302      The order of the bind mounts does matter, this is why it's an ordered
303      dict instead of a standard dict.
304    """
305    return self._bind_mounts
306
307  def _GetReadWriteFunction(self, build_config, source_dir):
308    """Returns a function that tells you how to mount a path.
309
310    Args:
311      build_config: A config.BuildConfig instance of the build target to be
312                    prepared.
313      source_dir: A string with the path to the Android platform source.
314
315    Returns:
316      A function that takes a string path as an input and returns
317      True if the path should be mounted read-write or False if
318      the path should be mounted read-only.
319    """
320
321    # The read/write allowlist provides paths relative to the source dir. It
322    # needs to be updated with absolute paths to make lookup possible.
323    rw_allowlist = {os.path.join(source_dir, p) for p in build_config.allow_readwrite}
324
325    def AllowReadWrite(path):
326      return build_config.allow_readwrite_all or path in rw_allowlist
327
328    return AllowReadWrite
329
330
331  def _GetAllowedProjects(self, build_config):
332    """Returns a set of paths that are allowed to contain .git projects.
333
334    Args:
335      build_config: A config.BuildConfig instance of the build target to be
336                    prepared.
337
338    Returns:
339      If the target has an allowed projects file: a set of paths. Any .git
340        project path not in this set should be excluded from overlaying.
341      Otherwise: None
342    """
343    if not build_config.allowed_projects_file:
344      return None
345    allowed_projects = ET.parse(build_config.allowed_projects_file)
346    paths = set()
347    for child in allowed_projects.getroot().findall("project"):
348      paths.add(child.attrib.get("path", child.attrib["name"]))
349    return paths
350
351
352  def __init__(self,
353               build_target,
354               source_dir,
355               cfg,
356               whiteout_list = [],
357               destination_dir=None,
358               quiet=False):
359    """Inits Overlay with the details of what is going to be overlaid.
360
361    Args:
362      build_target: A string with the name of the build target to be prepared.
363      source_dir: A string with the path to the Android platform source.
364      cfg: A config.Config instance.
365      whiteout_list: A list of directories to hide from the build system.
366      destination_dir: A string with the path where the overlay filesystem
367        will be created. If none is provided, the overlay filesystem
368        will be applied directly on top of source_dir.
369      quiet: A boolean that, when True, suppresses debug output.
370    """
371    self._quiet = quiet
372
373    if not destination_dir:
374      destination_dir = source_dir
375
376    self._overlay_dirs = None
377    # The order of the bind mounts does matter, this is why it's an ordered
378    # dict instead of a standard dict.
379    self._bind_mounts = collections.OrderedDict()
380
381    # We will be repeateadly searching for items to skip so a set
382    # seems appropriate
383    skip_subdirs = set(whiteout_list)
384
385    build_config = cfg.get_build_config(build_target)
386
387    allowed_read_write = self._GetReadWriteFunction(build_config, source_dir)
388    allowed_projects = self._GetAllowedProjects(build_config)
389
390    overlay_dirs = []
391    for overlay_dir in build_config.overlays:
392      overlay_dir = os.path.join(source_dir, 'overlays', overlay_dir)
393      overlay_dirs.append(overlay_dir)
394
395    self._AddOverlays(
396        source_dir, overlay_dirs, destination_dir,
397        skip_subdirs, allowed_projects, allowed_read_write)
398
399    # If specified for this target, create a custom filesystem view
400    for path_relative_from, path_relative_to in build_config.views:
401      path_from = os.path.join(source_dir, path_relative_from)
402      if os.path.isfile(path_from) or os.path.isdir(path_from):
403        path_to = os.path.join(destination_dir, path_relative_to)
404        if allowed_read_write(path_from):
405          self._AddBindMount(path_from, path_to, False)
406        else:
407          self._AddBindMount(path_from, path_to, True)
408      else:
409        raise ValueError("Path '%s' must be a file or directory" % path_from)
410
411    self._overlay_dirs = overlay_dirs
412    if not self._quiet:
413      print('Applied overlays ' + ' '.join(self._overlay_dirs))
414