1#!/usr/bin/env python3
2#
3# Copyright 2020 - The Android Open Source Project
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9#     http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16
17"""Separate the sources from multiple projects."""
18
19import os
20
21from aidegen import constant
22from aidegen.idea import iml
23from aidegen.lib import common_util
24from aidegen.lib import project_config
25
26_KEY_SOURCE_PATH = 'source_folder_path'
27_KEY_TEST_PATH = 'test_folder_path'
28_SOURCE_FOLDERS = [_KEY_SOURCE_PATH, _KEY_TEST_PATH]
29_KEY_SRCJAR_PATH = 'srcjar_path'
30_KEY_R_PATH = 'r_java_path'
31_KEY_JAR_PATH = 'jar_path'
32_EXCLUDE_ITEM = '\n            <excludeFolder url="file://%s" />'
33# Temporarily exclude test-dump and src_stub folders to prevent symbols from
34# resolving failure by incorrect reference. These two folders should be removed
35# after b/136982078 is resolved.
36_EXCLUDE_FOLDERS = ['.idea', '.repo', 'art', 'bionic', 'bootable', 'build',
37                    'dalvik', 'developers', 'device', 'hardware', 'kernel',
38                    'libnativehelper', 'pdk', 'prebuilts', 'sdk', 'system',
39                    'toolchain', 'tools', 'vendor', 'out',
40                    'art/tools/ahat/src/test-dump',
41                    'cts/common/device-side/device-info/src_stub']
42
43
44class ProjectSplitter:
45    """Splits the sources from multiple projects.
46
47    It's a specific solution to deal with the source folders in multiple
48    project case. Since the IntelliJ does not allow duplicate source folders,
49    AIDEGen needs to separate the source folders for each project. The single
50    project case has no different with current structure.
51
52    Usage:
53    project_splitter = ProjectSplitter(projects)
54
55    # Find the dependencies between the projects.
56    project_splitter.get_dependencies()
57
58    # Clear the source folders for each project.
59    project_splitter.revise_source_folders()
60
61    Attributes:
62        _projects: A list of ProjectInfo.
63        _all_srcs: A dictionary contains all sources of multiple projects.
64                   e.g.
65                   {
66                       'module_name': 'test',
67                       'path': ['path/to/module'],
68                       'srcs': ['src_folder1', 'src_folder2'],
69                       'tests': ['test_folder1', 'test_folder2']
70                       'jars': ['jar1.jar'],
71                       'srcjars': ['1.srcjar', '2.srcjar'],
72                       'dependencies': ['framework_srcjars', 'base'],
73                       'iml_name': '/abs/path/to/iml.iml'
74                   }
75        _framework_exist: A boolean, True if framework is one of the projects.
76        _framework_iml: A string, the name of the framework's iml.
77        _full_repo: A boolean, True if loading with full Android sources.
78        _full_repo_iml: A string, the name of the Android folder's iml.
79    """
80    def __init__(self, projects):
81        """ProjectSplitter initialize.
82
83        Args:
84            projects: A list of ProjectInfo object.
85        """
86        self._projects = projects
87        self._all_srcs = dict(projects[0].source_path)
88        self._framework_iml = None
89        self._framework_exist = any(
90            {p.project_relative_path == constant.FRAMEWORK_PATH
91             for p in self._projects})
92        if self._framework_exist:
93            self._framework_iml = iml.IMLGenerator.get_unique_iml_name(
94                os.path.join(common_util.get_android_root_dir(),
95                             constant.FRAMEWORK_PATH))
96        self._full_repo = project_config.ProjectConfig.get_instance().full_repo
97        if self._full_repo:
98            self._full_repo_iml = os.path.basename(
99                common_util.get_android_root_dir())
100
101    def revise_source_folders(self):
102        """Resets the source folders of each project.
103
104        There should be no duplicate source root path in IntelliJ. The issue
105        doesn't happen in single project case. Once users choose multiple
106        projects, there could be several same source paths of different
107        projects. In order to prevent that, we should remove the source paths
108        in dependencies.iml which are duplicate with the paths in [module].iml
109        files.
110
111        Steps to prevent the duplicate source root path in IntelliJ:
112        1. Copy all sources from sub-projects to main project.
113        2. Delete the source and test folders which are not under the
114           sub-projects.
115        3. Delete the sub-projects' source and test paths from the main project.
116        """
117        self._collect_all_srcs()
118        self._keep_local_sources()
119        self._remove_duplicate_sources()
120
121    def _collect_all_srcs(self):
122        """Copies all projects' sources to a dictionary."""
123        for project in self._projects[1:]:
124            for key, value in project.source_path.items():
125                self._all_srcs[key].update(value)
126
127    def _keep_local_sources(self):
128        """Removes source folders which are not under the project's path.
129
130        1. Remove the source folders which are not under the project.
131        2. Remove the duplicate project's source folders from the _all_srcs.
132        """
133        for project in self._projects:
134            srcs = project.source_path
135            relpath = project.project_relative_path
136            is_root = not relpath
137            for key in _SOURCE_FOLDERS:
138                srcs[key] = {s for s in srcs[key]
139                             if common_util.is_source_under_relative_path(
140                                 s, relpath) or is_root}
141                self._all_srcs[key] -= srcs[key]
142
143    def _remove_duplicate_sources(self):
144        """Removes the duplicate source folders from each sub project.
145
146        Priority processing with the longest path length, e.g.
147        frameworks/base/packages/SettingsLib must have priority over
148        frameworks/base.
149        (b/160303006): Remove the parent project's source and test paths under
150        the child's project path.
151        """
152        root = common_util.get_android_root_dir()
153        projects = sorted(self._projects, key=lambda k: len(
154            k.project_relative_path), reverse=True)
155        for child in projects:
156            for parent in self._projects:
157                is_root = not parent.project_relative_path
158                if parent is child:
159                    continue
160                if (common_util.is_source_under_relative_path(
161                        child.project_relative_path,
162                        parent.project_relative_path) or is_root):
163                    for key in _SOURCE_FOLDERS:
164                        parent.source_path[key] -= child.source_path[key]
165                        rm_paths = _remove_child_duplicate_sources_from_parent(
166                            child, parent.source_path[key], root)
167                        parent.source_path[key] -= rm_paths
168
169    def get_dependencies(self):
170        """Gets the dependencies between the projects.
171
172        Check if the current project's source folder exists in other projects.
173        If do, the current project is a dependency module to the other.
174        """
175        projects = sorted(self._projects, key=lambda k: len(
176            k.project_relative_path))
177        for project in projects:
178            proj_path = project.project_relative_path
179            project.dependencies = [constant.FRAMEWORK_SRCJARS]
180            if self._framework_exist and proj_path != constant.FRAMEWORK_PATH:
181                project.dependencies.append(self._framework_iml)
182            if self._full_repo and proj_path:
183                project.dependencies.append(self._full_repo_iml)
184            srcs = (project.source_path[_KEY_SOURCE_PATH]
185                    | project.source_path[_KEY_TEST_PATH])
186            dep_projects = sorted(self._projects, key=lambda k: len(
187                k.project_relative_path))
188            for dep_proj in dep_projects:
189                dep_path = dep_proj.project_relative_path
190                is_root = not dep_path
191                is_child = common_util.is_source_under_relative_path(dep_path,
192                                                                     proj_path)
193                is_dep = any({s for s in srcs
194                              if common_util.is_source_under_relative_path(
195                                  s, dep_path) or is_root})
196                if dep_proj is project or is_child or not is_dep:
197                    continue
198                dep = iml.IMLGenerator.get_unique_iml_name(os.path.join(
199                    common_util.get_android_root_dir(), dep_path))
200                if dep not in project.dependencies:
201                    project.dependencies.append(dep)
202            project.dependencies.append(constant.KEY_DEPENDENCIES)
203
204    def gen_framework_srcjars_iml(self):
205        """Generates the framework-srcjars.iml.
206
207        Create the iml file with only the srcjars of module framework-all. These
208        srcjars will be separated from the modules under frameworks/base.
209
210        Returns:
211            A string of the framework_srcjars.iml's absolute path.
212        """
213        mod = dict(self._projects[0].dep_modules[constant.FRAMEWORK_ALL])
214        mod[constant.KEY_DEPENDENCIES] = []
215        mod[constant.KEY_IML_NAME] = constant.FRAMEWORK_SRCJARS
216        if self._framework_exist:
217            mod[constant.KEY_DEPENDENCIES].append(self._framework_iml)
218        if self._full_repo:
219            mod[constant.KEY_DEPENDENCIES].append(self._full_repo_iml)
220        mod[constant.KEY_DEPENDENCIES].append(constant.KEY_DEPENDENCIES)
221        framework_srcjars_iml = iml.IMLGenerator(mod)
222        framework_srcjars_iml.create({constant.KEY_SRCJARS: True,
223                                      constant.KEY_DEPENDENCIES: True})
224        self._all_srcs[_KEY_SRCJAR_PATH] -= set(mod[constant.KEY_SRCJARS])
225        return framework_srcjars_iml.iml_path
226
227    def _gen_dependencies_iml(self):
228        """Generates the dependencies.iml."""
229        mod = {
230            constant.KEY_SRCS: self._all_srcs[_KEY_SOURCE_PATH],
231            constant.KEY_TESTS: self._all_srcs[_KEY_TEST_PATH],
232            constant.KEY_JARS: self._all_srcs[_KEY_JAR_PATH],
233            constant.KEY_SRCJARS: (self._all_srcs[_KEY_R_PATH]
234                                   | self._all_srcs[_KEY_SRCJAR_PATH]),
235            constant.KEY_DEPENDENCIES: [constant.FRAMEWORK_SRCJARS],
236            constant.KEY_PATH: [self._projects[0].project_relative_path],
237            constant.KEY_MODULE_NAME: constant.KEY_DEPENDENCIES,
238            constant.KEY_IML_NAME: constant.KEY_DEPENDENCIES
239        }
240        if self._framework_exist:
241            mod[constant.KEY_DEPENDENCIES].append(self._framework_iml)
242        if self._full_repo:
243            mod[constant.KEY_DEPENDENCIES].append(self._full_repo_iml)
244        dep_iml = iml.IMLGenerator(mod)
245        dep_iml.create({constant.KEY_DEP_SRCS: True,
246                        constant.KEY_SRCJARS: True,
247                        constant.KEY_JARS: True,
248                        constant.KEY_DEPENDENCIES: True})
249
250    def gen_projects_iml(self):
251        """Generates the projects' iml file."""
252        root_path = common_util.get_android_root_dir()
253        excludes = project_config.ProjectConfig.get_instance().exclude_paths
254        for project in self._projects:
255            relpath = project.project_relative_path
256            exclude_folders = []
257            if not relpath:
258                exclude_folders.extend(get_exclude_content(root_path))
259            if excludes:
260                exclude_folders.extend(get_exclude_content(root_path, excludes))
261            mod_info = {
262                constant.KEY_EXCLUDES: ''.join(exclude_folders),
263                constant.KEY_SRCS: project.source_path[_KEY_SOURCE_PATH],
264                constant.KEY_TESTS: project.source_path[_KEY_TEST_PATH],
265                constant.KEY_DEPENDENCIES: project.dependencies,
266                constant.KEY_PATH: [relpath],
267                constant.KEY_MODULE_NAME: project.module_name,
268                constant.KEY_IML_NAME: iml.IMLGenerator.get_unique_iml_name(
269                    os.path.join(root_path, relpath))
270            }
271            dep_iml = iml.IMLGenerator(mod_info)
272            dep_iml.create({constant.KEY_SRCS: True,
273                            constant.KEY_DEPENDENCIES: True})
274            project.iml_path = dep_iml.iml_path
275        self._gen_dependencies_iml()
276
277
278def get_exclude_content(root_path, excludes=None):
279    """Get the exclude folder content list.
280
281    It returns the exclude folders content list.
282    e.g.
283    ['<excludeFolder url="file://a/.idea" />',
284    '<excludeFolder url="file://a/.repo" />']
285
286    Args:
287        root_path: Android source file path.
288        excludes: A list of exclusive directories, the default value is None but
289                  will be assigned to _EXCLUDE_FOLDERS.
290
291    Returns:
292        String: exclude folder content list.
293    """
294    exclude_items = []
295    if not excludes:
296        excludes = _EXCLUDE_FOLDERS
297    for folder in excludes:
298        folder_path = os.path.join(root_path, folder)
299        if os.path.isdir(folder_path):
300            exclude_items.append(_EXCLUDE_ITEM % folder_path)
301    return exclude_items
302
303def _remove_child_duplicate_sources_from_parent(child, parent_sources, root):
304    """Removes the child's duplicate source folders from the parent source list.
305
306    Remove all the child's subdirectories from the parent's source list if thers
307    is any.
308
309    Args:
310        child: A child project of ProjectInfo instance.
311        parent: The parent project of ProjectInfo instance.
312        root: A string of the Android root.
313
314    Returns:
315        A set of the sources to be removed.
316    """
317    rm_paths = set()
318    for path in parent_sources:
319        if (common_util.is_source_under_relative_path(
320                os.path.relpath(path, root), child.project_relative_path)):
321            rm_paths.add(path)
322    return rm_paths
323