1#!/usr/bin/env python
2#
3# Copyright (C) 2017 The Android Open Source Project
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9#      http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16#
17
18import argparse
19import glob
20import logging
21import os
22import shutil
23import subprocess
24import tempfile
25import xml.etree.ElementTree as xml_tree
26
27import utils
28
29
30class GPLChecker(object):
31    """Checks that all GPL projects in a VNDK snapshot have released sources.
32
33    Makes sure that the current source tree have the sources for all GPL
34    prebuilt libraries in a specified VNDK snapshot version.
35    """
36    MANIFEST_XML = utils.MANIFEST_FILE_NAME
37    MODULE_PATHS_TXT = utils.MODULE_PATHS_FILE_NAME
38
39    def __init__(self, install_dir, android_build_top, temp_artifact_dir,
40                 remote_git):
41        """GPLChecker constructor.
42
43        Args:
44          install_dir: string, absolute path to the prebuilts/vndk/v{version}
45            directory where the build files will be generated.
46          android_build_top: string, absolute path to ANDROID_BUILD_TOP
47          temp_artifact_dir: string, temp directory to hold build artifacts
48            fetched from Android Build server.
49          remote_git: string, remote name to fetch and check if the revision of
50            VNDK snapshot is included in the source if it is not in the current
51            git repository.
52        """
53        self._android_build_top = android_build_top
54        self._install_dir = install_dir
55        self._remote_git = remote_git
56        self._manifest_file = os.path.join(temp_artifact_dir,
57                                           self.MANIFEST_XML)
58        self._notice_files_dir = os.path.join(install_dir,
59                                              utils.NOTICE_FILES_DIR_PATH)
60
61        if not os.path.isfile(self._manifest_file):
62            raise RuntimeError(
63                '{manifest} not found at {manifest_file}'.format(
64                    manifest=self.MANIFEST_XML,
65                    manifest_file=self._manifest_file))
66
67    def _parse_module_paths(self):
68        """Parses the module_paths.txt files into a dictionary,
69
70        Returns:
71          module_paths: dict, e.g. {libfoo.so: some/path/here}
72        """
73        module_paths = dict()
74        for file in utils.find(self._install_dir, [self.MODULE_PATHS_TXT]):
75            file_path = os.path.join(self._install_dir, file)
76            with open(file_path, 'r') as f:
77                for line in f.read().strip().split('\n'):
78                    paths = line.split(' ')
79                    if len(paths) > 1:
80                        if paths[0] not in module_paths:
81                            module_paths[paths[0]] = paths[1]
82        return module_paths
83
84    def _parse_manifest(self):
85        """Parses manifest.xml file and returns list of 'project' tags."""
86
87        root = xml_tree.parse(self._manifest_file).getroot()
88        return root.findall('project')
89
90    def _get_revision(self, module_path, manifest_projects):
91        """Returns revision value recorded in manifest.xml for given project.
92
93        Args:
94          module_path: string, project path relative to ANDROID_BUILD_TOP
95          manifest_projects: list of xml_tree.Element, list of 'project' tags
96        """
97        revision = None
98        for project in manifest_projects:
99            path = project.get('path')
100            if module_path.startswith(path):
101                revision = project.get('revision')
102                break
103        return revision
104
105    def _check_revision_exists(self, revision, git_project_path):
106        """Checks whether a revision is found in a git project of current tree.
107
108        Args:
109          revision: string, revision value recorded in manifest.xml
110          git_project_path: string, path relative to ANDROID_BUILD_TOP
111        """
112        path = utils.join_realpath(self._android_build_top, git_project_path)
113
114        def _check_rev_list(revision):
115            """Checks whether revision is reachable from HEAD of git project."""
116
117            logging.info('Checking if revision {rev} exists in {proj}'.format(
118                rev=revision, proj=git_project_path))
119            try:
120                cmd = [
121                    'git', '-C', path, 'rev-list', 'HEAD..{}'.format(revision)
122                ]
123                output = utils.check_output(cmd).strip()
124            except subprocess.CalledProcessError as error:
125                logging.error('Error: {}'.format(error))
126                return False
127            else:
128                if output:
129                    logging.debug(
130                        '{proj} does not have the following revisions: {rev}'.
131                        format(proj=git_project_path, rev=output))
132                    return False
133                else:
134                    logging.info(
135                        'Found revision {rev} in project {proj}'.format(
136                            rev=revision, proj=git_project_path))
137            return True
138
139        def _get_2nd_parent_if_merge_commit(revision):
140            """Checks if the commit is merge commit.
141
142            Returns:
143              revision: string, the 2nd parent which is the merged commit.
144              If the commit is not a merge commit, returns None.
145            """
146            logging.info(
147                'Checking if the parent of revision {rev} exists in {proj}'.
148                format(rev=revision, proj=git_project_path))
149            try:
150                cmd = [
151                    'git', '-C', path, 'rev-parse', '--verify',
152                    '{}^2'.format(revision)]
153                parent_revision = utils.check_output(cmd).strip()
154            except subprocess.CalledProcessError as error:
155                logging.error(
156                    'Failed to get parent of revision {rev} from "{remote}": '
157                    '{err}'.format(
158                        rev=revision, remote=self._remote_git, err=error))
159                logging.error('{} is not a merge commit and must be included '
160                    'in the current branch'.format(revision))
161                return None
162            else:
163                return parent_revision
164
165        if _check_rev_list(revision):
166            return True
167
168        # VNDK snapshots built from a *-release branch will have merge
169        # CLs in the manifest because the *-dev branch is merged to the
170        # *-release branch periodically. In order to extract the
171        # revision relevant to the source of the git_project_path,
172        # we find the parent of the merge commit.
173        try:
174            cmd = ['git', '-C', path, 'fetch', self._remote_git, revision]
175            utils.check_call(cmd)
176        except subprocess.CalledProcessError as error:
177            logging.error(
178                'Failed to fetch revision {rev} from "{remote}": '
179                '{err}'.format(
180                    rev=revision, remote=self._remote_git, err=error))
181            logging.error('Try --remote to manually set remote name')
182            raise
183
184        parent_revision = _get_2nd_parent_if_merge_commit(revision)
185        while True:
186            if not parent_revision:
187                return False
188            if _check_rev_list(parent_revision):
189                return True
190            parent_revision = _get_2nd_parent_if_merge_commit(parent_revision)
191
192    def check_gpl_projects(self):
193        """Checks that all GPL projects have released sources.
194
195        Raises:
196          ValueError: There are GPL projects with unreleased sources.
197        """
198        logging.info('Starting license check for GPL projects...')
199
200        notice_files = glob.glob('{}/*'.format(self._notice_files_dir))
201        if len(notice_files) == 0:
202            raise RuntimeError('No license files found in {}'.format(
203                self._notice_files_dir))
204
205        gpl_projects = []
206        pattern = 'GENERAL PUBLIC LICENSE'
207        for notice_file_path in notice_files:
208            with open(notice_file_path, 'r') as notice_file:
209                if pattern in notice_file.read():
210                    lib_name = os.path.splitext(
211                        os.path.basename(notice_file_path))[0]
212                    gpl_projects.append(lib_name)
213
214        if not gpl_projects:
215            logging.info('No GPL projects found.')
216            return
217
218        logging.info('GPL projects found: {}'.format(', '.join(gpl_projects)))
219
220        module_paths = self._parse_module_paths()
221        manifest_projects = self._parse_manifest()
222        released_projects = []
223        unreleased_projects = []
224
225        for lib in gpl_projects:
226            if lib in module_paths:
227                module_path = module_paths[lib]
228                revision = self._get_revision(module_path, manifest_projects)
229                if not revision:
230                    raise RuntimeError(
231                        'No project found for {path} in {manifest}'.format(
232                            path=module_path, manifest=self.MANIFEST_XML))
233                revision_exists = self._check_revision_exists(
234                    revision, module_path)
235                if not revision_exists:
236                    unreleased_projects.append((lib, module_path))
237                else:
238                    released_projects.append((lib, module_path))
239            else:
240                raise RuntimeError(
241                    'No module path was found for {lib} in {module_paths}'.
242                    format(lib=lib, module_paths=self.MODULE_PATHS_TXT))
243
244        if released_projects:
245            logging.info('Released GPL projects: {}'.format(released_projects))
246
247        if unreleased_projects:
248            raise ValueError(
249                ('FAIL: The following GPL projects have NOT been released in '
250                 'current tree: {}'.format(unreleased_projects)))
251
252        logging.info('PASS: All GPL projects have source in current tree.')
253
254
255def get_args():
256    parser = argparse.ArgumentParser()
257    parser.add_argument(
258        'vndk_version',
259        type=int,
260        help='VNDK snapshot version to check, e.g. "27".')
261    parser.add_argument('-b', '--branch', help='Branch to pull manifest from.')
262    parser.add_argument('--build', help='Build number to pull manifest from.')
263    parser.add_argument(
264        '--remote',
265        default='aosp',
266        help=('Remote name to fetch and check if the revision of VNDK snapshot '
267              'is included in the source to conform GPL license. default=aosp'))
268    parser.add_argument(
269        '-v',
270        '--verbose',
271        action='count',
272        default=0,
273        help='Increase output verbosity, e.g. "-v", "-vv".')
274    return parser.parse_args()
275
276
277def main():
278    """For local testing purposes.
279
280    Note: VNDK snapshot must be already installed under
281      prebuilts/vndk/v{version}.
282    """
283    ANDROID_BUILD_TOP = utils.get_android_build_top()
284    PREBUILTS_VNDK_DIR = utils.join_realpath(ANDROID_BUILD_TOP,
285                                             'prebuilts/vndk')
286
287    args = get_args()
288    vndk_version = args.vndk_version
289    install_dir = os.path.join(PREBUILTS_VNDK_DIR, 'v{}'.format(vndk_version))
290    remote = args.remote
291    if not os.path.isdir(install_dir):
292        raise ValueError(
293            'Please provide valid VNDK version. {} does not exist.'
294            .format(install_dir))
295    utils.set_logging_config(args.verbose)
296
297    temp_artifact_dir = tempfile.mkdtemp()
298    os.chdir(temp_artifact_dir)
299    manifest_pattern = 'manifest_{}.xml'.format(args.build)
300    manifest_dest = os.path.join(temp_artifact_dir, utils.MANIFEST_FILE_NAME)
301    logging.info('Fetching {file} from {branch} (bid: {build})'.format(
302        file=manifest_pattern, branch=args.branch, build=args.build))
303    utils.fetch_artifact(args.branch, args.build, manifest_pattern,
304                         manifest_dest)
305
306    license_checker = GPLChecker(install_dir, ANDROID_BUILD_TOP,
307                                 temp_artifact_dir, remote)
308    try:
309        license_checker.check_gpl_projects()
310    except ValueError as error:
311        logging.error('Error: {}'.format(error))
312        raise
313    finally:
314        logging.info(
315            'Deleting temp_artifact_dir: {}'.format(temp_artifact_dir))
316        shutil.rmtree(temp_artifact_dir)
317
318    logging.info('Done.')
319
320
321if __name__ == '__main__':
322    main()
323