1#!/usr/bin/env python3
2#
3# Copyright (C) 2020 The Android Open Source Project
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9#      http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16#
17"""Installs vendor snapshot under prebuilts/vendor/v{version}."""
18
19import argparse
20import glob
21import logging
22import os
23import re
24import shutil
25import subprocess
26import sys
27import tempfile
28import textwrap
29import json
30
31INDENT = ' ' * 4
32
33def get_notice_path(module_name):
34    return os.path.join('NOTICE_FILES', module_name+'.txt')
35
36def get_target_arch(json_rel_path):
37    return json_rel_path.split('/')[0]
38
39def get_arch(json_rel_path):
40    return json_rel_path.split('/')[1].split('-')[1]
41
42def get_variation(json_rel_path):
43    return json_rel_path.split('/')[2]
44
45# convert .bp prop dictionary to .bp prop string
46def gen_bp_prop(prop, ind):
47    bp = ''
48    for key in prop:
49        val = prop[key]
50
51        # Skip empty list or dict, rather than printing empty prop like
52        # "key: []," or "key: {},"
53        if type(val) == list or type(val) == dict:
54            if len(val) == 0:
55                continue
56
57        bp += ind + key + ": "
58        if type(val) == bool:
59            bp += "true,\n" if val else "false,\n"
60        elif type(val) == str:
61            bp += '"%s",\n' % val
62        elif type(val) == list:
63            bp += '[\n'
64            for elem in val:
65                bp += ind + INDENT + '"%s",\n' % elem
66            bp += ind + '],\n'
67        elif type(val) == dict:
68            bp += '{\n'
69            bp += gen_bp_prop(val, ind + INDENT)
70            bp += ind + '},\n'
71        else:
72            raise TypeError('unsupported type %s for gen_bp_prop' % type(val))
73    return bp
74
75# Remove non-existent dirs from given list. Emits warning for such dirs.
76def remove_invalid_dirs(paths, bp_dir, module_name):
77    ret = []
78    for path in paths:
79        if os.path.isdir(os.path.join(bp_dir, path)):
80            ret.append(path)
81        else:
82            logging.warning(
83                'Dir "%s" of module "%s" does not exist' % (path, module_name))
84    return ret
85
86JSON_TO_BP = {
87    'ModuleName':          'name',
88    'RelativeInstallPath': 'relative_install_path',
89    'ExportedDirs':        'export_include_dirs',
90    'ExportedSystemDirs':  'export_system_include_dirs',
91    'ExportedFlags':       'export_flags',
92    'SanitizeMinimalDep':  'sanitize_minimal_dep',
93    'SanitizeUbsanDep':    'sanitize_ubsan_dep',
94    'Symlinks':            'symlinks',
95    'InitRc':              'init_rc',
96    'VintfFragments':      'vintf_fragments',
97    'SharedLibs':          'shared_libs',
98    'RuntimeLibs':         'runtime_libs',
99    'Required':            'required',
100}
101
102# Converts parsed json dictionary (which is intermediate) to Android.bp prop
103# dictionary. This validates paths such as include directories and init_rc
104# files while converting.
105def convert_json_to_bp_prop(json_path, bp_dir):
106    prop = json.load(json_path)
107    ret = {}
108
109    module_name = prop['ModuleName']
110    ret['name'] = module_name
111
112    # Soong will complain about non-existing paths on Android.bp. There might
113    # be missing files among generated header files, so check all exported
114    # directories and filter out invalid ones. Emits warning for such dirs.
115    # TODO: fix soong to track all generated headers correctly
116    for key in {'ExportedDirs', 'ExportedSystemDirs'}:
117        if key in prop:
118            prop[key] = remove_invalid_dirs(prop[key], bp_dir, module_name)
119
120    for key in prop:
121        if key in JSON_TO_BP:
122            ret[JSON_TO_BP[key]] = prop[key]
123        else:
124            logging.warning(
125                'Unknown prop "%s" of module "%s"' % (key, module_name))
126
127    return ret
128
129def gen_bp_module(variation, name, version, target_arch, arch_props, bp_dir):
130    prop = {
131        # These three are common for all snapshot modules.
132        'version': str(version),
133        'target_arch': target_arch,
134        'vendor': True,
135        'arch': {},
136    }
137
138    # Factor out common prop among architectures to minimize Android.bp.
139    common_prop = None
140    for arch in arch_props:
141        if common_prop is None:
142            common_prop = dict()
143            for k in arch_props[arch]:
144                common_prop[k] = arch_props[arch][k]
145            continue
146        for k in list(common_prop.keys()):
147            if not k in arch_props[arch] or common_prop[k] != arch_props[arch][k]:
148                del common_prop[k]
149
150    # Forcing src to be arch_props prevents 32-bit only modules to be used as
151    # 64-bit modules, and vice versa.
152    if 'src' in common_prop:
153        del common_prop['src']
154    prop.update(common_prop)
155
156    stem32 = stem64 = ''
157
158    for arch in arch_props:
159        for k in common_prop:
160            if k in arch_props[arch]:
161                del arch_props[arch][k]
162        prop['arch'][arch] = arch_props[arch]
163        # Record stem for executable binary snapshots.
164        # We don't check existence of 'src'; src must exist for executables
165        if variation == 'binary':
166            if '64' in arch: # arm64, x86_64
167                stem64 = os.path.basename(arch_props[arch]['src'])
168            else:
169                stem32 = os.path.basename(arch_props[arch]['src'])
170
171    # For binary snapshots, compile_multilib must be assigned to 'both'
172    # in order to install both. Prefer 64bit if their stem collide and
173    # installing both is impossible
174    if variation == 'binary':
175        if stem32 and stem64:
176            if stem32 == stem64:
177                prop['compile_multilib'] = 'first'
178            else:
179                prop['compile_multilib'] = 'both'
180        elif stem32:
181            prop['compile_multilib'] = '32'
182        elif stem64:
183            prop['compile_multilib'] = '64'
184
185    bp = 'vendor_snapshot_%s {\n' % variation
186    bp += gen_bp_prop(prop, INDENT)
187    bp += '}\n\n'
188    return bp
189
190def get_args():
191    parser = argparse.ArgumentParser()
192    parser.add_argument(
193        'snapshot_version',
194        type=int,
195        help='Vendor snapshot version to install, e.g. "30".')
196    parser.add_argument(
197        '-v',
198        '--verbose',
199        action='count',
200        default=0,
201        help='Increase output verbosity, e.g. "-v", "-vv".')
202    return parser.parse_args()
203
204def main():
205    """Program entry point."""
206    args = get_args()
207    verbose_map = (logging.WARNING, logging.INFO, logging.DEBUG)
208    verbosity = min(args.verbose, 2)
209    logging.basicConfig(
210        format='%(levelname)-8s [%(filename)s:%(lineno)d] %(message)s',
211        level=verbose_map[verbosity])
212    install_dir = os.path.join('prebuilts', 'vendor', 'v'+str(args.snapshot_version))
213
214    # props[target_arch]["static"|"shared"|"binary"|"header"][name][arch] : json
215    props = dict()
216
217    # {target_arch}/{arch}/{variation}/{module}.json
218    for root, _, files in os.walk(install_dir):
219        for file_name in sorted(files):
220            if not file_name.endswith('.json'):
221                continue
222            full_path = os.path.join(root, file_name)
223            rel_path = os.path.relpath(full_path, install_dir)
224
225            target_arch = get_target_arch(rel_path)
226            arch = get_arch(rel_path)
227            variation = get_variation(rel_path)
228            bp_dir = os.path.join(install_dir, target_arch)
229
230            if not target_arch in props:
231                props[target_arch] = dict()
232            if not variation in props[target_arch]:
233                props[target_arch][variation] = dict()
234
235            with open(full_path, 'r') as f:
236                prop = convert_json_to_bp_prop(f, bp_dir)
237                # Remove .json after parsing?
238                # os.unlink(full_path)
239
240            if variation != 'header':
241                prop['src'] = os.path.relpath(
242                    rel_path[:-5], # removing .json
243                    target_arch)
244
245            module_name = prop['name']
246            notice_path = 'NOTICE_FILES/' + module_name + ".txt"
247            if os.path.exists(os.path.join(bp_dir, notice_path)):
248                prop['notice'] = notice_path
249
250            variation_dict = props[target_arch][variation]
251            if not module_name in variation_dict:
252                variation_dict[module_name] = dict()
253            variation_dict[module_name][arch] = prop
254
255    for target_arch in props:
256        androidbp = ''
257        bp_dir = os.path.join(install_dir, target_arch)
258        for variation in props[target_arch]:
259            for name in props[target_arch][variation]:
260                androidbp += gen_bp_module(
261                    variation,
262                    name,
263                    args.snapshot_version,
264                    target_arch,
265                    props[target_arch][variation][name],
266                    bp_dir)
267        with open(os.path.join(bp_dir, 'Android.bp'), 'w') as f:
268            f.write(androidbp)
269
270if __name__ == '__main__':
271    main()
272