1#!/usr/bin/env python3
2
3import gzip
4import os
5import shutil
6import subprocess
7import sys
8import tempfile
9import collections
10
11
12SCRIPT_DIR = os.path.abspath(os.path.dirname(__file__))
13
14try:
15    AOSP_DIR = os.environ['ANDROID_BUILD_TOP']
16except KeyError:
17    print('error: ANDROID_BUILD_TOP environment variable is not set.',
18          file=sys.stderr)
19    sys.exit(1)
20
21BUILTIN_HEADERS_DIR = (
22    os.path.join(AOSP_DIR, 'bionic', 'libc', 'include'),
23    os.path.join(AOSP_DIR, 'external', 'libcxx', 'include'),
24    os.path.join(AOSP_DIR, 'prebuilts', 'clang-tools', 'linux-x86',
25                 'clang-headers'),
26)
27
28SO_EXT = '.so'
29SOURCE_ABI_DUMP_EXT_END = '.lsdump'
30SOURCE_ABI_DUMP_EXT = SO_EXT + SOURCE_ABI_DUMP_EXT_END
31COMPRESSED_SOURCE_ABI_DUMP_EXT = SOURCE_ABI_DUMP_EXT + '.gz'
32VENDOR_SUFFIX = '.vendor'
33
34DEFAULT_CPPFLAGS = ['-x', 'c++', '-std=c++11']
35DEFAULT_CFLAGS = ['-std=gnu99']
36DEFAULT_HEADER_FLAGS = ["-dump-function-declarations"]
37DEFAULT_FORMAT = 'ProtobufTextFormat'
38
39
40class Target(object):
41    def __init__(self, is_2nd, product):
42        extra = '_2ND' if is_2nd else ''
43        build_vars_to_fetch = ['TARGET_ARCH',
44                               'TARGET{}_ARCH'.format(extra),
45                               'TARGET{}_ARCH_VARIANT'.format(extra),
46                               'TARGET{}_CPU_VARIANT'.format(extra)]
47        build_vars = get_build_vars_for_product(build_vars_to_fetch, product)
48        self.primary_arch = build_vars[0]
49        assert self.primary_arch != ''
50        self.arch = build_vars[1]
51        self.arch_variant = build_vars[2]
52        self.cpu_variant = build_vars[3]
53
54    def get_arch_str(self):
55        """Return a string that represents the architecture and the
56        architecture variant.
57
58        If TARGET_ARCH == TARGET_ARCH_VARIANT, soong makes targetArchVariant
59        empty. This is the case for aosp_x86_64.
60        """
61        if not self.arch_variant or self.arch_variant == self.arch:
62            arch_variant = ''
63        else:
64            arch_variant = '_' + self.arch_variant
65
66        return self.arch + arch_variant
67
68    def get_arch_cpu_str(self):
69        """Return a string that represents the architecture, the architecture
70        variant, and the CPU variant."""
71        if not self.cpu_variant or self.cpu_variant == 'generic':
72            cpu_variant = ''
73        else:
74            cpu_variant = '_' + self.cpu_variant
75
76        return self.get_arch_str() + cpu_variant
77
78
79def _validate_dump_content(dump_path):
80    """Make sure that the dump contains relative source paths."""
81    with open(dump_path, 'r') as f:
82        if AOSP_DIR in f.read():
83            raise ValueError(
84                dump_path + ' contains absolute path to $ANDROID_BUILD_TOP.')
85
86
87def copy_reference_dump(lib_path, reference_dump_dir, compress):
88    reference_dump_path = os.path.join(
89        reference_dump_dir, os.path.basename(lib_path))
90    if compress:
91        reference_dump_path += '.gz'
92    os.makedirs(os.path.dirname(reference_dump_path), exist_ok=True)
93    _validate_dump_content(lib_path)
94    if compress:
95        with open(lib_path, 'rb') as src_file:
96            with gzip.open(reference_dump_path, 'wb') as dst_file:
97                shutil.copyfileobj(src_file, dst_file)
98    else:
99        shutil.copyfile(lib_path, reference_dump_path)
100    print('Created abi dump at', reference_dump_path)
101    return reference_dump_path
102
103
104def run_header_abi_dumper(input_path, output_path, cflags=tuple(),
105                          export_include_dirs=tuple(), flags=tuple()):
106    """Run header-abi-dumper to dump ABI from `input_path` and the output is
107    written to `output_path`."""
108    input_ext = os.path.splitext(input_path)[1]
109    cmd = ['header-abi-dumper', '-o', output_path, input_path]
110    for dir in export_include_dirs:
111        cmd += ['-I', dir]
112    cmd += flags
113    if '-output-format' not in flags:
114        cmd += ['-output-format', DEFAULT_FORMAT]
115    if input_ext == ".h":
116        cmd += DEFAULT_HEADER_FLAGS
117    cmd += ['--']
118    cmd += cflags
119    if input_ext in ('.cpp', '.cc', '.h'):
120        cmd += DEFAULT_CPPFLAGS
121    else:
122        cmd += DEFAULT_CFLAGS
123
124    for dir in BUILTIN_HEADERS_DIR:
125        cmd += ['-isystem', dir]
126    # The export include dirs imply local include dirs.
127    for dir in export_include_dirs:
128        cmd += ['-I', dir]
129    subprocess.check_call(cmd, cwd=AOSP_DIR)
130    _validate_dump_content(output_path)
131
132
133def run_header_abi_linker(inputs, output_path, version_script, api, arch,
134                          flags=tuple()):
135    """Link inputs, taking version_script into account"""
136    cmd = ['header-abi-linker', '-o', output_path, '-v', version_script,
137           '-api', api, '-arch', arch]
138    cmd += flags
139    if '-input-format' not in flags:
140        cmd += ['-input-format', DEFAULT_FORMAT]
141    if '-output-format' not in flags:
142        cmd += ['-output-format', DEFAULT_FORMAT]
143    cmd += inputs
144    subprocess.check_call(cmd, cwd=AOSP_DIR)
145    _validate_dump_content(output_path)
146
147
148def make_targets(product, variant, targets):
149    make_cmd = ['build/soong/soong_ui.bash', '--make-mode', '-j',
150                'TARGET_PRODUCT=' + product, 'TARGET_BUILD_VARIANT=' + variant]
151    make_cmd += targets
152    subprocess.check_call(make_cmd, cwd=AOSP_DIR)
153
154
155def make_tree(product, variant):
156    """Build all lsdump files."""
157    return make_targets(product, variant, ['findlsdumps'])
158
159
160def make_libraries(product, variant, vndk_version, targets, libs):
161    """Build lsdump files for specific libs."""
162    lsdump_paths = read_lsdump_paths(product, variant, vndk_version, targets,
163                                     build=True)
164    make_target_paths = []
165    for name in libs:
166        if not (name in lsdump_paths and lsdump_paths[name]):
167            raise KeyError('Cannot find lsdump for %s.' % name)
168        for tag_path_dict in lsdump_paths[name].values():
169            make_target_paths.extend(tag_path_dict.values())
170    make_targets(product, variant, make_target_paths)
171
172
173def get_lsdump_paths_file_path(product, variant):
174    """Get the path to lsdump_paths.txt."""
175    product_out = get_build_vars_for_product(
176        ['PRODUCT_OUT'], product, variant)[0]
177    return os.path.join(product_out, 'lsdump_paths.txt')
178
179
180def _is_sanitizer_variation(variation):
181    """Check whether the variation is introduced by a sanitizer."""
182    return variation in {'asan', 'hwasan', 'tsan', 'intOverflow', 'cfi', 'scs'}
183
184
185def _get_module_variant_dir_name(tag, vndk_version, arch_cpu_str):
186    """Return the module variant directory name.
187
188    For example, android_x86_shared, android_vendor.R_arm_armv7-a-neon_shared.
189    """
190    if tag in ('LLNDK', 'NDK', 'PLATFORM'):
191        return 'android_%s_shared' % arch_cpu_str
192    if tag.startswith('VNDK'):
193        return 'android_vendor.%s_%s_shared' % (vndk_version, arch_cpu_str)
194    raise ValueError(tag + ' is not a known tag.')
195
196
197def _read_lsdump_paths(lsdump_paths_file_path, vndk_version, targets):
198    """Read lsdump paths from lsdump_paths.txt for each libname and variant.
199
200    This function returns a dictionary, {lib_name: {arch_cpu: {tag: path}}}.
201    For example,
202    {
203      "libc": {
204        "x86_x86_64": {
205          "NDK": "path/to/libc.so.lsdump"
206        }
207      }
208    }
209    """
210    lsdump_paths = collections.defaultdict(
211        lambda: collections.defaultdict(dict))
212    suffixes = collections.defaultdict(dict)
213
214    with open(lsdump_paths_file_path, 'r') as lsdump_paths_file:
215        for line in lsdump_paths_file:
216            tag, path = (x.strip() for x in line.split(':', 1))
217            if not path:
218                continue
219            dirname, filename = os.path.split(path)
220            if not filename.endswith(SOURCE_ABI_DUMP_EXT):
221                continue
222            libname = filename[:-len(SOURCE_ABI_DUMP_EXT)]
223            if not libname:
224                continue
225            variant = os.path.basename(dirname)
226            if not variant:
227                continue
228            for target in targets:
229                arch_cpu = target.get_arch_cpu_str()
230                prefix = _get_module_variant_dir_name(tag, vndk_version,
231                                                      arch_cpu)
232                if not variant.startswith(prefix):
233                    continue
234                new_suffix = variant[len(prefix):]
235                # Skip if the suffix contains APEX variations.
236                new_variations = [x for x in new_suffix.split('_') if x]
237                if new_variations and not all(_is_sanitizer_variation(x)
238                                              for x in new_variations):
239                    continue
240                old_suffix = suffixes[libname].get(arch_cpu)
241                if not old_suffix or new_suffix > old_suffix:
242                    lsdump_paths[libname][arch_cpu][tag] = path
243                    suffixes[libname][arch_cpu] = new_suffix
244    return lsdump_paths
245
246
247def read_lsdump_paths(product, variant, vndk_version, targets, build=True):
248    """Build lsdump_paths.txt and read the paths."""
249    lsdump_paths_file_path = get_lsdump_paths_file_path(product, variant)
250    if build:
251        make_targets(product, variant, [lsdump_paths_file_path])
252    lsdump_paths_file_abspath = os.path.join(AOSP_DIR, lsdump_paths_file_path)
253    return _read_lsdump_paths(lsdump_paths_file_abspath, vndk_version,
254                              targets)
255
256
257def find_lib_lsdumps(lsdump_paths, libs, target):
258    """Find the lsdump corresponding to libs for the given target.
259
260    This function returns a list of (tag, absolute_path).
261    For example,
262    [
263      (
264        "NDK",
265        "/path/to/libc.so.lsdump"
266      )
267    ]
268    """
269    arch_cpu = target.get_arch_cpu_str()
270    result = []
271    if libs:
272        for lib_name in libs:
273            if not (lib_name in lsdump_paths and
274                    arch_cpu in lsdump_paths[lib_name]):
275                raise KeyError('Cannot find lsdump for %s, %s.' %
276                               (lib_name, arch_cpu))
277            result.extend(lsdump_paths[lib_name][arch_cpu].items())
278    else:
279        for arch_tag_path_dict in lsdump_paths.values():
280            result.extend(arch_tag_path_dict[arch_cpu].items())
281    return [(tag, os.path.join(AOSP_DIR, path)) for tag, path in result]
282
283
284def run_abi_diff(old_test_dump_path, new_test_dump_path, arch, lib_name,
285                 flags=tuple()):
286    abi_diff_cmd = ['header-abi-diff', '-new', new_test_dump_path, '-old',
287                    old_test_dump_path, '-arch', arch, '-lib', lib_name]
288    with tempfile.TemporaryDirectory() as tmp:
289        output_name = os.path.join(tmp, lib_name) + '.abidiff'
290        abi_diff_cmd += ['-o', output_name]
291        abi_diff_cmd += flags
292        if '-input-format-old' not in flags:
293            abi_diff_cmd += ['-input-format-old', DEFAULT_FORMAT]
294        if '-input-format-new' not in flags:
295            abi_diff_cmd += ['-input-format-new', DEFAULT_FORMAT]
296        try:
297            subprocess.check_call(abi_diff_cmd)
298        except subprocess.CalledProcessError as err:
299            return err.returncode
300
301    return 0
302
303
304def get_build_vars_for_product(names, product=None, variant=None):
305    """ Get build system variable for the launched target."""
306
307    if product is None and 'ANDROID_PRODUCT_OUT' not in os.environ:
308        return None
309
310    env = os.environ.copy()
311    if product:
312        env['TARGET_PRODUCT'] = product
313    if variant:
314        env['TARGET_BUILD_VARIANT'] = variant
315    cmd = [
316        os.path.join('build', 'soong', 'soong_ui.bash'),
317        '--dumpvars-mode', '-vars', ' '.join(names),
318    ]
319
320    proc = subprocess.Popen(cmd, stdout=subprocess.PIPE,
321                            stderr=subprocess.PIPE, cwd=AOSP_DIR, env=env)
322    out, err = proc.communicate()
323
324    if proc.returncode != 0:
325        print("error: %s" % err.decode('utf-8'), file=sys.stderr)
326        return None
327
328    build_vars = out.decode('utf-8').strip().splitlines()
329
330    build_vars_list = []
331    for build_var in build_vars:
332        value = build_var.partition('=')[2]
333        build_vars_list.append(value.replace('\'', ''))
334    return build_vars_list
335