1#!/usr/bin/env python
2#
3# Copyright (C) 2016 The Android Open Source Project
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9#      http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16#
17
18"""app_profiler.py: Record cpu profiling data of an android app or native program.
19
20    It downloads simpleperf on device, uses it to collect profiling data on the selected app,
21    and pulls profiling data and related binaries on host.
22"""
23
24from __future__ import print_function
25import argparse
26import os
27import os.path
28import subprocess
29import sys
30import time
31
32from utils import AdbHelper, bytes_to_str, extant_dir, get_script_dir, get_target_binary_path
33from utils import log_debug, log_info, log_exit, ReadElf, remove, set_log_level, str_to_bytes
34
35NATIVE_LIBS_DIR_ON_DEVICE = '/data/local/tmp/native_libs/'
36
37class HostElfEntry(object):
38    """Represent a native lib on host in NativeLibDownloader."""
39    def __init__(self, path, name, score):
40        self.path = path
41        self.name = name
42        self.score = score
43
44    def __repr__(self):
45        return self.__str__()
46
47    def __str__(self):
48        return '[path: %s, name %s, score %s]' % (self.path, self.name, self.score)
49
50
51class NativeLibDownloader(object):
52    """Download native libs on device.
53
54    1. Collect info of all native libs in the native_lib_dir on host.
55    2. Check the available native libs in /data/local/tmp/native_libs on device.
56    3. Sync native libs on device.
57    """
58    def __init__(self, ndk_path, device_arch, adb):
59        self.adb = adb
60        self.readelf = ReadElf(ndk_path)
61        self.device_arch = device_arch
62        self.need_archs = self._get_need_archs()
63        self.host_build_id_map = {}  # Map from build_id to HostElfEntry.
64        self.device_build_id_map = {}  # Map from build_id to relative_path on device.
65        # Map from filename to HostElfEntry for elf files without build id.
66        self.no_build_id_file_map = {}
67        self.name_count_map = {}  # Used to give a unique name for each library.
68        self.dir_on_device = NATIVE_LIBS_DIR_ON_DEVICE
69        self.build_id_list_file = 'build_id_list'
70
71    def _get_need_archs(self):
72        """Return the archs of binaries needed on device."""
73        if self.device_arch == 'arm64':
74            return ['arm', 'arm64']
75        if self.device_arch == 'arm':
76            return ['arm']
77        if self.device_arch == 'x86_64':
78            return ['x86', 'x86_64']
79        if self.device_arch == 'x86':
80            return ['x86']
81        return []
82
83    def collect_native_libs_on_host(self, native_lib_dir):
84        self.host_build_id_map.clear()
85        for root, _, files in os.walk(native_lib_dir):
86            for name in files:
87                if not name.endswith('.so'):
88                    continue
89                self.add_native_lib_on_host(os.path.join(root, name), name)
90
91    def add_native_lib_on_host(self, path, name):
92        arch = self.readelf.get_arch(path)
93        if arch not in self.need_archs:
94            return
95        sections = self.readelf.get_sections(path)
96        score = 0
97        if '.debug_info' in sections:
98            score = 3
99        elif '.gnu_debugdata' in sections:
100            score = 2
101        elif '.symtab' in sections:
102            score = 1
103        build_id = self.readelf.get_build_id(path)
104        if build_id:
105            entry = self.host_build_id_map.get(build_id)
106            if entry:
107                if entry.score < score:
108                    entry.path = path
109                    entry.score = score
110            else:
111                repeat_count = self.name_count_map.get(name, 0)
112                self.name_count_map[name] = repeat_count + 1
113                unique_name = name if repeat_count == 0 else name + '_' + str(repeat_count)
114                self.host_build_id_map[build_id] = HostElfEntry(path, unique_name, score)
115        else:
116            entry = self.no_build_id_file_map.get(name)
117            if entry:
118                if entry.score < score:
119                    entry.path = path
120                    entry.score = score
121            else:
122                self.no_build_id_file_map[name] = HostElfEntry(path, name, score)
123
124    def collect_native_libs_on_device(self):
125        self.device_build_id_map.clear()
126        self.adb.check_run(['shell', 'mkdir', '-p', self.dir_on_device])
127        if os.path.exists(self.build_id_list_file):
128            os.remove(self.build_id_list_file)
129        result, output = self.adb.run_and_return_output(['shell', 'ls', self.dir_on_device])
130        if not result:
131            return
132        file_set = set(output.strip().split())
133        if self.build_id_list_file not in file_set:
134            return
135        self.adb.run(['pull', self.dir_on_device + self.build_id_list_file])
136        if os.path.exists(self.build_id_list_file):
137            with open(self.build_id_list_file, 'rb') as fh:
138                for line in fh.readlines():
139                    line = bytes_to_str(line).strip()
140                    items = line.split('=')
141                    if len(items) == 2:
142                        build_id, filename = items
143                        if filename in file_set:
144                            self.device_build_id_map[build_id] = filename
145            remove(self.build_id_list_file)
146
147    def sync_native_libs_on_device(self):
148        # Push missing native libs on device.
149        for build_id in self.host_build_id_map:
150            if build_id not in self.device_build_id_map:
151                entry = self.host_build_id_map[build_id]
152                self.adb.check_run(['push', entry.path, self.dir_on_device + entry.name])
153        # Remove native libs not exist on host.
154        for build_id in self.device_build_id_map:
155            if build_id not in self.host_build_id_map:
156                name = self.device_build_id_map[build_id]
157                self.adb.run(['shell', 'rm', self.dir_on_device + name])
158        # Push new build_id_list on device.
159        with open(self.build_id_list_file, 'wb') as fh:
160            for build_id in self.host_build_id_map:
161                s = str_to_bytes('%s=%s\n' % (build_id, self.host_build_id_map[build_id].name))
162                fh.write(s)
163        self.adb.check_run(['push', self.build_id_list_file,
164                            self.dir_on_device + self.build_id_list_file])
165        os.remove(self.build_id_list_file)
166
167        # Push elf files without build id on device.
168        for entry in self.no_build_id_file_map.values():
169            target = self.dir_on_device + entry.name
170
171            # Skip download if we have a file with the same name and size on device.
172            result, output = self.adb.run_and_return_output(
173                ['shell', 'ls', '-l', target], log_output=False, log_stderr=False)
174            if result:
175                items = output.split()
176                if len(items) > 5:
177                    try:
178                        file_size = int(items[4])
179                    except ValueError:
180                        file_size = 0
181                    if file_size == os.path.getsize(entry.path):
182                        continue
183            self.adb.check_run(['push', entry.path, target])
184
185
186class ProfilerBase(object):
187    """Base class of all Profilers."""
188    def __init__(self, args):
189        self.args = args
190        self.adb = AdbHelper(enable_switch_to_root=not args.disable_adb_root)
191        self.is_root_device = self.adb.switch_to_root()
192        self.android_version = self.adb.get_android_version()
193        if self.android_version < 7:
194            log_exit("""app_profiler.py isn't supported on Android < N, please switch to use
195                        simpleperf binary directly.""")
196        self.device_arch = self.adb.get_device_arch()
197        self.record_subproc = None
198
199    def profile(self):
200        log_info('prepare profiling')
201        self.prepare()
202        log_info('start profiling')
203        self.start()
204        self.wait_profiling()
205        log_info('collect profiling data')
206        self.collect_profiling_data()
207        log_info('profiling is finished.')
208
209    def prepare(self):
210        """Prepare recording. """
211        self.download_simpleperf()
212        if self.args.native_lib_dir:
213            self.download_libs()
214
215    def download_simpleperf(self):
216        simpleperf_binary = get_target_binary_path(self.device_arch, 'simpleperf')
217        self.adb.check_run(['push', simpleperf_binary, '/data/local/tmp'])
218        self.adb.check_run(['shell', 'chmod', 'a+x', '/data/local/tmp/simpleperf'])
219
220    def download_libs(self):
221        downloader = NativeLibDownloader(self.args.ndk_path, self.device_arch, self.adb)
222        downloader.collect_native_libs_on_host(self.args.native_lib_dir)
223        downloader.collect_native_libs_on_device()
224        downloader.sync_native_libs_on_device()
225
226    def start(self):
227        raise NotImplementedError
228
229    def start_profiling(self, target_args):
230        """Start simpleperf reocrd process on device."""
231        args = ['/data/local/tmp/simpleperf', 'record', '-o', '/data/local/tmp/perf.data',
232                self.args.record_options]
233        if self.adb.run(['shell', 'ls', NATIVE_LIBS_DIR_ON_DEVICE, '>/dev/null', '2>&1']):
234            args += ['--symfs', NATIVE_LIBS_DIR_ON_DEVICE]
235        args += ['--log', self.args.log]
236        args += target_args
237        adb_args = [self.adb.adb_path, 'shell'] + args
238        log_info('run adb cmd: %s' % adb_args)
239        self.record_subproc = subprocess.Popen(adb_args)
240
241    def wait_profiling(self):
242        """Wait until profiling finishes, or stop profiling when user presses Ctrl-C."""
243        returncode = None
244        try:
245            returncode = self.record_subproc.wait()
246        except KeyboardInterrupt:
247            self.stop_profiling()
248            self.record_subproc = None
249            # Don't check return value of record_subproc. Because record_subproc also
250            # receives Ctrl-C, and always returns non-zero.
251            returncode = 0
252        log_debug('profiling result [%s]' % (returncode == 0))
253        if returncode != 0:
254            log_exit('Failed to record profiling data.')
255
256    def stop_profiling(self):
257        """Stop profiling by sending SIGINT to simpleperf, and wait until it exits
258           to make sure perf.data is completely generated."""
259        has_killed = False
260        while True:
261            (result, _) = self.adb.run_and_return_output(['shell', 'pidof', 'simpleperf'])
262            if not result:
263                break
264            if not has_killed:
265                has_killed = True
266                self.adb.run_and_return_output(['shell', 'pkill', '-l', '2', 'simpleperf'])
267            time.sleep(1)
268
269    def collect_profiling_data(self):
270        self.adb.check_run_and_return_output(['pull', '/data/local/tmp/perf.data',
271                                              self.args.perf_data_path])
272        if not self.args.skip_collect_binaries:
273            binary_cache_args = [sys.executable,
274                                 os.path.join(get_script_dir(), 'binary_cache_builder.py')]
275            binary_cache_args += ['-i', self.args.perf_data_path, '--log', self.args.log]
276            if self.args.native_lib_dir:
277                binary_cache_args += ['-lib', self.args.native_lib_dir]
278            if self.args.disable_adb_root:
279                binary_cache_args += ['--disable_adb_root']
280            if self.args.ndk_path:
281                binary_cache_args += ['--ndk_path', self.args.ndk_path]
282            subprocess.check_call(binary_cache_args)
283
284
285class AppProfiler(ProfilerBase):
286    """Profile an Android app."""
287    def prepare(self):
288        super(AppProfiler, self).prepare()
289        if self.args.compile_java_code:
290            self.compile_java_code()
291
292    def compile_java_code(self):
293        self.kill_app_process()
294        # Fully compile Java code on Android >= N.
295        self.adb.set_property('debug.generate-debug-info', 'true')
296        self.adb.check_run(['shell', 'cmd', 'package', 'compile', '-f', '-m', 'speed',
297                            self.args.app])
298
299    def kill_app_process(self):
300        if self.find_app_process():
301            self.adb.check_run(['shell', 'am', 'force-stop', self.args.app])
302            count = 0
303            while True:
304                time.sleep(1)
305                pid = self.find_app_process()
306                if not pid:
307                    break
308                # When testing on Android N, `am force-stop` sometimes can't kill
309                # com.example.simpleperf.simpleperfexampleofkotlin. So use kill when this happens.
310                count += 1
311                if count >= 3:
312                    self.run_in_app_dir(['kill', '-9', str(pid)])
313
314    def find_app_process(self):
315        result, output = self.adb.run_and_return_output(['shell', 'pidof', self.args.app])
316        return int(output) if result else None
317
318    def run_in_app_dir(self, args):
319        if self.is_root_device:
320            adb_args = ['shell', 'cd /data/data/' + self.args.app + ' && ' + (' '.join(args))]
321        else:
322            adb_args = ['shell', 'run-as', self.args.app] + args
323        return self.adb.run_and_return_output(adb_args, log_output=False)
324
325    def start(self):
326        if self.args.activity or self.args.test:
327            self.kill_app_process()
328        self.start_profiling(['--app', self.args.app])
329        if self.args.activity:
330            self.start_activity()
331        elif self.args.test:
332            self.start_test()
333        # else: no need to start an activity or test.
334
335    def start_activity(self):
336        activity = self.args.app + '/' + self.args.activity
337        result = self.adb.run(['shell', 'am', 'start', '-n', activity])
338        if not result:
339            self.record_subproc.terminate()
340            log_exit("Can't start activity %s" % activity)
341
342    def start_test(self):
343        runner = self.args.app + '/androidx.test.runner.AndroidJUnitRunner'
344        result = self.adb.run(['shell', 'am', 'instrument', '-e', 'class',
345                               self.args.test, runner])
346        if not result:
347            self.record_subproc.terminate()
348            log_exit("Can't start instrumentation test  %s" % self.args.test)
349
350
351class NativeProgramProfiler(ProfilerBase):
352    """Profile a native program."""
353    def start(self):
354        pid = int(self.adb.check_run_and_return_output(['shell', 'pidof',
355                                                        self.args.native_program]))
356        self.start_profiling(['-p', str(pid)])
357
358
359class NativeCommandProfiler(ProfilerBase):
360    """Profile running a native command."""
361    def start(self):
362        self.start_profiling([self.args.cmd])
363
364
365class NativeProcessProfiler(ProfilerBase):
366    """Profile processes given their pids."""
367    def start(self):
368        self.start_profiling(['-p', ','.join(self.args.pid)])
369
370
371class NativeThreadProfiler(ProfilerBase):
372    """Profile threads given their tids."""
373    def start(self):
374        self.start_profiling(['-t', ','.join(self.args.tid)])
375
376
377class SystemWideProfiler(ProfilerBase):
378    """Profile system wide."""
379    def start(self):
380        self.start_profiling(['-a'])
381
382
383def main():
384    parser = argparse.ArgumentParser(description=__doc__,
385                                     formatter_class=argparse.RawDescriptionHelpFormatter)
386
387    target_group = parser.add_argument_group(title='Select profiling target'
388                                            ).add_mutually_exclusive_group(required=True)
389    target_group.add_argument('-p', '--app', help="""Profile an Android app, given the package name.
390                              Like `-p com.example.android.myapp`.""")
391
392    target_group.add_argument('-np', '--native_program', help="""Profile a native program running on
393                              the Android device. Like `-np surfaceflinger`.""")
394
395    target_group.add_argument('-cmd', help="""Profile running a command on the Android device.
396                              Like `-cmd "pm -l"`.""")
397
398    target_group.add_argument('--pid', nargs='+', help="""Profile native processes running on device
399                              given their process ids.""")
400
401    target_group.add_argument('--tid', nargs='+', help="""Profile native threads running on device
402                              given their thread ids.""")
403
404    target_group.add_argument('--system_wide', action='store_true', help="""Profile system wide.""")
405
406    app_target_group = parser.add_argument_group(title='Extra options for profiling an app')
407    app_target_group.add_argument('--compile_java_code', action='store_true', help="""Used with -p.
408                                  On Android N and Android O, we need to compile Java code into
409                                  native instructions to profile Java code. Android O also needs
410                                  wrap.sh in the apk to use the native instructions.""")
411
412    app_start_group = app_target_group.add_mutually_exclusive_group()
413    app_start_group.add_argument('-a', '--activity', help="""Used with -p. Profile the launch time
414                                 of an activity in an Android app. The app will be started or
415                                 restarted to run the activity. Like `-a .MainActivity`.""")
416
417    app_start_group.add_argument('-t', '--test', help="""Used with -p. Profile the launch time of an
418                                 instrumentation test in an Android app. The app will be started or
419                                 restarted to run the instrumentation test. Like
420                                 `-t test_class_name`.""")
421
422    record_group = parser.add_argument_group('Select recording options')
423    record_group.add_argument('-r', '--record_options',
424                              default='-e task-clock:u -f 1000 -g --duration 10', help="""Set
425                              recording options for `simpleperf record` command. Use
426                              `run_simpleperf_on_device.py record -h` to see all accepted options.
427                              Default is "-e task-clock:u -f 1000 -g --duration 10".""")
428
429    record_group.add_argument('-lib', '--native_lib_dir', type=extant_dir,
430                              help="""When profiling an Android app containing native libraries,
431                                      the native libraries are usually stripped and lake of symbols
432                                      and debug information to provide good profiling result. By
433                                      using -lib, you tell app_profiler.py the path storing
434                                      unstripped native libraries, and app_profiler.py will search
435                                      all shared libraries with suffix .so in the directory. Then
436                                      the native libraries will be downloaded on device and
437                                      collected in build_cache.""")
438
439    record_group.add_argument('-o', '--perf_data_path', default='perf.data',
440                              help='The path to store profiling data. Default is perf.data.')
441
442    record_group.add_argument('-nb', '--skip_collect_binaries', action='store_true',
443                              help="""By default we collect binaries used in profiling data from
444                                      device to binary_cache directory. It can be used to annotate
445                                      source code and disassembly. This option skips it.""")
446
447    other_group = parser.add_argument_group('Other options')
448    other_group.add_argument('--ndk_path', type=extant_dir,
449                             help="""Set the path of a ndk release. app_profiler.py needs some
450                                     tools in ndk, like readelf.""")
451
452    other_group.add_argument('--disable_adb_root', action='store_true',
453                             help="""Force adb to run in non root mode. By default, app_profiler.py
454                                     will try to switch to root mode to be able to profile released
455                                     Android apps.""")
456    other_group.add_argument(
457        '--log', choices=['debug', 'info', 'warning'], default='info', help='set log level')
458
459    def check_args(args):
460        if (not args.app) and (args.compile_java_code or args.activity or args.test):
461            log_exit('--compile_java_code, -a, -t can only be used when profiling an Android app.')
462
463    args = parser.parse_args()
464    set_log_level(args.log)
465    check_args(args)
466    if args.app:
467        profiler = AppProfiler(args)
468    elif args.native_program:
469        profiler = NativeProgramProfiler(args)
470    elif args.cmd:
471        profiler = NativeCommandProfiler(args)
472    elif args.pid:
473        profiler = NativeProcessProfiler(args)
474    elif args.tid:
475        profiler = NativeThreadProfiler(args)
476    elif args.system_wide:
477        profiler = SystemWideProfiler(args)
478    profiler.profile()
479
480if __name__ == '__main__':
481    main()
482