1#!/usr/bin/env python3
2#
3# Copyright 2018, The Android Open Source Project
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9#     http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16
17#
18#
19# Measure application start-up time by launching applications under various combinations.
20# See --help for more details.
21#
22#
23# Sample usage:
24# $> ./app_startup_runner.py -p com.google.android.calculator -r warm -r cold -lc 10  -o out.csv
25# $> ./analyze_metrics.py out.csv
26#
27#
28
29import argparse
30import csv
31import itertools
32import os
33import subprocess
34import sys
35import tempfile
36from typing import Any, Callable, Dict, Generic, Iterable, List, NamedTuple, TextIO, Tuple, TypeVar, Optional, Union
37
38# The following command line options participate in the combinatorial generation.
39# All other arguments have a global effect.
40_COMBINATORIAL_OPTIONS=['packages', 'readaheads', 'compiler_filters']
41_TRACING_READAHEADS=['mlock', 'fadvise']
42_FORWARD_OPTIONS={'loop_count': '--count'}
43_RUN_SCRIPT=os.path.join(os.path.dirname(os.path.realpath(__file__)), 'run_app_with_prefetch')
44
45RunCommandArgs = NamedTuple('RunCommandArgs', [('package', str), ('readahead', str), ('compiler_filter', Optional[str])])
46CollectorPackageInfo = NamedTuple('CollectorPackageInfo', [('package', str), ('compiler_filter', str)])
47_COLLECTOR_SCRIPT=os.path.join(os.path.dirname(os.path.realpath(__file__)), 'collector')
48_COLLECTOR_TIMEOUT_MULTIPLIER = 2 # take the regular --timeout and multiply by 2; systrace starts up slowly.
49
50_UNLOCK_SCREEN_SCRIPT=os.path.join(os.path.dirname(os.path.realpath(__file__)), 'unlock_screen')
51
52# This must be the only mutable global variable. All other global variables are constants to avoid magic literals.
53_debug = False  # See -d/--debug flag.
54_DEBUG_FORCE = None  # Ignore -d/--debug if this is not none.
55
56# Type hinting names.
57T = TypeVar('T')
58NamedTupleMeta = Callable[..., T]  # approximation of a (S : NamedTuple<T> where S() == T) metatype.
59
60def parse_options(argv: List[str] = None):
61  """Parse command line arguments and return an argparse Namespace object."""
62  parser = argparse.ArgumentParser(description="Run one or more Android applications under various settings in order to measure startup time.")
63  # argparse considers args starting with - and -- optional in --help, even though required=True.
64  # by using a named argument group --help will clearly say that it's required instead of optional.
65  required_named = parser.add_argument_group('required named arguments')
66  required_named.add_argument('-p', '--package', action='append', dest='packages', help='package of the application', required=True)
67  required_named.add_argument('-r', '--readahead', action='append', dest='readaheads', help='which readahead mode to use', choices=('warm', 'cold', 'mlock', 'fadvise'), required=True)
68
69  # optional arguments
70  # use a group here to get the required arguments to appear 'above' the optional arguments in help.
71  optional_named = parser.add_argument_group('optional named arguments')
72  optional_named.add_argument('-c', '--compiler-filter', action='append', dest='compiler_filters', help='which compiler filter to use. if omitted it does not enforce the app\'s compiler filter', choices=('speed', 'speed-profile', 'quicken'))
73  optional_named.add_argument('-s', '--simulate', dest='simulate', action='store_true', help='Print which commands will run, but don\'t run the apps')
74  optional_named.add_argument('-d', '--debug', dest='debug', action='store_true', help='Add extra debugging output')
75  optional_named.add_argument('-o', '--output', dest='output', action='store', help='Write CSV output to file.')
76  optional_named.add_argument('-t', '--timeout', dest='timeout', action='store', type=int, help='Timeout after this many seconds when executing a single run.')
77  optional_named.add_argument('-lc', '--loop-count', dest='loop_count', default=1, type=int, action='store', help='How many times to loop a single run.')
78  optional_named.add_argument('-in', '--inodes', dest='inodes', type=str, action='store', help='Path to inodes file (system/extras/pagecache/pagecache.py -d inodes)')
79
80  return parser.parse_args(argv)
81
82# TODO: refactor this with a common library file with analyze_metrics.py
83def _debug_print(*args, **kwargs):
84  """Print the args to sys.stderr if the --debug/-d flag was passed in."""
85  if _debug:
86    print(*args, **kwargs, file=sys.stderr)
87
88def _expand_gen_repr(args):
89  """Like repr but any generator-like object has its iterator consumed
90  and then called repr on."""
91  new_args_list = []
92  for i in args:
93    # detect iterable objects that do not have their own override of __str__
94    if hasattr(i, '__iter__'):
95      to_str = getattr(i, '__str__')
96      if to_str.__objclass__ == object:
97        # the repr for a generator is just type+address, expand it out instead.
98        new_args_list.append([_expand_gen_repr([j])[0] for j in i])
99        continue
100    # normal case: uses the built-in to-string
101    new_args_list.append(i)
102  return new_args_list
103
104def _debug_print_gen(*args, **kwargs):
105  """Like _debug_print but will turn any iterable args into a list."""
106  if not _debug:
107    return
108
109  new_args_list = _expand_gen_repr(args)
110  _debug_print(*new_args_list, **kwargs)
111
112def _debug_print_nd(*args, **kwargs):
113  """Like _debug_print but will turn any NamedTuple-type args into a string."""
114  if not _debug:
115    return
116
117  new_args_list = []
118  for i in args:
119    if hasattr(i, '_field_types'):
120      new_args_list.append("%s: %s" %(i.__name__, i._field_types))
121    else:
122      new_args_list.append(i)
123
124  _debug_print(*new_args_list, **kwargs)
125
126def dict_lookup_any_key(dictionary: dict, *keys: List[Any]):
127  for k in keys:
128    if k in dictionary:
129      return dictionary[k]
130  raise KeyError("None of the keys %s were in the dictionary" %(keys))
131
132def generate_run_combinations(named_tuple: NamedTupleMeta[T], opts_dict: Dict[str, List[Optional[str]]])\
133    -> Iterable[T]:
134  """
135  Create all possible combinations given the values in opts_dict[named_tuple._fields].
136
137  :type T: type annotation for the named_tuple type.
138  :param named_tuple: named tuple type, whose fields are used to make combinations for
139  :param opts_dict: dictionary of keys to value list. keys correspond to the named_tuple fields.
140  :return: an iterable over named_tuple instances.
141  """
142  combinations_list = []
143  for k in named_tuple._fields:
144    # the key can be either singular or plural , e.g. 'package' or 'packages'
145    val = dict_lookup_any_key(opts_dict, k, k + "s")
146
147    # treat {'x': None} key value pairs as if it was [None]
148    # otherwise itertools.product throws an exception about not being able to iterate None.
149    combinations_list.append(val or [None])
150
151  _debug_print("opts_dict: ", opts_dict)
152  _debug_print_nd("named_tuple: ", named_tuple)
153  _debug_print("combinations_list: ", combinations_list)
154
155  for combo in itertools.product(*combinations_list):
156    yield named_tuple(*combo)
157
158def key_to_cmdline_flag(key: str) -> str:
159  """Convert key into a command line flag, e.g. 'foo-bars' -> '--foo-bar' """
160  if key.endswith("s"):
161    key = key[:-1]
162  return "--" + key.replace("_", "-")
163
164def as_run_command(tpl: NamedTuple) -> List[Union[str, Any]]:
165  """
166  Convert a named tuple into a command-line compatible arguments list.
167
168  Example: ABC(1, 2, 3) -> ['--a', 1, '--b', 2, '--c', 3]
169  """
170  args = []
171  for key, value in tpl._asdict().items():
172    if value is None:
173      continue
174    args.append(key_to_cmdline_flag(key))
175    args.append(value)
176  return args
177
178def generate_group_run_combinations(run_combinations: Iterable[NamedTuple], dst_nt: NamedTupleMeta[T])\
179    -> Iterable[Tuple[T, Iterable[NamedTuple]]]:
180
181  def group_by_keys(src_nt):
182    src_d = src_nt._asdict()
183    # now remove the keys that aren't legal in dst.
184    for illegal_key in set(src_d.keys()) - set(dst_nt._fields):
185      if illegal_key in src_d:
186        del src_d[illegal_key]
187
188    return dst_nt(**src_d)
189
190  for args_list_it in itertools.groupby(run_combinations, group_by_keys):
191    (group_key_value, args_it) = args_list_it
192    yield (group_key_value, args_it)
193
194def parse_run_script_csv_file(csv_file: TextIO) -> List[int]:
195  """Parse a CSV file full of integers into a flat int list."""
196  csv_reader = csv.reader(csv_file)
197  arr = []
198  for row in csv_reader:
199    for i in row:
200      if i:
201        arr.append(int(i))
202  return arr
203
204def make_script_command_with_temp_output(script: str, args: List[str], **kwargs)\
205    -> Tuple[str, TextIO]:
206  """
207  Create a command to run a script given the args.
208  Appends --count <loop_count> --output <tmp-file-name>.
209  Returns a tuple (cmd, tmp_file)
210  """
211  tmp_output_file = tempfile.NamedTemporaryFile(mode='r')
212  cmd = [script] + args
213  for key, value in kwargs.items():
214    cmd += ['--%s' %(key), "%s" %(value)]
215  if _debug:
216    cmd += ['--verbose']
217  cmd = cmd + ["--output", tmp_output_file.name]
218  return cmd, tmp_output_file
219
220def execute_arbitrary_command(cmd: List[str], simulate: bool, timeout: int) -> Tuple[bool, str]:
221  if simulate:
222    print(" ".join(cmd))
223    return (True, "")
224  else:
225    _debug_print("[EXECUTE]", cmd)
226    proc = subprocess.Popen(cmd,
227                            stderr=subprocess.STDOUT,
228                            stdout=subprocess.PIPE,
229                            universal_newlines=True)
230    try:
231      script_output = proc.communicate(timeout=timeout)[0]
232    except subprocess.TimeoutExpired:
233      print("[TIMEDOUT]")
234      proc.kill()
235      script_output = proc.communicate()[0]
236
237    _debug_print("[STDOUT]", script_output)
238    return_code = proc.wait()
239    passed = (return_code == 0)
240    _debug_print("[$?]", return_code)
241    if not passed:
242      print("[FAILED, code:%s]" %(return_code), script_output, file=sys.stderr)
243
244    return (passed, script_output)
245
246def execute_run_combos(grouped_run_combos: Iterable[Tuple[CollectorPackageInfo, Iterable[RunCommandArgs]]], simulate: bool, inodes_path: str, timeout: int, loop_count: int, need_trace: bool):
247  # nothing will work if the screen isn't unlocked first.
248  execute_arbitrary_command([_UNLOCK_SCREEN_SCRIPT], simulate, timeout)
249
250  for collector_info, run_combos in grouped_run_combos:
251    #collector_args = ["--package", package_name]
252    collector_args = as_run_command(collector_info)
253    # TODO: forward --wait_time for how long systrace runs?
254    # TODO: forward --trace_buffer_size for size of systrace buffer size?
255    collector_cmd, collector_tmp_output_file = make_script_command_with_temp_output(_COLLECTOR_SCRIPT, collector_args, inodes=inodes_path)
256
257    with collector_tmp_output_file:
258      collector_passed = True
259      if need_trace:
260        collector_timeout = timeout and _COLLECTOR_TIMEOUT_MULTIPLIER * timeout
261        (collector_passed, collector_script_output) = execute_arbitrary_command(collector_cmd, simulate, collector_timeout)
262        # TODO: consider to print a ; collector wrote file to <...> into the CSV file so we know it was ran.
263
264      for combos in run_combos:
265        args = as_run_command(combos)
266
267        cmd, tmp_output_file = make_script_command_with_temp_output(_RUN_SCRIPT, args, count=loop_count, input=collector_tmp_output_file.name)
268        with tmp_output_file:
269          (passed, script_output) = execute_arbitrary_command(cmd, simulate, timeout)
270          parsed_output = simulate and [1,2,3] or parse_run_script_csv_file(tmp_output_file)
271          yield (passed, script_output, parsed_output)
272
273def gather_results(commands: Iterable[Tuple[bool, str, List[int]]], key_list: List[str], value_list: List[Tuple[str, ...]]):
274  _debug_print("gather_results: key_list = ", key_list)
275  yield key_list + ["time(ms)"]
276
277  stringify_none = lambda s: s is None and "<none>" or s
278
279  for ((passed, script_output, run_result_list), values) in itertools.zip_longest(commands, value_list):
280    if not passed:
281      continue
282    for result in run_result_list:
283      yield [stringify_none(i) for i in values] + [result]
284
285    yield ["; avg(%s), min(%s), max(%s), count(%s)" %(sum(run_result_list, 0.0) / len(run_result_list), min(run_result_list), max(run_result_list), len(run_result_list)) ]
286
287def eval_and_save_to_csv(output, annotated_result_values):
288  csv_writer = csv.writer(output)
289  for row in annotated_result_values:
290    csv_writer.writerow(row)
291    output.flush() # see the output live.
292
293def main():
294  global _debug
295
296  opts = parse_options()
297  _debug = opts.debug
298  if _DEBUG_FORCE is not None:
299    _debug = _DEBUG_FORCE
300  _debug_print("parsed options: ", opts)
301  need_trace = not not set(opts.readaheads).intersection(set(_TRACING_READAHEADS))
302  if need_trace and not opts.inodes:
303    print("Error: Missing -in/--inodes, required when using a readahead of %s" %(_TRACING_READAHEADS), file=sys.stderr)
304    return 1
305
306  output_file = opts.output and open(opts.output, 'w') or sys.stdout
307
308  combos = lambda: generate_run_combinations(RunCommandArgs, vars(opts))
309  _debug_print_gen("run combinations: ", combos())
310
311  grouped_combos = lambda: generate_group_run_combinations(combos(), CollectorPackageInfo)
312  _debug_print_gen("grouped run combinations: ", grouped_combos())
313
314  exec = execute_run_combos(grouped_combos(), opts.simulate, opts.inodes, opts.timeout, opts.loop_count, need_trace)
315  results = gather_results(exec, _COMBINATORIAL_OPTIONS, combos())
316  eval_and_save_to_csv(output_file, results)
317
318  return 0
319
320
321if __name__ == '__main__':
322  sys.exit(main())
323