1#!/usr/bin/env python
2#
3# Copyright (C) 2015 The Android Open Source Project
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9#      http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16#
17"""Simpleperf runtest runner: run simpleperf runtests on host or on device.
18
19For a simpleperf runtest like one_function test, it contains following steps:
201. Run simpleperf record command to record simpleperf_runtest_one_function's
21   running samples, which is generated in perf.data.
222. Run simpleperf report command to parse perf.data, generate perf.report.
234. Parse perf.report and see if it matches expectation.
24
25The information of all runtests is stored in runtest.conf.
26"""
27
28import os
29import os.path
30import re
31import subprocess
32import sys
33import xml.etree.ElementTree as ET
34
35
36class CallTreeNode(object):
37
38  def __init__(self, name):
39    self.name = name
40    self.children = []
41
42  def add_child(self, child):
43    self.children.append(child)
44
45  def __str__(self):
46    return 'CallTreeNode:\n' + '\n'.join(self._dump(1))
47
48  def _dump(self, indent):
49    indent_str = '  ' * indent
50    strs = [indent_str + self.name]
51    for child in self.children:
52      strs.extend(child._dump(indent + 1))
53    return strs
54
55
56class Symbol(object):
57
58  def __init__(self, name, comm, overhead, children_overhead):
59    self.name = name
60    self.comm = comm
61    self.overhead = overhead
62    # children_overhead is the overhead sum of this symbol and functions
63    # called by this symbol.
64    self.children_overhead = children_overhead
65    self.call_tree = None
66
67  def set_call_tree(self, call_tree):
68    self.call_tree = call_tree
69
70  def __str__(self):
71    strs = []
72    strs.append('Symbol name=%s comm=%s overhead=%f children_overhead=%f' % (
73        self.name, self.comm, self.overhead, self.children_overhead))
74    if self.call_tree:
75      strs.append('\t%s' % self.call_tree)
76    return '\n'.join(strs)
77
78
79class SymbolOverheadRequirement(object):
80
81  def __init__(self, symbol_name=None, comm=None, min_overhead=None,
82               max_overhead=None):
83    self.symbol_name = symbol_name
84    self.comm = comm
85    self.min_overhead = min_overhead
86    self.max_overhead = max_overhead
87
88  def __str__(self):
89    strs = []
90    strs.append('SymbolOverheadRequirement')
91    if self.symbol_name is not None:
92      strs.append('symbol_name=%s' % self.symbol_name)
93    if self.comm is not None:
94      strs.append('comm=%s' % self.comm)
95    if self.min_overhead is not None:
96      strs.append('min_overhead=%f' % self.min_overhead)
97    if self.max_overhead is not None:
98      strs.append('max_overhead=%f' % self.max_overhead)
99    return ' '.join(strs)
100
101  def is_match(self, symbol):
102    if self.symbol_name is not None:
103      if self.symbol_name != symbol.name:
104        return False
105    if self.comm is not None:
106      if self.comm != symbol.comm:
107        return False
108    return True
109
110  def check_overhead(self, overhead):
111    if self.min_overhead is not None:
112      if self.min_overhead > overhead:
113        return False
114    if self.max_overhead is not None:
115      if self.max_overhead < overhead:
116        return False
117    return True
118
119
120class SymbolRelationRequirement(object):
121
122  def __init__(self, symbol_name, comm=None):
123    self.symbol_name = symbol_name
124    self.comm = comm
125    self.children = []
126
127  def add_child(self, child):
128    self.children.append(child)
129
130  def __str__(self):
131    return 'SymbolRelationRequirement:\n' + '\n'.join(self._dump(1))
132
133  def _dump(self, indent):
134    indent_str = '  ' * indent
135    strs = [indent_str + self.symbol_name +
136            (' ' + self.comm if self.comm else '')]
137    for child in self.children:
138      strs.extend(child._dump(indent + 1))
139    return strs
140
141  def is_match(self, symbol):
142    if symbol.name != self.symbol_name:
143      return False
144    if self.comm is not None:
145      if symbol.comm != self.comm:
146        return False
147    return True
148
149  def check_relation(self, call_tree):
150    if not call_tree:
151      return False
152    if self.symbol_name != call_tree.name:
153      return False
154    for child in self.children:
155      child_matched = False
156      for node in call_tree.children:
157        if child.check_relation(node):
158          child_matched = True
159          break
160      if not child_matched:
161        return False
162    return True
163
164
165class Test(object):
166
167  def __init__(
168          self,
169          test_name,
170          executable_name,
171          disable_host,
172          record_options,
173          report_options,
174          symbol_overhead_requirements,
175          symbol_children_overhead_requirements,
176          symbol_relation_requirements):
177    self.test_name = test_name
178    self.executable_name = executable_name
179    self.disable_host = disable_host
180    self.record_options = record_options
181    self.report_options = report_options
182    self.symbol_overhead_requirements = symbol_overhead_requirements
183    self.symbol_children_overhead_requirements = (
184        symbol_children_overhead_requirements)
185    self.symbol_relation_requirements = symbol_relation_requirements
186
187  def __str__(self):
188    strs = []
189    strs.append('Test test_name=%s' % self.test_name)
190    strs.append('\texecutable_name=%s' % self.executable_name)
191    strs.append('\tdisable_host=%s' % self.disable_host)
192    strs.append('\trecord_options=%s' % (' '.join(self.record_options)))
193    strs.append('\treport_options=%s' % (' '.join(self.report_options)))
194    strs.append('\tsymbol_overhead_requirements:')
195    for req in self.symbol_overhead_requirements:
196      strs.append('\t\t%s' % req)
197    strs.append('\tsymbol_children_overhead_requirements:')
198    for req in self.symbol_children_overhead_requirements:
199      strs.append('\t\t%s' % req)
200    strs.append('\tsymbol_relation_requirements:')
201    for req in self.symbol_relation_requirements:
202      strs.append('\t\t%s' % req)
203    return '\n'.join(strs)
204
205
206def load_config_file(config_file):
207  tests = []
208  tree = ET.parse(config_file)
209  root = tree.getroot()
210  assert root.tag == 'runtests'
211  for test in root:
212    assert test.tag == 'test'
213    test_name = test.attrib['name']
214    executable_name = None
215    disable_host = False
216    record_options = []
217    report_options = []
218    symbol_overhead_requirements = []
219    symbol_children_overhead_requirements = []
220    symbol_relation_requirements = []
221    for test_item in test:
222      if test_item.tag == 'executable':
223        executable_name = test_item.attrib['name']
224      elif test_item.tag == 'disable_host':
225        disable_host = True
226      elif test_item.tag == 'record':
227        record_options = test_item.attrib['option'].split()
228      elif test_item.tag == 'report':
229        report_options = test_item.attrib['option'].split()
230      elif (test_item.tag == 'symbol_overhead' or
231              test_item.tag == 'symbol_children_overhead'):
232        for symbol_item in test_item:
233          assert symbol_item.tag == 'symbol'
234          symbol_name = None
235          if 'name' in symbol_item.attrib:
236            symbol_name = symbol_item.attrib['name']
237          comm = None
238          if 'comm' in symbol_item.attrib:
239            comm = symbol_item.attrib['comm']
240          overhead_min = None
241          if 'min' in symbol_item.attrib:
242            overhead_min = float(symbol_item.attrib['min'])
243          overhead_max = None
244          if 'max' in symbol_item.attrib:
245            overhead_max = float(symbol_item.attrib['max'])
246
247          if test_item.tag == 'symbol_overhead':
248            symbol_overhead_requirements.append(
249                SymbolOverheadRequirement(
250                    symbol_name,
251                    comm,
252                    overhead_min,
253                    overhead_max)
254            )
255          else:
256            symbol_children_overhead_requirements.append(
257                SymbolOverheadRequirement(
258                    symbol_name,
259                    comm,
260                    overhead_min,
261                    overhead_max))
262      elif test_item.tag == 'symbol_callgraph_relation':
263        for symbol_item in test_item:
264          req = load_symbol_relation_requirement(symbol_item)
265          symbol_relation_requirements.append(req)
266
267    tests.append(
268        Test(
269            test_name,
270            executable_name,
271            disable_host,
272            record_options,
273            report_options,
274            symbol_overhead_requirements,
275            symbol_children_overhead_requirements,
276            symbol_relation_requirements))
277  return tests
278
279
280def load_symbol_relation_requirement(symbol_item):
281  symbol_name = symbol_item.attrib['name']
282  comm = None
283  if 'comm' in symbol_item.attrib:
284    comm = symbol_item.attrib['comm']
285  req = SymbolRelationRequirement(symbol_name, comm)
286  for item in symbol_item:
287    child_req = load_symbol_relation_requirement(item)
288    req.add_child(child_req)
289  return req
290
291
292class Runner(object):
293
294  def __init__(self, target, perf_path):
295    self.target = target
296    self.is32 = target.endswith('32')
297    self.perf_path = perf_path
298    self.use_callgraph = False
299    self.sampler = 'cpu-cycles'
300
301  def record(self, test_executable_name, record_file, additional_options=[]):
302    call_args = [self.perf_path, 'record']
303    call_args += ['--duration', '2']
304    call_args += ['-e', '%s:u' % self.sampler]
305    if self.use_callgraph:
306      call_args += ['-f', '1000', '-g']
307    call_args += ['-o', record_file]
308    call_args += additional_options
309    test_executable_name += '32' if self.is32 else '64'
310    call_args += [test_executable_name]
311    self._call(call_args)
312
313  def report(self, record_file, report_file, additional_options=[]):
314    call_args = [self.perf_path, 'report']
315    call_args += ['-i', record_file]
316    if self.use_callgraph:
317      call_args += ['-g', 'callee']
318    call_args += additional_options
319    self._call(call_args, report_file)
320
321  def _call(self, args, output_file=None):
322    pass
323
324
325class HostRunner(Runner):
326
327  """Run perf test on host."""
328
329  def __init__(self, target):
330    perf_path = 'simpleperf32' if target.endswith('32') else 'simpleperf'
331    super(HostRunner, self).__init__(target, perf_path)
332
333  def _call(self, args, output_file=None):
334    output_fh = None
335    if output_file is not None:
336      output_fh = open(output_file, 'w')
337    subprocess.check_call(args, stdout=output_fh)
338    if output_fh is not None:
339      output_fh.close()
340
341
342class DeviceRunner(Runner):
343
344  """Run perf test on device."""
345
346  def __init__(self, target):
347    self.tmpdir = '/data/local/tmp/'
348    perf_path = 'simpleperf32' if target.endswith('32') else 'simpleperf'
349    super(DeviceRunner, self).__init__(target, self.tmpdir + perf_path)
350    self._download(os.environ['OUT'] + '/system/xbin/' + perf_path, self.tmpdir)
351
352  def _call(self, args, output_file=None):
353    output_fh = None
354    if output_file is not None:
355      output_fh = open(output_file, 'w')
356    args_with_adb = ['adb', 'shell']
357    args_with_adb.append('export LD_LIBRARY_PATH=' + self.tmpdir + ' && ' + ' '.join(args))
358    subprocess.check_call(args_with_adb, stdout=output_fh)
359    if output_fh is not None:
360      output_fh.close()
361
362  def _download(self, file, to_dir):
363    args = ['adb', 'push', file, to_dir]
364    subprocess.check_call(args)
365
366  def record(self, test_executable_name, record_file, additional_options=[]):
367    self._download(os.environ['OUT'] + '/system/bin/' + test_executable_name +
368                   ('32' if self.is32 else '64'), self.tmpdir)
369    super(DeviceRunner, self).record(self.tmpdir + test_executable_name,
370                                     self.tmpdir + record_file,
371                                     additional_options)
372
373  def report(self, record_file, report_file, additional_options=[]):
374    super(DeviceRunner, self).report(self.tmpdir + record_file,
375                                     report_file,
376                                     additional_options)
377
378class ReportAnalyzer(object):
379
380  """Check if perf.report matches expectation in Configuration."""
381
382  def _read_report_file(self, report_file, has_callgraph):
383    fh = open(report_file, 'r')
384    lines = fh.readlines()
385    fh.close()
386
387    lines = [x.rstrip() for x in lines]
388    blank_line_index = -1
389    for i in range(len(lines)):
390      if not lines[i]:
391        blank_line_index = i
392    assert blank_line_index != -1
393    assert blank_line_index + 1 < len(lines)
394    title_line = lines[blank_line_index + 1]
395    report_item_lines = lines[blank_line_index + 2:]
396
397    if has_callgraph:
398      assert re.search(r'^Children\s+Self\s+Command.+Symbol$', title_line)
399    else:
400      assert re.search(r'^Overhead\s+Command.+Symbol$', title_line)
401
402    return self._parse_report_items(report_item_lines, has_callgraph)
403
404  def _parse_report_items(self, lines, has_callgraph):
405    symbols = []
406    cur_symbol = None
407    call_tree_stack = {}
408    vertical_columns = []
409    last_node = None
410    last_depth = -1
411
412    for line in lines:
413      if not line:
414        continue
415      if not line[0].isspace():
416        if has_callgraph:
417          items = line.split(None, 6)
418          assert len(items) == 7
419          children_overhead = float(items[0][:-1])
420          overhead = float(items[1][:-1])
421          comm = items[2]
422          symbol_name = items[6]
423          cur_symbol = Symbol(symbol_name, comm, overhead, children_overhead)
424          symbols.append(cur_symbol)
425        else:
426          items = line.split(None, 5)
427          assert len(items) == 6
428          overhead = float(items[0][:-1])
429          comm = items[1]
430          symbol_name = items[5]
431          cur_symbol = Symbol(symbol_name, comm, overhead, 0)
432          symbols.append(cur_symbol)
433        # Each report item can have different column depths.
434        vertical_columns = []
435      else:
436        for i in range(len(line)):
437          if line[i] == '|':
438            if not vertical_columns or vertical_columns[-1] < i:
439              vertical_columns.append(i)
440
441        if not line.strip('| \t'):
442          continue
443        if line.find('-') == -1:
444          function_name = line.strip('| \t')
445          node = CallTreeNode(function_name)
446          last_node.add_child(node)
447          last_node = node
448          call_tree_stack[last_depth] = node
449        else:
450          pos = line.find('-')
451          depth = -1
452          for i in range(len(vertical_columns)):
453            if pos >= vertical_columns[i]:
454              depth = i
455          assert depth != -1
456
457          line = line.strip('|- \t')
458          m = re.search(r'^[\d\.]+%[-\s]+(.+)$', line)
459          if m:
460            function_name = m.group(1)
461          else:
462            function_name = line
463
464          node = CallTreeNode(function_name)
465          if depth == 0:
466            cur_symbol.set_call_tree(node)
467
468          else:
469            call_tree_stack[depth - 1].add_child(node)
470          call_tree_stack[depth] = node
471          last_node = node
472          last_depth = depth
473
474    return symbols
475
476  def check_report_file(self, test, report_file, has_callgraph):
477    symbols = self._read_report_file(report_file, has_callgraph)
478    if not self._check_symbol_overhead_requirements(test, symbols):
479      return False
480    if has_callgraph:
481      if not self._check_symbol_children_overhead_requirements(test, symbols):
482        return False
483      if not self._check_symbol_relation_requirements(test, symbols):
484        return False
485    return True
486
487  def _check_symbol_overhead_requirements(self, test, symbols):
488    result = True
489    matched = [False] * len(test.symbol_overhead_requirements)
490    matched_overhead = [0] * len(test.symbol_overhead_requirements)
491    for symbol in symbols:
492      for i in range(len(test.symbol_overhead_requirements)):
493        req = test.symbol_overhead_requirements[i]
494        if req.is_match(symbol):
495          matched[i] = True
496          matched_overhead[i] += symbol.overhead
497    for i in range(len(matched)):
498      if not matched[i]:
499        print 'requirement (%s) has no matched symbol in test %s' % (
500            test.symbol_overhead_requirements[i], test)
501        result = False
502      else:
503        fulfilled = req.check_overhead(matched_overhead[i])
504        if not fulfilled:
505          print "Symbol (%s) doesn't match requirement (%s) in test %s" % (
506              symbol, req, test)
507          result = False
508    return result
509
510  def _check_symbol_children_overhead_requirements(self, test, symbols):
511    result = True
512    matched = [False] * len(test.symbol_children_overhead_requirements)
513    for symbol in symbols:
514      for i in range(len(test.symbol_children_overhead_requirements)):
515        req = test.symbol_children_overhead_requirements[i]
516        if req.is_match(symbol):
517          matched[i] = True
518          fulfilled = req.check_overhead(symbol.children_overhead)
519          if not fulfilled:
520            print "Symbol (%s) doesn't match requirement (%s) in test %s" % (
521                symbol, req, test)
522            result = False
523    for i in range(len(matched)):
524      if not matched[i]:
525        print 'requirement (%s) has no matched symbol in test %s' % (
526            test.symbol_children_overhead_requirements[i], test)
527        result = False
528    return result
529
530  def _check_symbol_relation_requirements(self, test, symbols):
531    result = True
532    matched = [False] * len(test.symbol_relation_requirements)
533    for symbol in symbols:
534      for i in range(len(test.symbol_relation_requirements)):
535        req = test.symbol_relation_requirements[i]
536        if req.is_match(symbol):
537          matched[i] = True
538          fulfilled = req.check_relation(symbol.call_tree)
539          if not fulfilled:
540            print "Symbol (%s) doesn't match requirement (%s) in test %s" % (
541                symbol, req, test)
542            result = False
543    for i in range(len(matched)):
544      if not matched[i]:
545        print 'requirement (%s) has no matched symbol in test %s' % (
546            test.symbol_relation_requirements[i], test)
547        result = False
548    return result
549
550
551def build_runner(target, use_callgraph, sampler):
552  if target == 'host32' and use_callgraph:
553    print "Current 64bit linux host doesn't support `simpleperf32 record -g`"
554    return None
555  if target.startswith('host'):
556    runner = HostRunner(target)
557  else:
558    runner = DeviceRunner(target)
559  runner.use_callgraph = use_callgraph
560  runner.sampler = sampler
561  return runner
562
563
564def test_with_runner(runner, tests):
565  report_analyzer = ReportAnalyzer()
566  for test in tests:
567    if test.disable_host and runner.target.startswith('host'):
568      print('Skip test %s on %s' % (test.test_name, runner.target))
569      continue
570    runner.record(test.executable_name, 'perf.data', additional_options = test.record_options)
571    runner.report('perf.data', 'perf.report', additional_options = test.report_options)
572    result = report_analyzer.check_report_file(test, 'perf.report', runner.use_callgraph)
573    str = 'test %s on %s ' % (test.test_name, runner.target)
574    if runner.use_callgraph:
575      str += 'with call graph '
576    str += 'using %s ' % runner.sampler
577    str += ' Succeeded' if result else 'Failed'
578    print str
579    if not result:
580      exit(1)
581
582
583def runtest(target_options, use_callgraph_options, sampler_options, selected_tests):
584  tests = load_config_file(os.path.dirname(os.path.realpath(__file__)) + \
585                           '/runtest.conf')
586  if selected_tests is not None:
587    new_tests = []
588    for test in tests:
589      if test.test_name in selected_tests:
590        new_tests.append(test)
591    tests = new_tests
592  for target in target_options:
593    for use_callgraph in use_callgraph_options:
594      for sampler in sampler_options:
595        runner = build_runner(target, use_callgraph, sampler)
596        if runner is not None:
597          test_with_runner(runner, tests)
598
599
600def main():
601  target_options = ['host64', 'host32', 'device64', 'device32']
602  use_callgraph_options = [False, True]
603  sampler_options = ['cpu-cycles']
604  selected_tests = None
605  i = 1
606  while i < len(sys.argv):
607    if sys.argv[i] == '--host':
608      target_options = ['host64', 'host32']
609    elif sys.argv[i] == '--device':
610      target_options = ['device64', 'device32']
611    elif sys.argv[i] == '--normal':
612      use_callgraph_options = [False]
613    elif sys.argv[i] == '--callgraph':
614      use_callgraph_options = [True]
615    elif sys.argv[i] == '--test':
616      if i < len(sys.argv):
617        i += 1
618        for test in sys.argv[i].split(','):
619          if selected_tests is None:
620            selected_tests = {}
621          selected_tests[test] = True
622    i += 1
623  runtest(target_options, use_callgraph_options, sampler_options, selected_tests)
624
625if __name__ == '__main__':
626  main()
627