1#!/usr/bin/python
2# Copyright (C) 2015 The Android Open Source Project
3#
4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
7#
8#      http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15#
16
17import argparse, json, math, re, sys, zipfile
18import xml.etree.ElementTree as ET
19from collections import defaultdict, namedtuple
20
21
22class Size(namedtuple('Size', ['width', 'height'])):
23  """A namedtuple with width and height fields."""
24  def __str__(self):
25    return '%dx%d' % (self.width, self.height)
26
27def nicekey(v):
28  """Returns a nicer sort key for sorting strings.
29
30  This sorts using lower case, with numbers in numerical order first."""
31  key = []
32  num = False
33  for p in re.split('(\d+)', v.lower()):
34    if num:
35      key.append(('0', int(p)))
36    elif p:
37      key.append((p, 0))
38    num = not num
39  return key + [(v, 0)]
40
41def nice(v):
42  """Returns a nicer representation for objects in debug messages.
43
44  Dictionaries are sorted, size is WxH, unicode removed, and floats have 1 digit precision."""
45  if isinstance(v, dict):
46    return 'dict(' + ', '.join(k + '=' + nice(v) for k, v in sorted(v.items(), key=lambda i: nicekey(i[0]))) + ')'
47  if isinstance(v, str):
48    return repr(v)
49  if isinstance(v, int):
50    return str(v)
51  if isinstance(v, Size):
52    return repr(str(v))
53  if isinstance(v, float):
54    return '%.1f' % v
55  if isinstance(v, type(u'')):
56    return repr(str(v))
57  raise ValueError(v)
58
59class ResultParser:
60  @staticmethod
61  def _intify(value):
62    """Returns a value converted to int if possible, else the original value."""
63    try:
64      return int(value)
65    except ValueError:
66      return value
67
68  def _parseDict(self, value):
69    """Parses a MediaFormat from its string representation sans brackets."""
70    return dict((k, self._intify(v))
71                for k, v in re.findall(r'([^ =]+)=([^ [=]+(?:|\[[^\]]+\]))(?:, |$)', value))
72
73  def _cleanFormat(self, format):
74    """Removes internal fields from a parsed MediaFormat."""
75    format.pop('what', None)
76    format.pop('image-data', None)
77
78  MESSAGE_PATTERN = r'(?P<key>\w+)=(?P<value>\{[^}]*\}|[^ ,{}]+)'
79
80  def _parsePartialResult(self, message_match):
81    """Parses a partial test result conforming to the message pattern.
82
83    Returns:
84      A tuple of string key and int, string or dict value, where dict has
85      string keys mapping to int or string values.
86    """
87    key, value = message_match.group('key', 'value')
88    if value.startswith('{'):
89      value = self._parseDict(value[1:-1])
90      if key.endswith('Format'):
91        self._cleanFormat(value)
92    else:
93      value = self._intify(value)
94    return key, value
95
96
97def perc(data, p, fn=round):
98  """Returns a percentile value from a sorted array.
99
100  Arguments:
101    data: sorted data
102    p:    percentile value (0-100)
103    fn:   method used for rounding the percentile to an integer index in data
104  """
105  return data[int(fn((len(data) - 1) * p / 100))]
106
107
108def genXml(data, A=None):
109  yield '<?xml version="1.0" encoding="utf-8" ?>'
110  yield '<!-- Copyright 2016 The Android Open Source Project'
111  yield ''
112  yield '     Licensed under the Apache License, Version 2.0 (the "License");'
113  yield '     you may not use this file except in compliance with the License.'
114  yield '     You may obtain a copy of the License at'
115  yield ''
116  yield '          http://www.apache.org/licenses/LICENSE-2.0'
117  yield ''
118  yield '     Unless required by applicable law or agreed to in writing, software'
119  yield '     distributed under the License is distributed on an "AS IS" BASIS,'
120  yield '     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.'
121  yield '     See the License for the specific language governing permissions and'
122  yield '     limitations under the License.'
123  yield '-->'
124  yield ''
125  yield '<MediaCodecs>'
126  last_section = None
127  from collections import namedtuple
128  Comp = namedtuple('Comp', 'is_decoder google mime name')
129  Result = namedtuple('Result', 'mn mx p95 med geo p5')
130  for comp_, cdata in sorted(data.items()):
131    comp = Comp(*comp_)
132    section = 'Decoders' if comp.is_decoder else 'Encoders'
133    if section != last_section:
134      if last_section:
135        yield '    </%s>' % last_section
136      yield '    <%s>' % section
137      last_section = section
138    yield '        <MediaCodec name="%s" type="%s" update="true">' % (comp.name, comp.mime)
139    for size, sdata in sorted(cdata.items()):
140      data = sorted(sdata)
141      N = len(data)
142      mn, mx = data[0], data[-1]
143
144      if N < 20 and not A.ignore:
145        raise ValueError("need at least 20 data points for %s size %s; have %s" %
146                         (comp.name, size, N))
147
148      TO = 2.2     # tolerance with margin
149      T = TO / 1.1 # tolerance without margin
150
151      Final = namedtuple('Final', 'comment c2 var qual')
152      lastFinal = None
153      for RG in (10, 15, 20, 25, 30, 40, 50):
154        P = 50./RG
155        quality = 0
156        p95, med, p5 = perc(data, P, math.floor), perc(data, 50, round), perc(data, 100 - P, math.ceil)
157        geo = math.sqrt(p5 * p95)
158        comment = ''
159        pub_lo, pub_hi = min(int(p95 * T), round(geo)), max(math.ceil(p5 / T), round(geo))
160        if pub_lo > med:
161          if pub_lo > med * 1.1:
162            quality += 0.5
163            comment += ' SLOW'
164          pub_lo = int(med)
165        if N < 2 * RG:
166          comment += ' N=%d' % N
167          quality += 2
168        RGVAR = False
169        if p5 / p95 > T ** 3:
170          quality += 3
171          RGVAR = True
172          if pub_hi > pub_lo * TO:
173            quality += 1
174            if RG == 10:
175              # find best pub_lo and pub_hi
176              for i in range(N / 2):
177                pub_lo_, pub_hi_ = min(int(data[N / 2 - i - 1] * T), round(geo), int(med)), max(math.ceil(data[N / 2 + i] / T), round(geo))
178                if pub_hi_ > pub_lo_ * TO:
179                  # ???
180                  pub_lo = min(pub_lo, math.ceil(pub_hi_ / TO))
181                  break
182                pub_lo, pub_hi = pub_lo_, pub_hi_
183        if mn < pub_lo / T or mx > pub_hi * T or pub_lo <= pub_hi / T:
184          quality += 1
185          comment += ' FLAKY('
186          if round(mn, 1) < pub_lo / T:
187            comment += 'mn=%.1f < ' % mn
188          comment += 'RANGE'
189          if round(mx, 1) > pub_hi * T:
190            comment += ' < mx=%.1f' % mx
191          comment += ')'
192        if False:
193          comment += ' DATA(mn=%1.f p%d=%1.f accept=%1.f-%1.f p50=%1.f p%d=%1.f mx=%1.f)' % (
194            mn, 100-P, p95, pub_lo / T, pub_hi * T, med, P, p5, mx)
195        var = math.sqrt(p5/p95)
196        if p95 < geo / T or p5 > geo * T:
197          if RGVAR:
198            comment += ' RG.VARIANCE:%.1f' % ((p5/p95) ** (1./3))
199          else:
200            comment += ' variance:%.1f' % var
201        comment = comment.replace('RANGE', '%d - %d' % (math.ceil(pub_lo / T), int(pub_hi * T)))
202        c2 = ''
203        if N >= 2 * RG:
204          c2 += ' N=%d' % N
205        if var <= T or p5 / p95 > T ** 3:
206          c2 += ' v%d%%=%.1f' % (round(100 - 2 * P), var)
207        if A and A.dbg:
208          c2 += ' E=%s' % (str(quality))
209        if c2:
210          c2 = ' <!--%s -->' % c2
211
212        if comment:
213          comment = '            <!-- measured %d%%:%d-%d med:%d%s -->' % (round(100 - 2 * P), int(p95), math.ceil(p5), int(round(med)), comment)
214        if A and A.dbg: yield '<!-- --> %s%s' % (comment, c2)
215        c2 = '            <Limit name="measured-frame-rate-%s" range="%d-%d" />%s' % (size, pub_lo, pub_hi, c2)
216        final = Final(comment, c2, var, quality)
217        if lastFinal and final.var > lastFinal.var * math.sqrt(1.3):
218          if A and A.dbg: yield '<!-- RANGE JUMP -->'
219          break
220        elif not lastFinal or quality <= lastFinal.qual:
221          lastFinal = final
222        if N < 2 * RG or quality >= 4:
223          break
224      comment, c2, var, quality = lastFinal
225
226      if comment:
227        yield comment
228      yield c2
229    yield '        </MediaCodec>'
230  if last_section:
231    yield '    </%s>' % last_section
232  yield '</MediaCodecs>'
233
234
235class Data:
236  def __init__(self):
237    self.data = set()
238    self.kind = {}
239    self.devices = set()
240    self.parser = ResultParser()
241
242  def summarize(self, A=None):
243    devs = sorted(self.devices)
244    #           device  > (not encoder,goog,mime,codec)  >  size > fps
245    xmlInfo = defaultdict(lambda: defaultdict(lambda: defaultdict(list)))
246
247    for mime, encoder, goog in sorted(set(self.kind.values())):
248      for dev, build, codec, size, num, std, avg, p0, p5, p10, p20, p30, p40, p50, p60, p70, p80, p90, p95, p100 in self.data:
249        if self.kind[codec] != (mime, encoder, goog):
250          continue
251
252        if p95 > 2: # ignore measurements at or below 2fps
253          xmlInfo[dev][(not encoder, goog, mime, codec)][size].append(p95)
254        else:
255          print >> sys.stderr, "warning: p95 value is suspiciously low: %s" % (
256            nice(dict(config=dict(dev=dev, codec=codec, size=str(size), N=num),
257                 data=dict(std=std, avg=avg, p0=p0, p5=p5, p10=p10, p20=p20, p30=p30, p40=p40,
258                           p50=p50, p60=p60, p70=p70, p80=p80, p90=p90, p95=p95, p100=p100))))
259    for dev, ddata in xmlInfo.items():
260      outFile = '{}.media_codecs_performance.xml'.format(dev)
261      print >> sys.stderr, "generating", outFile
262      with open(outFile, "wt") as out:
263        for l in genXml(ddata, A=A):
264          out.write(l + '\n')
265          print l
266      print >> sys.stderr, "generated", outFile
267
268  def parse_fmt(self, fmt):
269    return self.parser._parseDict(fmt)
270
271  def parse_perf(self, a, device, build):
272    def rateFn(i):
273      if i is None:
274        return i
275      elif i == 0:
276        return 1e6
277      return 1000. / i
278
279    points = ('avg', 'min', 'p5', 'p10', 'p20', 'p30', 'p40', 'p50', 'p60', 'p70', 'p80', 'p90', 'p95', 'max')
280    a = dict(a)
281    codec = a['codec_name'] + ''
282    mime = a['mime_type']
283    size = Size(a['width'], a['height'])
284    if 'decode_to' in a:
285      fmt = self.parse_fmt(a['output_format'])
286      ofmt = self.parse_fmt(a['input_format'])
287    else:
288      fmt = self.parse_fmt(a['input_format'])
289      ofmt = self.parse_fmt(a['output_format'])
290    size = Size(max(fmt['width'], ofmt['width']), max(fmt['height'], ofmt['height']))
291
292    try:
293      prefix = 'time_avg_stats_'
294      if prefix + 'stdev' in a and a[prefix + 'avg']:
295        stdev = (a[prefix + 'stdev'] * 1e3 / a[prefix + 'avg'] ** 2)
296        data = ((device, build, codec, size, a[prefix  + 'num'], stdev) +
297                tuple(rateFn(a.get(prefix + i)) for i in points))
298        self.data.add(data)
299        self.kind[codec] = (mime, 'decode_to' not in a, codec.lower().startswith('omx.google.'))
300        self.devices.add(data[0])
301    except (KeyError, ZeroDivisionError):
302      print >> sys.stderr, a
303      raise
304
305  def parse_json(self, json, device, build):
306    for test, results in json:
307      if test in ("video_encoder_performance", "video_decoder_performance"):
308        try:
309          if isinstance(results, list) and len(results[0]) and len(results[0][0]) == 2 and len(results[0][0][0]):
310            for result in results:
311              self.parse_perf(result, device, build)
312          else:
313            self.parse_perf(results, device, build)
314        except KeyboardInterrupt:
315          raise
316
317  def parse_result(self, result):
318    device, build = '', ''
319    if not result.endswith('.zip'):
320      print >> sys.stderr, "cannot parse %s" % result
321      return
322
323    try:
324      with zipfile.ZipFile(result) as zip:
325        resultInfo, testInfos = None, []
326        for info in zip.infolist():
327          if re.search(r'/GenericDeviceInfo.deviceinfo.json$', info.filename):
328            resultInfo = info
329          elif re.search(r'/Cts(Media|Video)TestCases\.reportlog\.json$', info.filename):
330            testInfos.append(info)
331        if resultInfo:
332          try:
333            jsonFile = zip.open(resultInfo)
334            jsonData = json.load(jsonFile, encoding='utf-8')
335            device, build = jsonData['build_device'], jsonData['build_id']
336          except ValueError:
337            print >> sys.stderr, "could not parse %s" % resultInfo.filename
338        for info in testInfos:
339          jsonFile = zip.open(info)
340          try:
341            jsonData = json.load(jsonFile, encoding='utf-8', object_pairs_hook=lambda items: items)
342          except ValueError:
343            print >> sys.stderr, "cannot parse JSON in %s" % info.filename
344          self.parse_json(jsonData, device, build)
345
346    except zipfile.BadZipfile:
347      raise ValueError('bad zipfile')
348
349
350P = argparse.ArgumentParser("gar_v2")
351P.add_argument("--dbg", "-v", action='store_true', help="dump debug info into xml")
352P.add_argument("--ignore", "-I", action='store_true', help="ignore minimum sample count")
353P.add_argument("result_zip", nargs="*")
354A = P.parse_args()
355
356D = Data()
357for res in A.result_zip:
358  D.parse_result(res)
359D.summarize(A=A)
360