1 /*
2  * Copyright (C) 2015 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the License);
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an AS IS BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 package com.android.benchmark.results;
18 
19 import android.annotation.TargetApi;
20 import android.view.FrameMetrics;
21 
22 import org.apache.commons.math.stat.descriptive.DescriptiveStatistics;
23 import org.apache.commons.math.stat.descriptive.SummaryStatistics;
24 
25 import java.util.ArrayList;
26 import java.util.List;
27 
28 /**
29  * Utility for storing and analyzing UI benchmark results.
30  */
31 @TargetApi(24)
32 public class UiBenchmarkResult {
33     private static final int BASE_SCORE = 100;
34     private static final int ZERO_SCORE_TOTAL_DURATION_MS = 32;
35     private static final int JANK_PENALTY_THRESHOLD_MS = 12;
36     private static final int ZERO_SCORE_ABOVE_THRESHOLD_MS =
37             ZERO_SCORE_TOTAL_DURATION_MS - JANK_PENALTY_THRESHOLD_MS;
38     private static final double JANK_PENALTY_PER_MS_ABOVE_THRESHOLD =
39             BASE_SCORE / (double)ZERO_SCORE_ABOVE_THRESHOLD_MS;
40     private static final int CONSISTENCY_BONUS_MAX = 100;
41 
42     private static final int METRIC_WAS_JANKY = -1;
43 
44     private static final int[] METRICS = new int[] {
45             FrameMetrics.UNKNOWN_DELAY_DURATION,
46             FrameMetrics.INPUT_HANDLING_DURATION,
47             FrameMetrics.ANIMATION_DURATION,
48             FrameMetrics.LAYOUT_MEASURE_DURATION,
49             FrameMetrics.DRAW_DURATION,
50             FrameMetrics.SYNC_DURATION,
51             FrameMetrics.COMMAND_ISSUE_DURATION,
52             FrameMetrics.SWAP_BUFFERS_DURATION,
53             FrameMetrics.TOTAL_DURATION,
54     };
55     public static final int FRAME_PERIOD_MS = 16;
56 
57     private final DescriptiveStatistics[] mStoredStatistics;
58 
UiBenchmarkResult(List<FrameMetrics> instances)59     public UiBenchmarkResult(List<FrameMetrics> instances) {
60         mStoredStatistics = new DescriptiveStatistics[METRICS.length];
61         insertMetrics(instances);
62     }
63 
UiBenchmarkResult(double[] values)64     public UiBenchmarkResult(double[] values) {
65         mStoredStatistics = new DescriptiveStatistics[METRICS.length];
66         insertValues(values);
67     }
68 
update(List<FrameMetrics> instances)69     public void update(List<FrameMetrics> instances) {
70         insertMetrics(instances);
71     }
72 
update(double[] values)73     public void update(double[] values) {
74         insertValues(values);
75     }
76 
getAverage(int id)77     public double getAverage(int id) {
78         int pos = getMetricPosition(id);
79         return mStoredStatistics[pos].getMean();
80     }
81 
getMinimum(int id)82     public double getMinimum(int id) {
83         int pos = getMetricPosition(id);
84         return mStoredStatistics[pos].getMin();
85     }
86 
getMaximum(int id)87     public double getMaximum(int id) {
88         int pos = getMetricPosition(id);
89         return mStoredStatistics[pos].getMax();
90     }
91 
getMaximumIndex(int id)92     public int getMaximumIndex(int id) {
93         int pos = getMetricPosition(id);
94         double[] storedMetrics = mStoredStatistics[pos].getValues();
95         int maxIdx = 0;
96         for (int i = 0; i < storedMetrics.length; i++) {
97             if (storedMetrics[i] >= storedMetrics[maxIdx]) {
98                 maxIdx = i;
99             }
100         }
101 
102         return maxIdx;
103     }
104 
getMetricAtIndex(int index, int metricId)105     public double getMetricAtIndex(int index, int metricId) {
106         return mStoredStatistics[getMetricPosition(metricId)].getElement(index);
107     }
108 
getPercentile(int id, int percentile)109     public double getPercentile(int id, int percentile) {
110         if (percentile > 100) percentile = 100;
111         if (percentile < 0) percentile = 0;
112 
113         int metricPos = getMetricPosition(id);
114         return mStoredStatistics[metricPos].getPercentile(percentile);
115     }
116 
getTotalFrameCount()117     public int getTotalFrameCount() {
118         if (mStoredStatistics.length == 0) {
119             return 0;
120         }
121 
122         return (int) mStoredStatistics[0].getN();
123     }
124 
getScore()125     public int getScore() {
126         SummaryStatistics badFramesStats = new SummaryStatistics();
127 
128         int totalFrameCount = getTotalFrameCount();
129         for (int i = 0; i < totalFrameCount; i++) {
130             double totalDuration = getMetricAtIndex(i, FrameMetrics.TOTAL_DURATION);
131             if (totalDuration >= 12) {
132                 badFramesStats.addValue(totalDuration);
133             }
134         }
135 
136         int length = getSortedJankFrameIndices().length;
137         double jankFrameCount = 100 * length / (double) totalFrameCount;
138 
139         System.out.println("Mean: " + badFramesStats.getMean() + " JankP: " + jankFrameCount
140                 + " StdDev: " + badFramesStats.getStandardDeviation() +
141                 " Count Bad: " + badFramesStats.getN() + " Count Jank: " + length);
142 
143         return (int) Math.round(
144                 (badFramesStats.getMean()) * jankFrameCount * badFramesStats.getStandardDeviation());
145     }
146 
getJankPenalty()147     public int getJankPenalty() {
148         double total95th = mStoredStatistics[getMetricPosition(FrameMetrics.TOTAL_DURATION)]
149                 .getPercentile(95);
150         System.out.println("95: " + total95th);
151         double aboveThreshold = total95th - JANK_PENALTY_THRESHOLD_MS;
152         if (aboveThreshold <= 0) {
153             return 0;
154         }
155 
156         if (aboveThreshold > ZERO_SCORE_ABOVE_THRESHOLD_MS) {
157             return BASE_SCORE;
158         }
159 
160         return (int) Math.ceil(JANK_PENALTY_PER_MS_ABOVE_THRESHOLD * aboveThreshold);
161     }
162 
getConsistencyBonus()163     public int getConsistencyBonus() {
164         DescriptiveStatistics totalDurationStats =
165                 mStoredStatistics[getMetricPosition(FrameMetrics.TOTAL_DURATION)];
166 
167         double standardDeviation = totalDurationStats.getStandardDeviation();
168         if (standardDeviation == 0) {
169             return CONSISTENCY_BONUS_MAX;
170         }
171 
172         // 1 / CV of the total duration.
173         double bonus = totalDurationStats.getMean() / standardDeviation;
174         return (int) Math.min(Math.round(bonus), CONSISTENCY_BONUS_MAX);
175     }
176 
getSortedJankFrameIndices()177     public int[] getSortedJankFrameIndices() {
178         ArrayList<Integer> jankFrameIndices = new ArrayList<>();
179         boolean tripleBuffered = false;
180         int totalFrameCount = getTotalFrameCount();
181         int totalDurationPos = getMetricPosition(FrameMetrics.TOTAL_DURATION);
182 
183         for (int i = 0; i < totalFrameCount; i++) {
184             double thisDuration = mStoredStatistics[totalDurationPos].getElement(i);
185             if (!tripleBuffered) {
186                 if (thisDuration > FRAME_PERIOD_MS) {
187                     tripleBuffered = true;
188                     jankFrameIndices.add(i);
189                 }
190             } else {
191                 if (thisDuration > 2 * FRAME_PERIOD_MS) {
192                     tripleBuffered = false;
193                     jankFrameIndices.add(i);
194                 }
195             }
196         }
197 
198         int[] res = new int[jankFrameIndices.size()];
199         int i = 0;
200         for (Integer index : jankFrameIndices) {
201             res[i++] = index;
202         }
203         return res;
204     }
205 
getMetricPosition(int id)206     private int getMetricPosition(int id) {
207         for (int i = 0; i < METRICS.length; i++) {
208             if (id == METRICS[i]) {
209                 return i;
210             }
211         }
212 
213         return -1;
214     }
215 
insertMetrics(List<FrameMetrics> instances)216     private void insertMetrics(List<FrameMetrics> instances) {
217         for (FrameMetrics frame : instances) {
218             for (int i = 0; i < METRICS.length; i++) {
219                 DescriptiveStatistics stats = mStoredStatistics[i];
220                 if (stats == null) {
221                     stats = new DescriptiveStatistics();
222                     mStoredStatistics[i] = stats;
223                 }
224 
225                 mStoredStatistics[i].addValue(frame.getMetric(METRICS[i]) / (double) 1000000);
226             }
227         }
228     }
229 
insertValues(double[] values)230     private void insertValues(double[] values) {
231         if (values.length != METRICS.length) {
232             throw new IllegalArgumentException("invalid values array");
233         }
234 
235         for (int i = 0; i < values.length; i++) {
236             DescriptiveStatistics stats = mStoredStatistics[i];
237             if (stats == null) {
238                 stats = new DescriptiveStatistics();
239                 mStoredStatistics[i] = stats;
240             }
241 
242             mStoredStatistics[i].addValue(values[i]);
243         }
244     }
245  }
246