1 /**
2  * Copyright 2016 Google Inc. All Rights Reserved.
3  *
4  * <p>Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
5  * except in compliance with the License. You may obtain a copy of the License at
6  *
7  * <p>http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * <p>Unless required by applicable law or agreed to in writing, software distributed under the
10  * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
11  * express or implied. See the License for the specific language governing permissions and
12  * limitations under the License.
13  */
14 package com.android.vts.util;
15 
16 import com.android.vts.entity.ProfilingPointEntity;
17 import com.android.vts.entity.ProfilingPointRunEntity;
18 import com.android.vts.entity.ProfilingPointSummaryEntity;
19 import com.android.vts.proto.VtsReportMessage.VtsProfilingRegressionMode;
20 import com.google.appengine.api.datastore.DatastoreService;
21 import com.google.appengine.api.datastore.DatastoreServiceFactory;
22 import com.google.appengine.api.datastore.Entity;
23 import com.google.appengine.api.datastore.Query;
24 import java.io.IOException;
25 import java.math.RoundingMode;
26 import java.text.DecimalFormat;
27 import java.util.ArrayList;
28 import java.util.HashMap;
29 import java.util.List;
30 import java.util.Map;
31 import java.util.Set;
32 import java.util.concurrent.TimeUnit;
33 import java.util.logging.Logger;
34 import org.apache.commons.lang.StringUtils;
35 
36 /** PerformanceUtil, a helper class for analyzing profiling and performance data. */
37 public class PerformanceUtil {
38     protected static Logger logger = Logger.getLogger(PerformanceUtil.class.getName());
39 
40     private static final int MAX_BATCH_SIZE = 2000;
41     private static final DecimalFormat FORMATTER;
42     private static final String NAME_DELIMITER = ", ";
43     private static final String OPTION_DELIMITER = "=";
44 
45     /** Initialize the decimal formatter. */
46     static {
47         FORMATTER = new DecimalFormat("#.##");
48         FORMATTER.setRoundingMode(RoundingMode.HALF_UP);
49     }
50 
51     /**
52      * Creates the HTML for a table cell representing the percent change between two numbers.
53      *
54      * <p>Computes the percent change (after - before)/before * 100 and inserts it into a table cell
55      * with the specified style. The color of the cell is white if 'after' is less than before.
56      * Otherwise, the cell is colored red with opacity according to the percent change (100%+ delta
57      * means 100% opacity). If the before value is 0 and the after value is positive, then the color
58      * of the cell is 100% red to indicate an increase of undefined magnitude.
59      *
60      * @param baseline The baseline value observed.
61      * @param test The value to compare against the baseline.
62      * @param classNames A string containing HTML classes to apply to the table cell.
63      * @param style A string containing additional CSS styles.
64      * @returns An HTML string for a colored table cell containing the percent change.
65      */
getPercentChangeHTML( double baseline, double test, String classNames, String style, VtsProfilingRegressionMode mode)66     public static String getPercentChangeHTML(
67             double baseline,
68             double test,
69             String classNames,
70             String style,
71             VtsProfilingRegressionMode mode) {
72         String pctChangeString = "0 %";
73         double alpha = 0;
74         double delta = test - baseline;
75         if (baseline != 0) {
76             double pctChange = delta / baseline;
77             alpha = pctChange * 2;
78             pctChangeString = FORMATTER.format(pctChange * 100) + " %";
79         } else if (delta != 0) {
80             // If the percent change is undefined, the cell will be solid red or white
81             alpha = (int) Math.signum(delta); // get the sign of the delta (+1, 0, -1)
82             pctChangeString = "";
83         }
84         if (mode == VtsProfilingRegressionMode.VTS_REGRESSION_MODE_DECREASING) {
85             alpha = -alpha;
86         }
87         String color = "background-color: rgba(255, 0, 0, " + alpha + "); ";
88         String html = "<td class='" + classNames + "' style='" + color + style + "'>";
89         html += pctChangeString + "</td>";
90         return html;
91     }
92 
93     /**
94      * Compares a test StatSummary to a baseline StatSummary using best-case performance.
95      *
96      * @param baseline The StatSummary object containing initial values to compare against
97      * @param test The StatSummary object containing test values to be compared against the baseline
98      * @param innerClasses Class names to apply to cells on the inside of the grid
99      * @param outerClasses Class names to apply to cells on the outside of the grid
100      * @param innerStyles CSS styles to apply to cells on the inside of the grid
101      * @param outerStyles CSS styles to apply to cells on the outside of the grid
102      * @return HTML string representing the performance of the test versus the baseline
103      */
getBestCasePerformanceComparisonHTML( StatSummary baseline, StatSummary test, String innerClasses, String outerClasses, String innerStyles, String outerStyles)104     public static String getBestCasePerformanceComparisonHTML(
105             StatSummary baseline,
106             StatSummary test,
107             String innerClasses,
108             String outerClasses,
109             String innerStyles,
110             String outerStyles) {
111         if (test == null || baseline == null) {
112             return "<td></td><td></td><td></td><td></td>";
113         }
114         String row = "";
115         // Intensity of red color is a function of the relative (percent) change
116         // in the new value compared to the previous day's. Intensity is a linear function
117         // of percentage change, reaching a ceiling at 100% change (e.g. a doubling).
118         row +=
119                 getPercentChangeHTML(
120                         baseline.getBestCase(),
121                         test.getBestCase(),
122                         innerClasses,
123                         innerStyles,
124                         test.getRegressionMode());
125         row += "<td class='" + innerClasses + "' style='" + innerStyles + "'>";
126         row += FORMATTER.format(baseline.getBestCase());
127         row += "<td class='" + innerClasses + "' style='" + innerStyles + "'>";
128         row += FORMATTER.format(baseline.getMean());
129         row += "<td class='" + outerClasses + "' style='" + outerStyles + "'>";
130         row += FORMATTER.format(baseline.getStd()) + "</td>";
131         return row;
132     }
133 
134     /**
135      * Updates a PerformanceSummary object with data in the specified window.
136      *
137      * @param testName The name of the table whose profiling vectors to retrieve.
138      * @param startTime The (inclusive) start time in microseconds to scan from.
139      * @param endTime The (inclusive) end time in microseconds at which to stop scanning.
140      * @param selectedDevice The name of the device whose data to query for, or null for unfiltered.
141      * @param summaries The list of PerformanceSummary objects to populate with data.
142      * @throws IOException
143      */
updatePerformanceSummary( String testName, long startTime, long endTime, String selectedDevice, List<PerformanceSummary> summaries)144     public static void updatePerformanceSummary(
145             String testName,
146             long startTime,
147             long endTime,
148             String selectedDevice,
149             List<PerformanceSummary> summaries) {
150         DatastoreService datastore = DatastoreServiceFactory.getDatastoreService();
151         Query profilingPointQuery =
152                 new Query(ProfilingPointEntity.KIND)
153                         .setFilter(
154                                 new Query.FilterPredicate(
155                                         ProfilingPointEntity.TEST_NAME,
156                                         Query.FilterOperator.EQUAL,
157                                         testName));
158 
159         List<ProfilingPointEntity> profilingPoints = new ArrayList<>();
160         for (Entity e :
161                 datastore
162                         .prepare(profilingPointQuery)
163                         .asIterable(DatastoreHelper.getLargeBatchOptions())) {
164             ProfilingPointEntity pp = ProfilingPointEntity.fromEntity(e);
165             if (pp == null) continue;
166             profilingPoints.add(pp);
167         }
168 
169         Query.Filter startFilter =
170                 new Query.FilterPredicate(
171                         ProfilingPointSummaryEntity.START_TIME,
172                         Query.FilterOperator.GREATER_THAN_OR_EQUAL,
173                         startTime);
174         Query.Filter endFilter =
175                 new Query.FilterPredicate(
176                         ProfilingPointSummaryEntity.START_TIME,
177                         Query.FilterOperator.LESS_THAN_OR_EQUAL,
178                         endTime);
179         Query.Filter timeFilter = Query.CompositeFilterOperator.and(startFilter, endFilter);
180 
181         Query.Filter deviceFilter;
182         if (selectedDevice != null) {
183             deviceFilter = FilterUtil.FilterKey.TARGET.getFilterForString(selectedDevice);
184         } else {
185             deviceFilter =
186                     FilterUtil.FilterKey.TARGET.getFilterForString(ProfilingPointSummaryEntity.ALL);
187         }
188         deviceFilter =
189                 Query.CompositeFilterOperator.and(
190                         deviceFilter,
191                         FilterUtil.FilterKey.BRANCH.getFilterForString(
192                                 ProfilingPointSummaryEntity.ALL));
193         Query.Filter filter = Query.CompositeFilterOperator.and(timeFilter, deviceFilter);
194 
195         Map<ProfilingPointEntity, Iterable<Entity>> asyncEntities = new HashMap<>();
196         for (ProfilingPointEntity pp : profilingPoints) {
197             Query profilingQuery =
198                     new Query(ProfilingPointSummaryEntity.KIND)
199                             .setAncestor(pp.getKey())
200                             .setFilter(filter);
201             asyncEntities.put(
202                     pp,
203                     datastore
204                             .prepare(profilingQuery)
205                             .asIterable(DatastoreHelper.getLargeBatchOptions()));
206         }
207 
208         for (ProfilingPointEntity pp : asyncEntities.keySet()) {
209             for (Entity ppSummaryEntity : asyncEntities.get(pp)) {
210                 ProfilingPointSummaryEntity ppSummary =
211                         ProfilingPointSummaryEntity.fromEntity(ppSummaryEntity);
212                 if (ppSummary == null) continue;
213                 for (PerformanceSummary perfSummary : summaries) {
214                     if (perfSummary.contains(ppSummary.getStartTime())) {
215                         perfSummary.addData(pp, ppSummaryEntity);
216                     }
217                 }
218             }
219         }
220     }
221 
222     /**
223      * Compares a test StatSummary to a baseline StatSummary using average-case performance.
224      *
225      * @param baseline The StatSummary object containing initial values to compare against
226      * @param test The StatSummary object containing test values to be compared against the baseline
227      * @param innerClasses Class names to apply to cells on the inside of the grid
228      * @param outerClasses Class names to apply to cells on the outside of the grid
229      * @param innerStyles CSS styles to apply to cells on the inside of the grid
230      * @param outerStyles CSS styles to apply to cells on the outside of the grid
231      * @return HTML string representing the performance of the test versus the baseline
232      */
getAvgCasePerformanceComparisonHTML( StatSummary baseline, StatSummary test, String innerClasses, String outerClasses, String innerStyles, String outerStyles)233     public static String getAvgCasePerformanceComparisonHTML(
234             StatSummary baseline,
235             StatSummary test,
236             String innerClasses,
237             String outerClasses,
238             String innerStyles,
239             String outerStyles) {
240         if (test == null || baseline == null) {
241             return "<td></td><td></td><td></td><td></td>";
242         }
243         String row = "";
244         // Intensity of red color is a function of the relative (percent) change
245         // in the new value compared to the previous day's. Intensity is a linear function
246         // of percentage change, reaching a ceiling at 100% change (e.g. a doubling).
247         row +=
248                 getPercentChangeHTML(
249                         baseline.getMean(),
250                         test.getMean(),
251                         innerClasses,
252                         innerStyles,
253                         test.getRegressionMode());
254         row += "<td class='" + innerClasses + "' style='" + innerStyles + "'>";
255         row += FORMATTER.format(baseline.getBestCase());
256         row += "<td class='" + innerClasses + "' style='" + innerStyles + "'>";
257         row += FORMATTER.format(baseline.getMean());
258         row += "<td class='" + outerClasses + "' style='" + outerStyles + "'>";
259         row += FORMATTER.format(baseline.getStd()) + "</td>";
260         return row;
261     }
262 
263     /**
264      * Generates a string of the values in optionsList which have matches in the profiling entity.
265      *
266      * @param profilingRun The entity for a profiling point run.
267      * @param optionKeys A list of keys to match against the optionsList key value pairs.
268      * @return The values in optionsList whose key match a key in optionKeys.
269      */
getOptionAlias( ProfilingPointRunEntity profilingRun, Set<String> optionKeys)270     public static String getOptionAlias(
271             ProfilingPointRunEntity profilingRun, Set<String> optionKeys) {
272         String name = "";
273         if (profilingRun.getOptions() != null) {
274             name = getOptionAlias(profilingRun.getOptions(), optionKeys);
275         }
276         return name;
277     }
278 
279     /**
280      * Generates a string of the values in optionsList which have matches in the profiling entity.
281      *
282      * @param optionList The list of key=value option pair strings.
283      * @param optionKeys A list of keys to match against the optionsList key value pairs.
284      * @return The values in optionsList whose key match a key in optionKeys.
285      */
getOptionAlias(List<String> optionList, Set<String> optionKeys)286     public static String getOptionAlias(List<String> optionList, Set<String> optionKeys) {
287         String name = "";
288         List<String> nameSuffixes = new ArrayList<>();
289         for (String optionString : optionList) {
290             String[] optionParts = optionString.split(OPTION_DELIMITER);
291             if (optionParts.length != 2) {
292                 continue;
293             }
294             if (optionKeys.contains(optionParts[0].trim().toLowerCase())) {
295                 nameSuffixes.add(optionParts[1].trim().toLowerCase());
296             }
297         }
298         if (nameSuffixes.size() > 0) {
299             StringUtils.join(nameSuffixes, NAME_DELIMITER);
300             name += StringUtils.join(nameSuffixes, NAME_DELIMITER);
301         }
302         return name;
303     }
304 }
305