1 /*
2  * Copyright (c) 2016 Google Inc. All Rights Reserved.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License"); you
5  * may not use this file except in compliance with the License. You may
6  * obtain a copy of the License at
7  *
8  *     http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
13  * implied. See the License for the specific language governing
14  * permissions and limitations under the License.
15  */
16 
17 package com.android.vts.job;
18 
19 import com.android.vts.entity.TestEntity;
20 import com.android.vts.util.EmailHelper;
21 import com.android.vts.util.PerformanceSummary;
22 import com.android.vts.util.PerformanceUtil;
23 import com.android.vts.util.ProfilingPointSummary;
24 import com.android.vts.util.StatSummary;
25 import com.android.vts.util.TaskQueueHelper;
26 import com.google.appengine.api.datastore.DatastoreService;
27 import com.google.appengine.api.datastore.DatastoreServiceFactory;
28 import com.google.appengine.api.datastore.Entity;
29 import com.google.appengine.api.datastore.Key;
30 import com.google.appengine.api.datastore.KeyFactory;
31 import com.google.appengine.api.datastore.Query;
32 import com.google.appengine.api.taskqueue.Queue;
33 import com.google.appengine.api.taskqueue.QueueFactory;
34 import com.google.appengine.api.taskqueue.TaskOptions;
35 import java.io.IOException;
36 import java.math.RoundingMode;
37 import java.text.DecimalFormat;
38 import java.util.ArrayList;
39 import java.util.List;
40 import java.util.concurrent.TimeUnit;
41 import java.util.logging.Level;
42 import java.util.logging.Logger;
43 import javax.servlet.http.HttpServlet;
44 import javax.servlet.http.HttpServletRequest;
45 import javax.servlet.http.HttpServletResponse;
46 
47 /** Represents the notifications service which is automatically called on a fixed schedule. */
48 public class VtsPerformanceJobServlet extends BaseJobServlet {
49     protected static final Logger logger =
50             Logger.getLogger(VtsPerformanceJobServlet.class.getName());
51 
52     private static final String PERFORMANCE_JOB_URL = "/cron/vts_performance_job";
53     private static final String MEAN = "Mean";
54     private static final String MAX = "Max";
55     private static final String MIN = "Min";
56     private static final String MIN_DELTA = "ΔMin (%)";
57     private static final String MAX_DELTA = "ΔMax (%)";
58     private static final String HIGHER_IS_BETTER =
59             "Note: Higher values are better. Maximum is the best-case performance.";
60     private static final String LOWER_IS_BETTER =
61             "Note: Lower values are better. Minimum is the best-case performance.";
62     private static final String STD = "Std";
63     private static final String SUBJECT_PREFIX = "Daily Performance Digest: ";
64     private static final String LAST_WEEK = "Last Week";
65     private static final String LABEL_STYLE = "font-family: arial";
66     private static final String SUBTEXT_STYLE = "font-family: arial; font-size: 12px";
67     private static final String TABLE_STYLE =
68             "width: 100%; border-collapse: collapse; border: 1px solid black; font-size: 12px; font-family: arial;";
69     private static final String SECTION_LABEL_STYLE =
70             "border: 1px solid black; border-bottom: none; background-color: lightgray;";
71     private static final String COL_LABEL_STYLE =
72             "border: 1px solid black; border-bottom-width: 2px; border-top: 1px dotted gray; background-color: lightgray;";
73     private static final String HEADER_COL_STYLE =
74             "border-top: 1px dotted gray; border-right: 2px solid black; text-align: right; background-color: lightgray;";
75     private static final String INNER_CELL_STYLE =
76             "border-top: 1px dotted gray; border-right: 1px dotted gray; text-align: right;";
77     private static final String OUTER_CELL_STYLE =
78             "border-top: 1px dotted gray; border-right: 2px solid black; text-align: right;";
79 
80     private static final DecimalFormat FORMATTER;
81 
82     /** Initialize the decimal formatter. */
83     static {
84         FORMATTER = new DecimalFormat("#.##");
85         FORMATTER.setRoundingMode(RoundingMode.HALF_UP);
86     }
87 
88     /**
89      * Generates an HTML summary of the performance changes for the profiling results in the
90      * specified table.
91      *
92      * <p>Retrieves the past 24 hours of profiling data and compares it to the 24 hours that
93      * preceded it. Creates a table representation of the mean and standard deviation for each
94      * profiling point. When performance degrades, the cell is shaded red.
95      *
96      * @param testName The name of the test whose profiling data to summarize.
97      * @param perfSummaries List of PerformanceSummary objects for each profiling run (in reverse
98      *     chronological order).
99      * @param labels List of string labels for use as the column headers.
100      * @returns An HTML string containing labeled table summaries.
101      */
getPerformanceSummary( String testName, List<PerformanceSummary> perfSummaries, List<String> labels)102     public static String getPerformanceSummary(
103             String testName, List<PerformanceSummary> perfSummaries, List<String> labels) {
104         if (perfSummaries.size() == 0) return "";
105         PerformanceSummary now = perfSummaries.get(0);
106         String tableHTML = "<p style='" + LABEL_STYLE + "'><b>";
107         tableHTML += testName + "</b></p>";
108         for (String profilingPoint : now.getProfilingPointNames()) {
109             ProfilingPointSummary summary = now.getProfilingPointSummary(profilingPoint);
110             tableHTML += "<table cellpadding='2' style='" + TABLE_STYLE + "'>";
111 
112             // Format header rows
113             String[] headerRows = new String[] {profilingPoint, summary.yLabel};
114             int colspan = labels.size() * 4;
115             for (String content : headerRows) {
116                 tableHTML += "<tr><td colspan='" + colspan + "'>" + content + "</td></tr>";
117             }
118 
119             // Format section labels
120             tableHTML += "<tr>";
121             for (int i = 0; i < labels.size(); i++) {
122                 String content = labels.get(i);
123                 tableHTML += "<th style='" + SECTION_LABEL_STYLE + "' ";
124                 if (i == 0) tableHTML += "colspan='1'";
125                 else if (i == 1) tableHTML += "colspan='3'";
126                 else tableHTML += "colspan='4'";
127                 tableHTML += ">" + content + "</th>";
128             }
129             tableHTML += "</tr>";
130 
131             String deltaString;
132             String bestCaseString;
133             String subtext;
134             switch (now.getProfilingPointSummary(profilingPoint).getRegressionMode()) {
135                 case VTS_REGRESSION_MODE_DECREASING:
136                     deltaString = MAX_DELTA;
137                     bestCaseString = MAX;
138                     subtext = HIGHER_IS_BETTER;
139                     break;
140                 default:
141                     deltaString = MIN_DELTA;
142                     bestCaseString = MIN;
143                     subtext = LOWER_IS_BETTER;
144                     break;
145             }
146 
147             // Format column labels
148             tableHTML += "<tr>";
149             for (int i = 0; i < labels.size(); i++) {
150                 if (i > 1) {
151                     tableHTML += "<th style='" + COL_LABEL_STYLE + "'>" + deltaString + "</th>";
152                 }
153                 if (i == 0) {
154                     tableHTML += "<th style='" + COL_LABEL_STYLE + "'>";
155                     tableHTML += summary.xLabel + "</th>";
156                 } else if (i > 0) {
157                     tableHTML += "<th style='" + COL_LABEL_STYLE + "'>" + bestCaseString + "</th>";
158                     tableHTML += "<th style='" + COL_LABEL_STYLE + "'>" + MEAN + "</th>";
159                     tableHTML += "<th style='" + COL_LABEL_STYLE + "'>" + STD + "</th>";
160                 }
161             }
162             tableHTML += "</tr>";
163 
164             // Populate data cells
165             for (StatSummary stats : summary) {
166                 String label = stats.getLabel();
167                 tableHTML += "<tr><td style='" + HEADER_COL_STYLE + "'>" + label;
168                 tableHTML += "</td><td style='" + INNER_CELL_STYLE + "'>";
169                 tableHTML += FORMATTER.format(stats.getBestCase()) + "</td>";
170                 tableHTML += "<td style='" + INNER_CELL_STYLE + "'>";
171                 tableHTML += FORMATTER.format(stats.getMean()) + "</td>";
172                 tableHTML += "<td style='" + OUTER_CELL_STYLE + "'>";
173                 if (stats.getCount() < 2) {
174                     tableHTML += " - </td>";
175                 } else {
176                     tableHTML += FORMATTER.format(stats.getStd()) + "</td>";
177                 }
178                 for (int i = 1; i < perfSummaries.size(); i++) {
179                     PerformanceSummary oldPerfSummary = perfSummaries.get(i);
180                     if (oldPerfSummary.hasProfilingPoint(profilingPoint)) {
181                         StatSummary baseline =
182                                 oldPerfSummary
183                                         .getProfilingPointSummary(profilingPoint)
184                                         .getStatSummary(label);
185                         tableHTML +=
186                                 PerformanceUtil.getBestCasePerformanceComparisonHTML(
187                                         baseline,
188                                         stats,
189                                         "",
190                                         "",
191                                         INNER_CELL_STYLE,
192                                         OUTER_CELL_STYLE);
193                     } else tableHTML += "<td></td><td></td><td></td><td></td>";
194                 }
195                 tableHTML += "</tr>";
196             }
197             tableHTML += "</table>";
198             tableHTML += "<i style='" + SUBTEXT_STYLE + "'>" + subtext + "</i><br><br>";
199         }
200         return tableHTML;
201     }
202 
203     @Override
doGet(HttpServletRequest request, HttpServletResponse response)204     public void doGet(HttpServletRequest request, HttpServletResponse response) throws IOException {
205         DatastoreService datastore = DatastoreServiceFactory.getDatastoreService();
206         Queue queue = QueueFactory.getDefaultQueue();
207         Query q = new Query(TestEntity.KIND).setKeysOnly();
208         List<TaskOptions> tasks = new ArrayList<>();
209         for (Entity test : datastore.prepare(q).asIterable()) {
210             if (test.getKey().getName() == null) {
211                 continue;
212             }
213             TaskOptions task =
214                     TaskOptions.Builder.withUrl(PERFORMANCE_JOB_URL)
215                             .param("testKey", KeyFactory.keyToString(test.getKey()))
216                             .method(TaskOptions.Method.POST);
217             tasks.add(task);
218         }
219         TaskQueueHelper.addToQueue(queue, tasks);
220     }
221 
222     @Override
doPost(HttpServletRequest request, HttpServletResponse response)223     public void doPost(HttpServletRequest request, HttpServletResponse response)
224             throws IOException {
225         String testKeyString = request.getParameter("testKey");
226         Key testKey;
227         try {
228             testKey = KeyFactory.stringToKey(testKeyString);
229         } catch (IllegalArgumentException e) {
230             logger.log(Level.WARNING, "Invalid key specified: " + testKeyString);
231             return;
232         }
233 
234         long nowMicro = TimeUnit.MILLISECONDS.toMicros(System.currentTimeMillis());
235 
236         // Add today to the list of time intervals to analyze
237         List<PerformanceSummary> summaries = new ArrayList<>();
238         PerformanceSummary today =
239                 new PerformanceSummary(nowMicro - TimeUnit.DAYS.toMicros(1), nowMicro);
240         summaries.add(today);
241 
242         // Add yesterday as a baseline time interval for analysis
243         long oneDayAgo = nowMicro - TimeUnit.DAYS.toMicros(1);
244         PerformanceSummary yesterday =
245                 new PerformanceSummary(oneDayAgo - TimeUnit.DAYS.toMicros(1), oneDayAgo);
246         summaries.add(yesterday);
247 
248         // Add last week as a baseline time interval for analysis
249         long oneWeek = TimeUnit.DAYS.toMicros(7);
250         long oneWeekAgo = nowMicro - oneWeek;
251 
252         String spanString = "<span class='date-label'>";
253         String label =
254                 spanString + TimeUnit.MICROSECONDS.toMillis(oneWeekAgo - oneWeek) + "</span>";
255         label += " - " + spanString + TimeUnit.MICROSECONDS.toMillis(oneWeekAgo) + "</span>";
256         PerformanceSummary lastWeek =
257                 new PerformanceSummary(oneWeekAgo - oneWeek, oneWeekAgo, label);
258         summaries.add(lastWeek);
259         PerformanceUtil.updatePerformanceSummary(
260                 testKey.getName(), oneWeekAgo - oneWeek, nowMicro, null, summaries);
261 
262         List<PerformanceSummary> nonEmptySummaries = new ArrayList<>();
263         List<String> labels = new ArrayList<>();
264         labels.add("");
265         for (PerformanceSummary perfSummary : summaries) {
266             if (perfSummary.size() == 0) continue;
267             nonEmptySummaries.add(perfSummary);
268             labels.add(perfSummary.label);
269         }
270         String body = getPerformanceSummary(testKey.getName(), nonEmptySummaries, labels);
271         if (body == null || body.equals("")) {
272             return;
273         }
274         List<String> emails = EmailHelper.getSubscriberEmails(testKey);
275         if (emails.size() == 0) {
276             return;
277         }
278         String subject = SUBJECT_PREFIX + testKey.getName();
279         EmailHelper.send(emails, subject, body);
280     }
281 }
282