1 /*
2  * Copyright (c) 2017 Google Inc. All Rights Reserved.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License"); you
5  * may not use this file except in compliance with the License. You may
6  * obtain a copy of the License at
7  *
8  *     http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
13  * implied. See the License for the specific language governing
14  * permissions and limitations under the License.
15  */
16 
17 package com.android.vts.servlet;
18 
19 import com.android.vts.entity.DeviceInfoEntity;
20 import com.android.vts.entity.ProfilingPointRunEntity;
21 import com.android.vts.entity.TestCaseRunEntity;
22 import com.android.vts.entity.TestEntity;
23 import com.android.vts.entity.TestRunEntity;
24 import com.android.vts.proto.VtsReportMessage.TestCaseResult;
25 import com.android.vts.util.DatastoreHelper;
26 import com.android.vts.util.FilterUtil;
27 import com.android.vts.util.TestRunDetails;
28 import com.android.vts.util.TestRunMetadata;
29 import com.google.appengine.api.datastore.DatastoreService;
30 import com.google.appengine.api.datastore.DatastoreServiceFactory;
31 import com.google.appengine.api.datastore.Entity;
32 import com.google.appengine.api.datastore.Key;
33 import com.google.appengine.api.datastore.KeyFactory;
34 import com.google.appengine.api.datastore.Query;
35 import com.google.appengine.api.datastore.Query.Filter;
36 import com.google.appengine.api.datastore.Query.SortDirection;
37 import com.google.gson.Gson;
38 import com.google.gson.JsonObject;
39 import java.io.IOException;
40 import java.util.ArrayList;
41 import java.util.Comparator;
42 import java.util.HashMap;
43 import java.util.HashSet;
44 import java.util.List;
45 import java.util.Map;
46 import java.util.Set;
47 import java.util.logging.Level;
48 import javax.servlet.RequestDispatcher;
49 import javax.servlet.ServletException;
50 import javax.servlet.http.HttpServletRequest;
51 import javax.servlet.http.HttpServletResponse;
52 
53 /**
54  * Servlet for handling requests to load individual tables.
55  */
56 public class ShowTreeServlet extends BaseServlet {
57 
58   private static final String TABLE_JSP = "WEB-INF/jsp/show_tree.jsp";
59   // Error message displayed on the webpage is tableName passed is null.
60   private static final String TABLE_NAME_ERROR = "Error : Table name must be passed!";
61   private static final String PROFILING_DATA_ALERT = "No profiling data was found.";
62   private static final int MAX_RESULT_COUNT = 60;
63   private static final int MAX_PREFETCH_COUNT = 10;
64 
65   @Override
getNavParentType()66   public PageType getNavParentType() {
67     return PageType.TOT;
68   }
69 
70   @Override
getBreadcrumbLinks(HttpServletRequest request)71   public List<Page> getBreadcrumbLinks(HttpServletRequest request) {
72     List<Page> links = new ArrayList<>();
73     String testName = request.getParameter("testName");
74     links.add(new Page(PageType.TREE, testName, "?testName=" + testName));
75     return links;
76   }
77 
78   /**
79    * Get the test run details for a test run.
80    *
81    * @param metadata The metadata for the test run whose details will be fetched.
82    * @return The TestRunDetails object for the provided test run.
83    */
processTestDetails(TestRunMetadata metadata)84   public static TestRunDetails processTestDetails(TestRunMetadata metadata) {
85     DatastoreService datastore = DatastoreServiceFactory.getDatastoreService();
86     TestRunDetails details = new TestRunDetails();
87     List<Key> gets = new ArrayList<>();
88         for (long testCaseId : metadata.testRun.getTestCaseIds()) {
89             gets.add(KeyFactory.createKey(TestCaseRunEntity.KIND, testCaseId));
90         }
91     Map<Key, Entity> entityMap = datastore.get(gets);
92     for (int i = 0; i < 1; i++) {
93       for (Key key : entityMap.keySet()) {
94         TestCaseRunEntity testCaseRun = TestCaseRunEntity.fromEntity(entityMap.get(key));
95         if (testCaseRun == null) {
96           continue;
97         }
98         details.addTestCase(testCaseRun);
99       }
100     }
101     return details;
102   }
103 
104   @Override
doGetHandler(HttpServletRequest request, HttpServletResponse response)105   public void doGetHandler(HttpServletRequest request, HttpServletResponse response)
106       throws IOException {
107     boolean unfiltered = request.getParameter("unfiltered") != null;
108     boolean showPresubmit = request.getParameter("showPresubmit") != null;
109     boolean showPostsubmit = request.getParameter("showPostsubmit") != null;
110     Long startTime = null; // time in microseconds
111     Long endTime = null; // time in microseconds
112     DatastoreService datastore = DatastoreServiceFactory.getDatastoreService();
113     RequestDispatcher dispatcher = null;
114 
115     // message to display if profiling point data is not available
116     String profilingDataAlert = "";
117 
118     if (request.getParameter("testName") == null) {
119       request.setAttribute("testName", TABLE_NAME_ERROR);
120       return;
121     }
122     String testName = request.getParameter("testName");
123 
124     if (request.getParameter("startTime") != null) {
125       String time = request.getParameter("startTime");
126       try {
127         startTime = Long.parseLong(time);
128         startTime = startTime > 0 ? startTime : null;
129       } catch (NumberFormatException e) {
130         startTime = null;
131       }
132     }
133     if (request.getParameter("endTime") != null) {
134       String time = request.getParameter("endTime");
135       try {
136         endTime = Long.parseLong(time);
137         endTime = endTime > 0 ? endTime : null;
138       } catch (NumberFormatException e) {
139         endTime = null;
140       }
141     }
142 
143     // If no params are specified, set to default of postsubmit-only.
144     if (!(showPresubmit || showPostsubmit)) {
145       showPostsubmit = true;
146     }
147 
148     // If unfiltered, set showPre- and Post-submit to true for accurate UI.
149     if (unfiltered) {
150       showPostsubmit = true;
151       showPresubmit = true;
152     }
153 
154     // Add result names to list
155     List<String> resultNames = new ArrayList<>();
156     for (TestCaseResult r : TestCaseResult.values()) {
157       resultNames.add(r.name());
158     }
159 
160     SortDirection dir = SortDirection.DESCENDING;
161     if (startTime != null && endTime == null) {
162       dir = SortDirection.ASCENDING;
163     }
164     Key testKey = KeyFactory.createKey(TestEntity.KIND, testName);
165 
166     Filter typeFilter = FilterUtil.getTestTypeFilter(showPresubmit, showPostsubmit, unfiltered);
167     Filter testFilter =
168         FilterUtil.getTimeFilter(
169             testKey, TestRunEntity.KIND, startTime, endTime, typeFilter);
170 
171     Map<String, String[]> parameterMap = request.getParameterMap();
172     List<Filter> userTestFilters = FilterUtil.getUserTestFilters(parameterMap);
173     userTestFilters.add(0, testFilter);
174     Filter userDeviceFilter = FilterUtil.getUserDeviceFilter(parameterMap);
175 
176     List<TestRunMetadata> testRunMetadata = new ArrayList<>();
177     Map<Key, TestRunMetadata> metadataMap = new HashMap<>();
178     Key minKey = null;
179     Key maxKey = null;
180     List<Key> gets =
181         FilterUtil.getMatchingKeys(
182             testKey,
183             TestRunEntity.KIND,
184             userTestFilters,
185             userDeviceFilter,
186             dir,
187             MAX_RESULT_COUNT);
188     Map<Key, Entity> entityMap = datastore.get(gets);
189     for (Key key : gets) {
190       if (!entityMap.containsKey(key)) {
191         continue;
192       }
193       TestRunEntity testRunEntity = TestRunEntity.fromEntity(entityMap.get(key));
194       if (testRunEntity == null) {
195         continue;
196       }
197       if (minKey == null || key.compareTo(minKey) < 0) {
198         minKey = key;
199       }
200       if (maxKey == null || key.compareTo(maxKey) > 0) {
201         maxKey = key;
202       }
203       TestRunMetadata metadata = new TestRunMetadata(testName, testRunEntity);
204       testRunMetadata.add(metadata);
205       metadataMap.put(key, metadata);
206     }
207 
208     List<String> profilingPointNames = new ArrayList<>();
209     if (minKey != null && maxKey != null) {
210       Filter deviceFilter =
211           FilterUtil.getDeviceTimeFilter(
212               testKey, TestRunEntity.KIND, minKey.getId(), maxKey.getId());
213       Query deviceQuery =
214           new Query(DeviceInfoEntity.KIND)
215               .setAncestor(testKey)
216               .setFilter(deviceFilter)
217               .setKeysOnly();
218       List<Key> deviceGets = new ArrayList<>();
219       for (Entity device :
220           datastore
221               .prepare(deviceQuery)
222               .asIterable(DatastoreHelper.getLargeBatchOptions())) {
223         if (metadataMap.containsKey(device.getParent())) {
224           deviceGets.add(device.getKey());
225         }
226       }
227       Map<Key, Entity> devices = datastore.get(deviceGets);
228       for (Key key : devices.keySet()) {
229         if (!metadataMap.containsKey(key.getParent())) {
230           continue;
231         }
232         DeviceInfoEntity device = DeviceInfoEntity.fromEntity(devices.get(key));
233         if (device == null) {
234           continue;
235         }
236         TestRunMetadata metadata = metadataMap.get(key.getParent());
237         metadata.addDevice(device);
238       }
239 
240       Filter profilingFilter =
241           FilterUtil.getProfilingTimeFilter(
242               testKey, TestRunEntity.KIND, minKey.getId(), maxKey.getId());
243 
244       Set<String> profilingPoints = new HashSet<>();
245       Query profilingPointQuery =
246           new Query(ProfilingPointRunEntity.KIND)
247               .setAncestor(testKey)
248               .setFilter(profilingFilter)
249               .setKeysOnly();
250       for (Entity e : datastore.prepare(profilingPointQuery).asIterable()) {
251         profilingPoints.add(e.getKey().getName());
252       }
253 
254       if (profilingPoints.size() == 0) {
255         profilingDataAlert = PROFILING_DATA_ALERT;
256       }
257       profilingPointNames.addAll(profilingPoints);
258       profilingPointNames.sort(Comparator.naturalOrder());
259     }
260 
261     testRunMetadata.sort(
262         (t1, t2) ->
263             new Long(t2.testRun.getStartTimestamp()).compareTo(t1.testRun.getStartTimestamp()));
264     List<JsonObject> testRunObjects = new ArrayList<>();
265 
266     int prefetchCount = 0;
267     for (TestRunMetadata metadata : testRunMetadata) {
268       if (metadata.testRun.getFailCount() > 0 && prefetchCount < MAX_PREFETCH_COUNT) {
269         // process
270         metadata.addDetails(processTestDetails(metadata));
271         ++prefetchCount;
272       }
273       testRunObjects.add(metadata.toJson());
274     }
275 
276     int[] topBuildResultCounts = null;
277     String topBuild = "";
278     if (testRunMetadata.size() > 0) {
279       TestRunMetadata firstRun = testRunMetadata.get(0);
280       topBuild = firstRun.getDeviceInfo();
281       endTime = firstRun.testRun.getStartTimestamp();
282       TestRunDetails topDetails = firstRun.getDetails();
283       if (topDetails == null) {
284         topDetails = processTestDetails(firstRun);
285       }
286       topBuildResultCounts = topDetails.resultCounts;
287 
288       TestRunMetadata lastRun = testRunMetadata.get(testRunMetadata.size() - 1);
289       startTime = lastRun.testRun.getStartTimestamp();
290     }
291 
292     FilterUtil.setAttributes(request, parameterMap);
293 
294     request.setAttribute("testName", request.getParameter("testName"));
295 
296     request.setAttribute("error", profilingDataAlert);
297 
298     request.setAttribute("profilingPointNames", profilingPointNames);
299     request.setAttribute("resultNames", resultNames);
300     request.setAttribute("resultNamesJson", new Gson().toJson(resultNames));
301     request.setAttribute("testRuns", new Gson().toJson(testRunObjects));
302 
303     // data for pie chart
304     request.setAttribute("topBuildResultCounts", new Gson().toJson(topBuildResultCounts));
305     request.setAttribute("topBuildId", topBuild);
306     request.setAttribute("startTime", new Gson().toJson(startTime));
307     request.setAttribute("endTime", new Gson().toJson(endTime));
308     request.setAttribute(
309         "hasNewer",
310         new Gson().toJson(DatastoreHelper.hasNewer(testKey, TestRunEntity.KIND, endTime)));
311     request.setAttribute(
312         "hasOlder",
313         new Gson()
314             .toJson(DatastoreHelper.hasOlder(testKey, TestRunEntity.KIND, startTime)));
315     request.setAttribute("unfiltered", unfiltered);
316     request.setAttribute("showPresubmit", showPresubmit);
317     request.setAttribute("showPostsubmit", showPostsubmit);
318 
319     request.setAttribute("branches", new Gson().toJson(DatastoreHelper.getAllBranches()));
320     request.setAttribute("devices", new Gson().toJson(DatastoreHelper.getAllBuildFlavors()));
321 
322     dispatcher = request.getRequestDispatcher(TABLE_JSP);
323     try {
324       dispatcher.forward(request, response);
325     } catch (ServletException e) {
326       logger.log(Level.SEVERE, "Servlet Exception caught : " + e.toString());
327     }
328   }
329 }
330