1 /*
2  * Copyright (c) 2017 Google Inc. All Rights Reserved.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License"); you
5  * may not use this file except in compliance with the License. You may
6  * obtain a copy of the License at
7  *
8  *     http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
13  * implied. See the License for the specific language governing
14  * permissions and limitations under the License.
15  */
16 
17 package com.android.vts.servlet;
18 
19 import com.android.vts.entity.DeviceInfoEntity;
20 import com.android.vts.entity.TestPlanEntity;
21 import com.android.vts.entity.TestPlanRunEntity;
22 import com.android.vts.entity.TestSuiteResultEntity;
23 import com.android.vts.util.DatastoreHelper;
24 import com.android.vts.util.FilterUtil;
25 import com.android.vts.util.Pagination;
26 
27 import com.google.appengine.api.datastore.DatastoreService;
28 import com.google.appengine.api.datastore.DatastoreServiceFactory;
29 import com.google.appengine.api.datastore.Entity;
30 import com.google.appengine.api.datastore.Key;
31 import com.google.appengine.api.datastore.KeyFactory;
32 import com.google.appengine.api.datastore.Query;
33 import com.google.appengine.api.datastore.Query.Filter;
34 import com.google.appengine.api.datastore.Query.SortDirection;
35 import com.google.gson.Gson;
36 import com.google.gson.JsonObject;
37 import com.google.gson.JsonPrimitive;
38 import org.apache.commons.lang.StringUtils;
39 
40 import javax.servlet.RequestDispatcher;
41 import javax.servlet.ServletException;
42 import javax.servlet.http.HttpServletRequest;
43 import javax.servlet.http.HttpServletResponse;
44 import java.io.IOException;
45 
46 import java.util.ArrayList;
47 import java.util.Comparator;
48 import java.util.HashMap;
49 import java.util.HashSet;
50 import java.util.LinkedHashSet;
51 import java.util.List;
52 import java.util.Map;
53 import java.util.Objects;
54 import java.util.Set;
55 import java.util.logging.Level;
56 import java.util.stream.Collectors;
57 import java.util.stream.IntStream;
58 
59 import static com.googlecode.objectify.ObjectifyService.ofy;
60 
61 public class ShowPlanReleaseServlet extends BaseServlet {
62     private static final int MAX_RUNS_PER_PAGE = 90;
63 
64     /** the previous cursor string token list where to start */
65     private static final LinkedHashSet<String> pageCountTokenSet = new LinkedHashSet<>();
66 
67     @Override
getNavParentType()68     public PageType getNavParentType() {
69         return PageType.RELEASE;
70     }
71 
72     @Override
getBreadcrumbLinks(HttpServletRequest request)73     public List<Page> getBreadcrumbLinks(HttpServletRequest request) {
74         String testType =
75                 request.getParameter("type") == null ? "plan" : request.getParameter("type");
76         List<Page> links = new ArrayList<>();
77         String planName = request.getParameter("plan");
78         if (testType.equals("plan")) {
79             links.add(new Page(PageType.RELEASE, "TEST PLANS", "?type=" + testType, true));
80             links.add(new Page(PageType.PLAN_RELEASE, planName, "?plan=" + planName));
81         } else {
82             links.add(new Page(PageType.RELEASE, "TEST SUITES", "?type=" + testType, true));
83             links.add(
84                     new Page(
85                             PageType.PLAN_RELEASE,
86                             planName,
87                             "?plan=" + planName + "&type=" + testType));
88         }
89         return links;
90     }
91 
92     /** Model to describe each test plan run . */
93     private class TestPlanRunMetadata implements Comparable<TestPlanRunMetadata> {
94         public final TestPlanRunEntity testPlanRun;
95         public final List<String> devices;
96         public final Set<DeviceInfoEntity> deviceSet;
97 
TestPlanRunMetadata(TestPlanRunEntity testPlanRun)98         public TestPlanRunMetadata(TestPlanRunEntity testPlanRun) {
99             this.testPlanRun = testPlanRun;
100             this.devices = new ArrayList<>();
101             this.deviceSet = new HashSet<>();
102         }
103 
addDevice(DeviceInfoEntity device)104         public void addDevice(DeviceInfoEntity device) {
105             if (device == null || deviceSet.contains(device)) return;
106             devices.add(
107                     device.getBranch()
108                             + "/"
109                             + device.getBuildFlavor()
110                             + " ("
111                             + device.getBuildId()
112                             + ")");
113             deviceSet.add(device);
114         }
115 
toJson()116         public JsonObject toJson() {
117             JsonObject obj = new JsonObject();
118             obj.add("testPlanRun", testPlanRun.toJson());
119             obj.add("deviceInfo", new JsonPrimitive(StringUtils.join(devices, ", ")));
120             return obj;
121         }
122 
123         @Override
compareTo(TestPlanRunMetadata o)124         public int compareTo(TestPlanRunMetadata o) {
125             return new Long(o.testPlanRun.getStartTimestamp())
126                     .compareTo(this.testPlanRun.getStartTimestamp());
127         }
128     }
129 
130     @Override
doGetHandler(HttpServletRequest request, HttpServletResponse response)131     public void doGetHandler(HttpServletRequest request, HttpServletResponse response)
132             throws IOException {
133         String testType =
134                 request.getParameter("type") == null ? "plan" : request.getParameter("type");
135 
136         RequestDispatcher dispatcher;
137         if (testType.equalsIgnoreCase("plan")) {
138             dispatcher = this.getTestPlanDispatcher(request, response);
139         } else {
140             dispatcher = this.getTestSuiteDispatcher(request, response);
141         }
142 
143         try {
144             request.setAttribute("testType", testType);
145             response.setStatus(HttpServletResponse.SC_OK);
146             dispatcher.forward(request, response);
147         } catch (ServletException e) {
148             logger.log(Level.SEVERE, "Servlet Excpetion caught : ", e);
149         }
150     }
151 
getTestPlanDispatcher( HttpServletRequest request, HttpServletResponse response)152     private RequestDispatcher getTestPlanDispatcher(
153             HttpServletRequest request, HttpServletResponse response) {
154         String PLAN_RELEASE_JSP = "WEB-INF/jsp/show_plan_release.jsp";
155 
156         DatastoreService datastore = DatastoreServiceFactory.getDatastoreService();
157 
158         Long startTime = null; // time in microseconds
159         Long endTime = null; // time in microseconds
160         if (request.getParameter("startTime") != null) {
161             String time = request.getParameter("startTime");
162             try {
163                 startTime = Long.parseLong(time);
164                 startTime = startTime > 0 ? startTime : null;
165             } catch (NumberFormatException e) {
166                 startTime = null;
167             }
168         }
169         if (request.getParameter("endTime") != null) {
170             String time = request.getParameter("endTime");
171             try {
172                 endTime = Long.parseLong(time);
173                 endTime = endTime > 0 ? endTime : null;
174             } catch (NumberFormatException e) {
175                 endTime = null;
176             }
177         }
178         SortDirection dir = SortDirection.DESCENDING;
179         if (startTime != null && endTime == null) {
180             dir = SortDirection.ASCENDING;
181         }
182         boolean unfiltered = request.getParameter("unfiltered") != null;
183         boolean showPresubmit = request.getParameter("showPresubmit") != null;
184         boolean showPostsubmit = request.getParameter("showPostsubmit") != null;
185         // If no params are specified, set to default of postsubmit-only.
186         if (!(showPresubmit || showPostsubmit)) {
187             showPostsubmit = true;
188         }
189 
190         // If unfiltered, set showPre- and Post-submit to true for accurate UI.
191         if (unfiltered) {
192             showPostsubmit = true;
193             showPresubmit = true;
194         }
195         Filter typeFilter = FilterUtil.getTestTypeFilter(showPresubmit, showPostsubmit, unfiltered);
196         String testPlan = request.getParameter("plan");
197         Key testPlanKey = KeyFactory.createKey(TestPlanEntity.KIND, testPlan);
198         Filter testPlanRunFilter =
199                 FilterUtil.getTimeFilter(
200                         testPlanKey, TestPlanRunEntity.KIND, startTime, endTime, typeFilter);
201         Map<String, String[]> parameterMap = request.getParameterMap();
202         List<Filter> userTestFilters = FilterUtil.getUserTestFilters(parameterMap);
203         userTestFilters.add(0, testPlanRunFilter);
204         Filter userDeviceFilter = FilterUtil.getUserDeviceFilter(parameterMap);
205 
206         List<TestPlanRunMetadata> testPlanRuns = new ArrayList<>();
207         Map<Key, TestPlanRunMetadata> testPlanMap = new HashMap<>();
208         Key minKey = null;
209         Key maxKey = null;
210         List<Key> gets =
211                 FilterUtil.getMatchingKeys(
212                         testPlanKey,
213                         TestPlanRunEntity.KIND,
214                         userTestFilters,
215                         userDeviceFilter,
216                         dir,
217                         MAX_RUNS_PER_PAGE);
218         Map<Key, Entity> entityMap = datastore.get(gets);
219         logger.log(Level.INFO, "entityMap => " + entityMap);
220         for (Key key : gets) {
221             if (!entityMap.containsKey(key)) {
222                 continue;
223             }
224             TestPlanRunEntity testPlanRun = TestPlanRunEntity.fromEntity(entityMap.get(key));
225             if (testPlanRun == null) {
226                 continue;
227             }
228             TestPlanRunMetadata metadata = new TestPlanRunMetadata(testPlanRun);
229             testPlanRuns.add(metadata);
230             testPlanMap.put(key, metadata);
231             if (minKey == null || key.compareTo(minKey) < 0) {
232                 minKey = key;
233             }
234             if (maxKey == null || key.compareTo(maxKey) > 0) {
235                 maxKey = key;
236             }
237         }
238         if (minKey != null && maxKey != null) {
239             Filter deviceFilter =
240                     FilterUtil.getDeviceTimeFilter(
241                             testPlanKey, TestPlanRunEntity.KIND, minKey.getId(), maxKey.getId());
242             Query deviceQuery =
243                     new Query(DeviceInfoEntity.KIND)
244                             .setAncestor(testPlanKey)
245                             .setFilter(deviceFilter)
246                             .setKeysOnly();
247             List<Key> deviceGets = new ArrayList<>();
248             for (Entity device :
249                     datastore
250                             .prepare(deviceQuery)
251                             .asIterable(DatastoreHelper.getLargeBatchOptions())) {
252                 if (testPlanMap.containsKey(device.getParent())) {
253                     deviceGets.add(device.getKey());
254                 }
255             }
256             logger.log(Level.INFO, "deviceGets => " + deviceGets);
257             Map<Key, Entity> devices = datastore.get(deviceGets);
258             for (Key key : devices.keySet()) {
259                 if (!testPlanMap.containsKey(key.getParent())) continue;
260                 DeviceInfoEntity device = DeviceInfoEntity.fromEntity(devices.get(key));
261                 if (device == null) continue;
262                 TestPlanRunMetadata metadata = testPlanMap.get(key.getParent());
263                 metadata.addDevice(device);
264             }
265         }
266 
267         testPlanRuns.sort(Comparator.naturalOrder());
268         logger.log(Level.INFO, "testPlanRuns => " + testPlanRuns);
269 
270         if (testPlanRuns.size() > 0) {
271             TestPlanRunMetadata firstRun = testPlanRuns.get(0);
272             endTime = firstRun.testPlanRun.getStartTimestamp();
273 
274             TestPlanRunMetadata lastRun = testPlanRuns.get(testPlanRuns.size() - 1);
275             startTime = lastRun.testPlanRun.getStartTimestamp();
276         }
277 
278         List<JsonObject> testPlanRunObjects = new ArrayList<>();
279         for (TestPlanRunMetadata metadata : testPlanRuns) {
280             testPlanRunObjects.add(metadata.toJson());
281         }
282 
283         FilterUtil.setAttributes(request, parameterMap);
284 
285         request.setAttribute("plan", request.getParameter("plan"));
286         request.setAttribute(
287                 "hasNewer",
288                 new Gson()
289                         .toJson(
290                                 DatastoreHelper.hasNewer(
291                                         testPlanKey, TestPlanRunEntity.KIND, endTime)));
292         request.setAttribute(
293                 "hasOlder",
294                 new Gson()
295                         .toJson(
296                                 DatastoreHelper.hasOlder(
297                                         testPlanKey, TestPlanRunEntity.KIND, startTime)));
298         request.setAttribute("planRuns", new Gson().toJson(testPlanRunObjects));
299 
300         request.setAttribute("unfiltered", unfiltered);
301         request.setAttribute("showPresubmit", showPresubmit);
302         request.setAttribute("showPostsubmit", showPostsubmit);
303         request.setAttribute("startTime", new Gson().toJson(startTime));
304         request.setAttribute("endTime", new Gson().toJson(endTime));
305         request.setAttribute("branches", new Gson().toJson(DatastoreHelper.getAllBranches()));
306         request.setAttribute("devices", new Gson().toJson(DatastoreHelper.getAllBuildFlavors()));
307 
308         RequestDispatcher dispatcher = request.getRequestDispatcher(PLAN_RELEASE_JSP);
309         return dispatcher;
310     }
311 
getTestSuiteDispatcher( HttpServletRequest request, HttpServletResponse response)312     private RequestDispatcher getTestSuiteDispatcher(
313             HttpServletRequest request, HttpServletResponse response) {
314         String PLAN_RELEASE_JSP = "WEB-INF/jsp/show_suite_release.jsp";
315 
316         String testPlan = request.getParameter("plan");
317         String testCategoryType =
318                 Objects.isNull(request.getParameter("testCategoryType"))
319                         ? "1"
320                         : request.getParameter("testCategoryType");
321         int page =
322                 Objects.isNull(request.getParameter("page"))
323                         ? 1
324                         : Integer.valueOf(request.getParameter("page"));
325         String nextPageToken =
326                 Objects.isNull(request.getParameter("nextPageToken"))
327                         ? ""
328                         : request.getParameter("nextPageToken");
329 
330         com.googlecode.objectify.cmd.Query<TestSuiteResultEntity> testSuiteResultEntityQuery =
331                 ofy().load()
332                         .type(TestSuiteResultEntity.class)
333                         .filter("suitePlan", testPlan)
334                         .filter(this.getTestTypeFieldName(testCategoryType), true);
335 
336         if (Objects.nonNull(request.getParameter("branch"))) {
337             request.setAttribute("branch", request.getParameter("branch"));
338             testSuiteResultEntityQuery =
339                     testSuiteResultEntityQuery.filter("branch", request.getParameter("branch"));
340         }
341         if (Objects.nonNull(request.getParameter("hostName"))) {
342             request.setAttribute("hostName", request.getParameter("hostName"));
343             testSuiteResultEntityQuery =
344                     testSuiteResultEntityQuery.filter("hostName", request.getParameter("hostName"));
345         }
346         if (Objects.nonNull(request.getParameter("buildId"))) {
347             request.setAttribute("buildId", request.getParameter("buildId"));
348             testSuiteResultEntityQuery =
349                     testSuiteResultEntityQuery.filter("buildId", request.getParameter("buildId"));
350         }
351         if (Objects.nonNull(request.getParameter("deviceName"))) {
352             request.setAttribute("deviceName", request.getParameter("deviceName"));
353             testSuiteResultEntityQuery =
354                     testSuiteResultEntityQuery.filter(
355                             "deviceName", request.getParameter("deviceName"));
356         }
357         testSuiteResultEntityQuery = testSuiteResultEntityQuery.orderKey(true);
358 
359         Pagination<TestSuiteResultEntity> testSuiteResultEntityPagination =
360                 new Pagination(
361                         testSuiteResultEntityQuery,
362                         page,
363                         Pagination.DEFAULT_PAGE_SIZE,
364                         nextPageToken,
365                         pageCountTokenSet);
366 
367         String nextPageTokenPagination = testSuiteResultEntityPagination.getNextPageCountToken();
368         if (!nextPageTokenPagination.trim().isEmpty()) {
369             this.pageCountTokenSet.add(nextPageTokenPagination);
370         }
371 
372         logger.log(Level.INFO, "pageCountTokenSet => " + pageCountTokenSet);
373 
374         logger.log(Level.INFO, "list => " + testSuiteResultEntityPagination.getList());
375         logger.log(
376                 Level.INFO,
377                 "next page count token => "
378                         + testSuiteResultEntityPagination.getNextPageCountToken());
379         logger.log(
380                 Level.INFO,
381                 "page min range => " + testSuiteResultEntityPagination.getMinPageRange());
382         logger.log(
383                 Level.INFO,
384                 "page max range => " + testSuiteResultEntityPagination.getMaxPageRange());
385         logger.log(Level.INFO, "page size => " + testSuiteResultEntityPagination.getPageSize());
386         logger.log(Level.INFO, "total count => " + testSuiteResultEntityPagination.getTotalCount());
387 
388         request.setAttribute("plan", testPlan);
389         request.setAttribute("page", page);
390         request.setAttribute("testType", "suite");
391         request.setAttribute("testCategoryType", testCategoryType);
392         request.setAttribute("testSuiteResultEntityPagination", testSuiteResultEntityPagination);
393         RequestDispatcher dispatcher = request.getRequestDispatcher(PLAN_RELEASE_JSP);
394         return dispatcher;
395     }
396 
getTestTypeFieldName(String testCategoryType)397     private String getTestTypeFieldName(String testCategoryType) {
398         String fieldName;
399         switch (testCategoryType) {
400             case "1": // TOT
401                 fieldName = "testTypeIndex.TOT";
402                 break;
403             case "2": // OTA
404                 fieldName = "testTypeIndex.OTA";
405                 break;
406             case "4": // SIGNED
407                 fieldName = "testTypeIndex.SIGNED";
408                 break;
409             default:
410                 fieldName = "testTypeIndex.TOT";
411                 break;
412         }
413         return fieldName;
414     }
415 }
416