1 /*
2  * Copyright (C) 2016 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 package com.android.tradefed.testtype;
17 
18 import com.android.tradefed.device.CollectingOutputReceiver;
19 import com.android.tradefed.log.LogUtil.CLog;
20 import com.android.tradefed.metrics.proto.MetricMeasurement.Metric;
21 import com.android.tradefed.result.ITestInvocationListener;
22 import com.android.tradefed.result.TestDescription;
23 import com.android.tradefed.util.proto.TfMetricProtoUtil;
24 
25 import org.w3c.dom.Document;
26 import org.w3c.dom.Element;
27 import org.w3c.dom.NodeList;
28 import org.xml.sax.SAXException;
29 import org.xml.sax.helpers.DefaultHandler;
30 
31 import java.io.File;
32 import java.io.IOException;
33 import java.util.ArrayList;
34 import java.util.Collection;
35 import java.util.HashMap;
36 import java.util.Map;
37 
38 import javax.xml.parsers.DocumentBuilder;
39 import javax.xml.parsers.DocumentBuilderFactory;
40 import javax.xml.parsers.ParserConfigurationException;
41 
42 /**
43  * Parses the 'xml output mode' results of native tests using GTest that run from shell,
44  * and informs a ITestRunListener of the results.
45  */
46 public class GTestXmlResultParser {
47 
48     private final static String TEST_SUITE_TAG = "testsuite";
49     private final static String TEST_CASE_TAG = "testcase";
50     private static final String RESULT_ATTRIBUTE = "result";
51     private static final String SKIPPED_VALUE = "skipped";
52 
53     private final String mTestRunName;
54     private int mNumTestsRun = 0;
55     private int mNumTestsExpected = 0;
56     private long mTotalRunTime = 0;
57     private final Collection<ITestInvocationListener> mTestListeners;
58 
59     /**
60      * Creates the GTestXmlResultParser.
61      *
62      * @param testRunName the test run name to provide to {@link
63      *     ITestInvocationListener#testRunStarted(String, int)}
64      * @param listeners informed of test results as the tests are executing
65      */
GTestXmlResultParser(String testRunName, Collection<ITestInvocationListener> listeners)66     public GTestXmlResultParser(String testRunName, Collection<ITestInvocationListener> listeners) {
67         mTestRunName = testRunName;
68         mTestListeners = new ArrayList<>(listeners);
69     }
70 
71     /**
72      * Creates the GTestXmlResultParser for a single listener.
73      *
74      * @param testRunName the test run name to provide to {@link
75      *     ITestInvocationListener#testRunStarted(String, int)}
76      * @param listener informed of test results as the tests are executing
77      */
GTestXmlResultParser(String testRunName, ITestInvocationListener listener)78     public GTestXmlResultParser(String testRunName, ITestInvocationListener listener) {
79         mTestRunName = testRunName;
80         mTestListeners = new ArrayList<>();
81         if (listener != null) {
82             mTestListeners.add(listener);
83         }
84     }
85 
86     /**
87      * Parse the xml results
88      * @param f {@link File} containing the outputed xml
89      * @param output The output collected from the execution run to complete the logs if necessary
90      */
parseResult(File f, CollectingOutputReceiver output)91     public void parseResult(File f, CollectingOutputReceiver output) {
92         DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
93         Document result = null;
94         try {
95             DocumentBuilder db = dbf.newDocumentBuilder();
96             db.setErrorHandler(new DefaultHandler());
97             result = db.parse(f);
98         } catch (SAXException | IOException | ParserConfigurationException e) {
99             reportTestRunStarted();
100             for (ITestInvocationListener listener : mTestListeners) {
101                 String errorMessage = String.format("Failed to get an xml output from tests,"
102                         + " it probably crashed");
103                 if (output != null) {
104                     errorMessage += "\nlogs:\n" + output.getOutput();
105                     CLog.e(errorMessage);
106                 }
107                 listener.testRunFailed(errorMessage);
108                 listener.testRunEnded(mTotalRunTime, new HashMap<String, Metric>());
109             }
110             return;
111         }
112         Element rootNode = result.getDocumentElement();
113         // Parse root node: "testsuites" for generic infos.
114         getTestSuitesInfo(rootNode);
115         reportTestRunStarted();
116         // Iterate other "testsuite" for each test results.
117         NodeList testSuiteList = rootNode.getElementsByTagName(TEST_SUITE_TAG);
118         if (testSuiteList != null && testSuiteList.getLength() > 0) {
119             for (int i = 0; i < testSuiteList.getLength() ; i++) {
120                 NodeList testcasesList =
121                         ((Element)testSuiteList.item(i)).getElementsByTagName(TEST_CASE_TAG);
122                 // Iterate other the test cases in the test suite.
123                 if (testcasesList != null && testcasesList.getLength() > 0) {
124                     for (int j = 0 ; j < testcasesList.getLength(); j++) {
125                         processTestResult((Element)testcasesList.item(j));
126                     }
127                 }
128             }
129         }
130 
131         if (mNumTestsExpected > mNumTestsRun) {
132             for (ITestInvocationListener listener : mTestListeners) {
133                 listener.testRunFailed(
134                         String.format("Test run incomplete. Expected %d tests, received %d",
135                         mNumTestsExpected, mNumTestsRun));
136             }
137         }
138         for (ITestInvocationListener listener : mTestListeners) {
139             listener.testRunEnded(mTotalRunTime, new HashMap<String, Metric>());
140         }
141     }
142 
getTestSuitesInfo(Element rootNode)143     private void getTestSuitesInfo(Element rootNode) {
144         mNumTestsExpected = Integer.parseInt(rootNode.getAttribute("tests"));
145         mTotalRunTime = (long) (Double.parseDouble(rootNode.getAttribute("time")) * 1000d);
146     }
147 
148     /**
149      * Reports the start of a test run, and the total test count, if it has not been previously
150      * reported.
151      */
reportTestRunStarted()152     private void reportTestRunStarted() {
153         for (ITestInvocationListener listener : mTestListeners) {
154             listener.testRunStarted(mTestRunName, mNumTestsExpected);
155         }
156     }
157 
158     /**
159      * Processes and informs listener when we encounter a tag indicating that a test has started.
160      *
161      * @param testcase Raw log output of the form classname.testname, with an optional time (x ms)
162      */
processTestResult(Element testcase)163     private void processTestResult(Element testcase) {
164         String classname = testcase.getAttribute("classname");
165         String testname = testcase.getAttribute("name");
166         String runtime = testcase.getAttribute("time");
167         boolean skipped = false;
168         if (testcase.hasAttribute(RESULT_ATTRIBUTE)) {
169             skipped = SKIPPED_VALUE.equals(testcase.getAttribute(RESULT_ATTRIBUTE));
170         }
171         ParsedTestInfo parsedResults = new ParsedTestInfo(testname, classname, runtime);
172         TestDescription testId =
173                 new TestDescription(parsedResults.mTestClassName, parsedResults.mTestName);
174         mNumTestsRun++;
175         for (ITestInvocationListener listener : mTestListeners) {
176             listener.testStarted(testId);
177         }
178 
179         if (skipped) {
180             for (ITestInvocationListener listener : mTestListeners) {
181                 listener.testIgnored(testId);
182             }
183         }
184         // If there is a failure tag report failure
185         if (testcase.getElementsByTagName("failure").getLength() != 0) {
186             String trace = ((Element)testcase.getElementsByTagName("failure").item(0))
187                     .getAttribute("message");
188             if (!trace.contains("Failed")) {
189                 // For some reason, the alternative GTest format doesn't specify Failed in the
190                 // trace and error doesn't show properly in reporter, so adding it here.
191                 trace += "\nFailed";
192             }
193             for (ITestInvocationListener listener : mTestListeners) {
194                 listener.testFailed(testId, trace);
195             }
196         }
197 
198         Map<String, String> map = new HashMap<>();
199         map.put("runtime", parsedResults.mTestRunTime);
200         for (ITestInvocationListener listener : mTestListeners) {
201             listener.testEnded(testId, TfMetricProtoUtil.upgradeConvert(map));
202         }
203     }
204 
205     /** Internal helper struct to store parsed test info. */
206     private static class ParsedTestInfo {
207         String mTestName = null;
208         String mTestClassName = null;
209         String mTestRunTime = null;
210 
ParsedTestInfo(String testName, String testClassName, String testRunTime)211         public ParsedTestInfo(String testName, String testClassName, String testRunTime) {
212             mTestName = testName;
213             mTestClassName = testClassName;
214             mTestRunTime = testRunTime;
215         }
216     }
217 }
218