1 /*
2  * Copyright (C) 2015 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 package com.android.tradefed.result;
17 
18 import com.android.ddmlib.testrunner.TestResult.TestStatus;
19 import com.android.tradefed.config.Option;
20 import com.android.tradefed.config.OptionClass;
21 import com.android.tradefed.invoker.IInvocationContext;
22 
23 import java.io.PrintStream;
24 import java.text.SimpleDateFormat;
25 import java.util.ArrayList;
26 import java.util.Collections;
27 import java.util.Date;
28 import java.util.LinkedHashMap;
29 import java.util.LinkedHashSet;
30 import java.util.List;
31 import java.util.Map;
32 import java.util.Map.Entry;
33 import java.util.Set;
34 
35 /**
36  * Result reporter to print the test results to the console.
37  *
38  * <p>Prints each test run, each test case, and test metrics, test logs, and test file locations.
39  *
40  * <p>
41  */
42 @OptionClass(alias = "console-result-reporter")
43 public class ConsoleResultReporter extends TestResultListener
44         implements ILogSaverListener, ITestInvocationListener {
45 
46     private static final SimpleDateFormat sTimeStampFormat = new SimpleDateFormat("HH:mm:ss");
47 
48     @Option(
49             name = "suppress-passed-tests",
50             description =
51                     "For functional tests, ommit summary for "
52                             + "passing tests, only print failed and ignored ones")
53     private boolean mSuppressPassedTest = false;
54 
55     @Option(
56             name = "display-failure-summary",
57             description = "Display all the failures at the very end for easier visualization.")
58     private boolean mDisplayFailureSummary = true;
59 
60     private final PrintStream mStream;
61     private Set<LogFile> mLoggedFiles = new LinkedHashSet<>();
62     private Map<TestDescription, TestResult> mFailures = new LinkedHashMap<>();
63     private String mTestTag;
64     private String mRunInProgress;
65     private CountingTestResultListener mResultCountListener = new CountingTestResultListener();
66 
ConsoleResultReporter()67     public ConsoleResultReporter() {
68         this(System.out);
69     }
70 
ConsoleResultReporter(PrintStream outputStream)71     ConsoleResultReporter(PrintStream outputStream) {
72         mStream = outputStream;
73     }
74 
75     @Override
invocationStarted(IInvocationContext context)76     public void invocationStarted(IInvocationContext context) {
77         mTestTag = context.getTestTag();
78     }
79 
80     @Override
testResult(TestDescription test, TestResult result)81     public void testResult(TestDescription test, TestResult result) {
82         mResultCountListener.testResult(test, result);
83         if (mSuppressPassedTest && TestStatus.PASSED.equals(result.getStatus())) {
84             return;
85         }
86         if (mDisplayFailureSummary && TestStatus.FAILURE.equals(result.getStatus())) {
87             mFailures.put(test, result);
88         }
89         print(getTestSummary(mTestTag, test, result));
90     }
91 
92     @Override
testRunStarted(String runName, int testCount)93     public void testRunStarted(String runName, int testCount) {
94         super.testRunStarted(runName, testCount);
95         mRunInProgress = runName;
96     }
97 
98     @Override
testRunFailed(String errorMessage)99     public void testRunFailed(String errorMessage) {
100         print(String.format("%s: run failed: %s\n", mRunInProgress, errorMessage));
101     }
102 
103     @Override
testRunEnded(long elapsedTimeMillis, Map<String, String> metrics)104     public void testRunEnded(long elapsedTimeMillis, Map<String, String> metrics) {
105         super.testRunEnded(elapsedTimeMillis, metrics);
106         if (metrics != null && !metrics.isEmpty()) {
107             String tag = mTestTag != null ? mTestTag : "unknown";
108             String runName = mRunInProgress != null ? mRunInProgress : "unknown";
109             StringBuilder sb = new StringBuilder(tag);
110             sb.append(": ");
111             sb.append(runName);
112             sb.append(": ");
113             List<String> metricKeys = new ArrayList<String>(metrics.keySet());
114             Collections.sort(metricKeys);
115             for (String metricKey : metricKeys) {
116                 sb.append(String.format("%s=%s\n", metricKey, metrics.get(metricKey)));
117             }
118             print(sb.toString());
119         }
120         mRunInProgress = null;
121     }
122 
123     /** {@inheritDoc} */
124     @Override
invocationEnded(long elapsedTime)125     public void invocationEnded(long elapsedTime) {
126         int[] results = mResultCountListener.getResultCounts();
127         StringBuilder sb = new StringBuilder();
128         sb.append("========== Result Summary ==========");
129         sb.append(String.format("\nResults summary for test-tag '%s': ", mTestTag));
130         sb.append(mResultCountListener.getTotalTests());
131         sb.append(" Tests [");
132         sb.append(results[TestStatus.PASSED.ordinal()]);
133         sb.append(" Passed");
134         if (results[TestStatus.FAILURE.ordinal()] > 0) {
135             sb.append(" ");
136             sb.append(results[TestStatus.FAILURE.ordinal()]);
137             sb.append(" Failed");
138         }
139         if (results[TestStatus.IGNORED.ordinal()] > 0) {
140             sb.append(" ");
141             sb.append(results[TestStatus.IGNORED.ordinal()]);
142             sb.append(" Ignored");
143         }
144         if (results[TestStatus.ASSUMPTION_FAILURE.ordinal()] > 0) {
145             sb.append(" ");
146             sb.append(results[TestStatus.ASSUMPTION_FAILURE.ordinal()]);
147             sb.append(" Assumption failures");
148         }
149         if (results[TestStatus.INCOMPLETE.ordinal()] > 0) {
150             sb.append(" ");
151             sb.append(results[TestStatus.INCOMPLETE.ordinal()]);
152             sb.append(" Incomplete");
153         }
154         sb.append("] \r\n");
155         print(sb.toString());
156         if (mDisplayFailureSummary) {
157             for (Entry<TestDescription, TestResult> entry : mFailures.entrySet()) {
158                 print(getTestSummary(mTestTag, entry.getKey(), entry.getValue()));
159             }
160         }
161         // Print the logs
162         for (LogFile logFile : mLoggedFiles) {
163             printLog(logFile);
164         }
165     }
166 
167     /** {@inheritDoc} */
168     @Override
logAssociation(String dataName, LogFile logFile)169     public void logAssociation(String dataName, LogFile logFile) {
170         mLoggedFiles.add(logFile);
171     }
172 
173     /** {@inheritDoc} */
174     @Override
testLogSaved( String dataName, LogDataType dataType, InputStreamSource dataStream, LogFile logFile)175     public void testLogSaved(
176             String dataName, LogDataType dataType, InputStreamSource dataStream, LogFile logFile) {
177         mLoggedFiles.add(logFile);
178     }
179 
printLog(LogFile logFile)180     private void printLog(LogFile logFile) {
181         if (mSuppressPassedTest && !mResultCountListener.hasFailedTests()) {
182             // all tests passed, skip logging
183             return;
184         }
185         String logDesc = logFile.getUrl() == null ? logFile.getPath() : logFile.getUrl();
186         print("Log: " + logDesc + "\r\n");
187     }
188 
189     /** Get the test summary as string including test metrics. */
getTestSummary(String testTag, TestDescription testId, TestResult testResult)190     static String getTestSummary(String testTag, TestDescription testId, TestResult testResult) {
191         StringBuilder sb = new StringBuilder();
192         sb.append(
193                 String.format(
194                         "%s: %s: %s (%dms)\n",
195                         testTag,
196                         testId.toString(),
197                         testResult.getStatus(),
198                         testResult.getEndTime() - testResult.getStartTime()));
199         String stack = testResult.getStackTrace();
200         if (stack != null && !stack.isEmpty()) {
201             sb.append("  stack=\n");
202             String lines[] = stack.split("\\r?\\n");
203             for (String line : lines) {
204                 sb.append(String.format("    %s\n", line));
205             }
206         }
207         Map<String, String> metrics = testResult.getMetrics();
208         if (metrics != null && !metrics.isEmpty()) {
209             List<String> metricKeys = new ArrayList<String>(metrics.keySet());
210             Collections.sort(metricKeys);
211             for (String metricKey : metricKeys) {
212                 sb.append(String.format("    %s: %s\n", metricKey, metrics.get(metricKey)));
213             }
214         }
215 
216         return sb.toString();
217     }
218 
print(String msg)219     private void print(String msg) {
220         mStream.print(sTimeStampFormat.format(new Date()) + " " + msg);
221     }
222 }
223