1 /*
2  * Copyright (C) 2011 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 package com.android.tradefed.invoker;
17 
18 import com.android.ddmlib.Log.LogLevel;
19 import com.android.ddmlib.testrunner.TestResult.TestStatus;
20 import com.android.tradefed.log.LogUtil.CLog;
21 import com.android.tradefed.metrics.proto.MetricMeasurement.Metric;
22 import com.android.tradefed.result.CollectingTestListener;
23 import com.android.tradefed.result.FailureDescription;
24 import com.android.tradefed.result.ILogSaverListener;
25 import com.android.tradefed.result.ITestInvocationListener;
26 import com.android.tradefed.result.InputStreamSource;
27 import com.android.tradefed.result.LogDataType;
28 import com.android.tradefed.result.LogFile;
29 import com.android.tradefed.result.TestDescription;
30 import com.android.tradefed.result.TestResult;
31 import com.android.tradefed.result.TestRunResult;
32 import com.android.tradefed.result.retry.ISupportGranularResults;
33 import com.android.tradefed.util.MultiMap;
34 import com.android.tradefed.util.TimeUtil;
35 
36 import java.util.ArrayList;
37 import java.util.Collection;
38 import java.util.HashMap;
39 import java.util.List;
40 import java.util.Map;
41 import java.util.Map.Entry;
42 
43 /**
44  * A {@link ITestInvocationListener} that collects results from a invocation shard (aka an
45  * invocation split to run on multiple resources in parallel), and forwards them to another
46  * listener.
47  */
48 public class ShardListener extends CollectingTestListener implements ISupportGranularResults {
49 
50     private ITestInvocationListener mMainListener;
51     private IInvocationContext mModuleContext = null;
52     private int mAttemptInProgress = 0;
53     private boolean mEnableGranularResults = false;
54 
55     /**
56      * Create a {@link ShardListener}.
57      *
58      * @param main the {@link ITestInvocationListener} the results should be forwarded. To prevent
59      *     collisions with other {@link ShardListener}s, this object will synchronize on
60      *     <var>main</var> when forwarding results. And results will only be sent once the
61      *     invocation shard completes.
62      */
ShardListener(ITestInvocationListener main)63     public ShardListener(ITestInvocationListener main) {
64         mMainListener = main;
65     }
66 
67     /** {@inheritDoc} */
68     @Override
supportGranularResults()69     public boolean supportGranularResults() {
70         return mEnableGranularResults;
71     }
72 
setSupportGranularResults(boolean enableGranularResults)73     public void setSupportGranularResults(boolean enableGranularResults) {
74         mEnableGranularResults = enableGranularResults;
75     }
76 
77     /**
78      * {@inheritDoc}
79      */
80     @Override
invocationStarted(IInvocationContext context)81     public void invocationStarted(IInvocationContext context) {
82         super.invocationStarted(context);
83         synchronized (mMainListener) {
84             mMainListener.invocationStarted(context);
85         }
86     }
87 
88     /**
89      * {@inheritDoc}
90      */
91     @Override
invocationFailed(Throwable cause)92     public void invocationFailed(Throwable cause) {
93         super.invocationFailed(cause);
94         synchronized (mMainListener) {
95             mMainListener.invocationFailed(cause);
96         }
97     }
98 
99     /** {@inheritDoc} */
100     @Override
invocationFailed(FailureDescription failure)101     public void invocationFailed(FailureDescription failure) {
102         super.invocationFailed(failure);
103         synchronized (mMainListener) {
104             mMainListener.invocationFailed(failure);
105         }
106     }
107 
108     /**
109      * {@inheritDoc}
110      */
111     @Override
testLog(String dataName, LogDataType dataType, InputStreamSource dataStream)112     public void testLog(String dataName, LogDataType dataType, InputStreamSource dataStream) {
113         // forward testLog results immediately, since result reporters might take action on it.
114         synchronized (mMainListener) {
115             if (mMainListener instanceof ShardMainResultForwarder) {
116                 // If the listener is a log saver, we should simply forward the testLog not save
117                 // again.
118                 ((ShardMainResultForwarder) mMainListener)
119                         .testLogForward(dataName, dataType, dataStream);
120             } else {
121                 mMainListener.testLog(dataName, dataType, dataStream);
122             }
123         }
124     }
125 
126     /** {@inheritDoc} */
127     @Override
testLogSaved( String dataName, LogDataType dataType, InputStreamSource dataStream, LogFile logFile)128     public void testLogSaved(
129             String dataName, LogDataType dataType, InputStreamSource dataStream, LogFile logFile) {
130         super.testLogSaved(dataName, dataType, dataStream, logFile);
131         // Forward the testLogSaved callback.
132         synchronized (mMainListener) {
133             if (mMainListener instanceof ILogSaverListener) {
134                 ((ILogSaverListener) mMainListener)
135                         .testLogSaved(dataName, dataType, dataStream, logFile);
136             }
137         }
138     }
139 
140     /** {@inheritDoc} */
141     @Override
testModuleStarted(IInvocationContext moduleContext)142     public void testModuleStarted(IInvocationContext moduleContext) {
143         super.testModuleStarted(moduleContext);
144         mModuleContext = moduleContext;
145     }
146 
147     /** {@inheritDoc} */
148     @Override
testRunStarted(String name, int numTests, int attemptNumber, long startTime)149     public void testRunStarted(String name, int numTests, int attemptNumber, long startTime) {
150         super.testRunStarted(name, numTests, attemptNumber, startTime);
151         mAttemptInProgress = attemptNumber;
152     }
153 
154     /**
155      * {@inheritDoc}
156      */
157     @Override
testRunFailed(String failureMessage)158     public void testRunFailed(String failureMessage) {
159         super.testRunFailed(failureMessage);
160         CLog.logAndDisplay(LogLevel.ERROR, "FAILED: %s failed with message: %s",
161                 getCurrentRunResults().getName(), failureMessage);
162     }
163 
164     /** {@inheritDoc} */
165     @Override
testRunFailed(FailureDescription failure)166     public void testRunFailed(FailureDescription failure) {
167         super.testRunFailed(failure);
168         CLog.logAndDisplay(
169                 LogLevel.ERROR,
170                 "FAILED: %s failed with message: %s",
171                 getCurrentRunResults().getName(),
172                 failure.toString());
173     }
174 
175     /** {@inheritDoc} */
176     @Override
testRunEnded(long elapsedTime, HashMap<String, Metric> runMetrics)177     public void testRunEnded(long elapsedTime, HashMap<String, Metric> runMetrics) {
178         super.testRunEnded(elapsedTime, runMetrics);
179         CLog.logAndDisplay(
180                 LogLevel.INFO, "Sharded test completed: %s", getCurrentRunResults().getName());
181         if (mModuleContext == null) {
182             // testRunEnded only forwards if it's not part of a module. If it's a module
183             // testModuleEnded is in charge of forwarding all run results.
184             synchronized (mMainListener) {
185                 forwardRunResults(getCurrentRunResults(), mAttemptInProgress);
186             }
187             mAttemptInProgress = 0;
188         }
189 
190     }
191 
192     /** {@inheritDoc} */
193     @Override
testModuleEnded()194     public void testModuleEnded() {
195         super.testModuleEnded();
196 
197         synchronized (mMainListener) {
198             mMainListener.testModuleStarted(mModuleContext);
199             List<String> resultNames = new ArrayList<String>();
200             if (mEnableGranularResults) {
201                 for (int i = 0; i < mAttemptInProgress + 1; i++) {
202                     List<TestRunResult> runResults = getTestRunForAttempts(i);
203                     for (TestRunResult runResult : runResults) {
204                         forwardRunResults(runResult, i);
205                         resultNames.add(runResult.getName());
206                     }
207                 }
208             } else {
209                 for (TestRunResult runResult : getMergedTestRunResults()) {
210                     // Forward the run level results
211                     forwardRunResults(runResult, 0);
212                     resultNames.add(runResult.getName());
213                 }
214             }
215 
216             // Ensure we don't carry results from one module to another.
217             for (String name : resultNames) {
218                 clearResultsForName(name);
219             }
220             mMainListener.testModuleEnded();
221         }
222         mModuleContext = null;
223     }
224 
225     /** {@inheritDoc} */
226     @Override
invocationEnded(long elapsedTime)227     public void invocationEnded(long elapsedTime) {
228         super.invocationEnded(elapsedTime);
229         synchronized (mMainListener) {
230             logShardContent(getMergedTestRunResults());
231             // Report all logs not associated with test runs
232             forwardLogAssociation(getNonAssociatedLogFiles(), mMainListener);
233             mMainListener.invocationEnded(elapsedTime);
234         }
235     }
236 
forwardRunResults(TestRunResult runResult, int attempt)237     private void forwardRunResults(TestRunResult runResult, int attempt) {
238         mMainListener.testRunStarted(
239                 runResult.getName(),
240                 runResult.getExpectedTestCount(),
241                 attempt,
242                 runResult.getStartTime());
243         forwardTestResults(runResult.getTestResults());
244         if (runResult.isRunFailure()) {
245             mMainListener.testRunFailed(runResult.getRunFailureDescription());
246         }
247 
248         // Provide a strong association of the run to its logs.
249         forwardLogAssociation(runResult.getRunLoggedFiles(), mMainListener);
250 
251         mMainListener.testRunEnded(runResult.getElapsedTime(), runResult.getRunProtoMetrics());
252     }
253 
forwardTestResults(Map<TestDescription, TestResult> testResults)254     private void forwardTestResults(Map<TestDescription, TestResult> testResults) {
255         for (Map.Entry<TestDescription, TestResult> testEntry : testResults.entrySet()) {
256             mMainListener.testStarted(testEntry.getKey(), testEntry.getValue().getStartTime());
257             switch (testEntry.getValue().getStatus()) {
258                 case FAILURE:
259                     mMainListener.testFailed(testEntry.getKey(), testEntry.getValue().getFailure());
260                     break;
261                 case ASSUMPTION_FAILURE:
262                     mMainListener.testAssumptionFailure(
263                             testEntry.getKey(), testEntry.getValue().getStackTrace());
264                     break;
265                 case IGNORED:
266                     mMainListener.testIgnored(testEntry.getKey());
267                     break;
268                 default:
269                     break;
270             }
271             // Provide a strong association of the test to its logs.
272             forwardLogAssociation(testEntry.getValue().getLoggedFiles(), mMainListener);
273 
274             if (!testEntry.getValue().getStatus().equals(TestStatus.INCOMPLETE)) {
275                 mMainListener.testEnded(
276                         testEntry.getKey(),
277                         testEntry.getValue().getEndTime(),
278                         testEntry.getValue().getProtoMetrics());
279             }
280         }
281     }
282 
283     /** Forward to the listener the logAssociated callback on the files. */
forwardLogAssociation( MultiMap<String, LogFile> loggedFiles, ITestInvocationListener listener)284     private void forwardLogAssociation(
285             MultiMap<String, LogFile> loggedFiles, ITestInvocationListener listener) {
286         for (String key : loggedFiles.keySet()) {
287             for (LogFile logFile : loggedFiles.get(key)) {
288                 if (listener instanceof ILogSaverListener) {
289                     ((ILogSaverListener) listener).logAssociation(key, logFile);
290                 }
291             }
292         }
293     }
294 
295     /** Forward test cases logged files. */
forwardLogAssociation( Map<String, LogFile> loggedFiles, ITestInvocationListener listener)296     private void forwardLogAssociation(
297             Map<String, LogFile> loggedFiles, ITestInvocationListener listener) {
298         for (Entry<String, LogFile> logFile : loggedFiles.entrySet()) {
299             if (listener instanceof ILogSaverListener) {
300                 ((ILogSaverListener) listener).logAssociation(logFile.getKey(), logFile.getValue());
301             }
302         }
303     }
304 
305     /** Log the content of the shard for easier debugging. */
logShardContent(Collection<TestRunResult> listResults)306     private void logShardContent(Collection<TestRunResult> listResults) {
307         StringBuilder sb = new StringBuilder();
308         sb.append("=================================================\n");
309         sb.append(
310                 String.format(
311                         "========== Shard Primary Device %s ==========\n",
312                         getInvocationContext().getDevices().get(0).getSerialNumber()));
313         for (TestRunResult runRes : listResults) {
314             sb.append(
315                     String.format(
316                             "\tRan '%s' in %s\n",
317                             runRes.getName(), TimeUtil.formatElapsedTime(runRes.getElapsedTime())));
318         }
319         sb.append("=================================================\n");
320         CLog.logAndDisplay(LogLevel.DEBUG, sb.toString());
321     }
322 }
323