1 /*
2  * Copyright (C) 2018 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 package com.android.tradefed.testtype.suite;
18 
19 import com.android.tradefed.config.IConfiguration;
20 import com.android.tradefed.device.DeviceNotAvailableException;
21 import com.android.tradefed.device.DeviceUnresponsiveException;
22 import com.android.tradefed.device.metric.CollectorHelper;
23 import com.android.tradefed.device.metric.IMetricCollector;
24 import com.android.tradefed.device.metric.IMetricCollectorReceiver;
25 import com.android.tradefed.error.IHarnessException;
26 import com.android.tradefed.invoker.IInvocationContext;
27 import com.android.tradefed.invoker.TestInformation;
28 import com.android.tradefed.invoker.logger.CurrentInvocation;
29 import com.android.tradefed.log.LogUtil.CLog;
30 import com.android.tradefed.result.FailureDescription;
31 import com.android.tradefed.result.ILogSaver;
32 import com.android.tradefed.result.ITestInvocationListener;
33 import com.android.tradefed.result.TestRunResult;
34 import com.android.tradefed.result.error.ErrorIdentifier;
35 import com.android.tradefed.retry.IRetryDecision;
36 import com.android.tradefed.retry.MergeStrategy;
37 import com.android.tradefed.retry.RetryLogSaverResultForwarder;
38 import com.android.tradefed.retry.RetryStatistics;
39 import com.android.tradefed.testtype.IRemoteTest;
40 import com.android.tradefed.testtype.ITestCollector;
41 import com.android.tradefed.testtype.ITestFilterReceiver;
42 
43 import com.google.common.annotations.VisibleForTesting;
44 
45 import java.util.ArrayList;
46 import java.util.LinkedHashMap;
47 import java.util.List;
48 import java.util.Map;
49 
50 /**
51  * A wrapper class works on the {@link IRemoteTest} to granulate the IRemoteTest in testcase level.
52  * An IRemoteTest can contain multiple testcases. Previously, these testcases are treated as a
53  * whole: When IRemoteTest runs, all testcases will run. Some IRemoteTest (The ones that implements
54  * ITestFilterReceiver) can accept an allowlist of testcases and only run those testcases. This
55  * class takes advantage of the existing feature and provides a more flexible way to run test suite.
56  *
57  * <ul>
58  *   <li>Single testcase can be retried multiple times (within the same IRemoteTest run) to reduce
59  *       the non-test-error failure rates.
60  *   <li>The retried testcases are dynamically collected from previous run failures.
61  * </ul>
62  *
63  * <p>Note:
64  *
65  * <ul>
66  *   <li>The prerequisite to run a subset of test cases is that the test type should implement the
67  *       interface {@link ITestFilterReceiver}.
68  *   <li>X is customized max retry number.
69  * </ul>
70  */
71 public class GranularRetriableTestWrapper implements IRemoteTest, ITestCollector {
72 
73     private IRetryDecision mRetryDecision;
74     private IRemoteTest mTest;
75     private List<IMetricCollector> mRunMetricCollectors;
76     private TestFailureListener mFailureListener;
77     private IInvocationContext mModuleInvocationContext;
78     private IConfiguration mModuleConfiguration;
79     private ModuleListener mMainGranularRunListener;
80     private RetryLogSaverResultForwarder mRetryAttemptForwarder;
81     private List<ITestInvocationListener> mModuleLevelListeners;
82     private ILogSaver mLogSaver;
83     private String mModuleId;
84     private int mMaxRunLimit;
85 
86     private boolean mCollectTestsOnly = false;
87 
88     // Tracking of the metrics
89     private RetryStatistics mRetryStats = null;
90 
GranularRetriableTestWrapper( IRemoteTest test, ITestInvocationListener mainListener, TestFailureListener failureListener, List<ITestInvocationListener> moduleLevelListeners, int maxRunLimit)91     public GranularRetriableTestWrapper(
92             IRemoteTest test,
93             ITestInvocationListener mainListener,
94             TestFailureListener failureListener,
95             List<ITestInvocationListener> moduleLevelListeners,
96             int maxRunLimit) {
97         mTest = test;
98         mMainGranularRunListener = new ModuleListener(mainListener);
99         mFailureListener = failureListener;
100         mModuleLevelListeners = moduleLevelListeners;
101         mMaxRunLimit = maxRunLimit;
102     }
103 
104     /** Sets the {@link IRetryDecision} to be used. */
setRetryDecision(IRetryDecision decision)105     public void setRetryDecision(IRetryDecision decision) {
106         mRetryDecision = decision;
107     }
108 
109     /**
110      * Set the {@link ModuleDefinition} name as a {@link GranularRetriableTestWrapper} attribute.
111      *
112      * @param moduleId the name of the moduleDefinition.
113      */
setModuleId(String moduleId)114     public void setModuleId(String moduleId) {
115         mModuleId = moduleId;
116     }
117 
118     /**
119      * Set the {@link ModuleDefinition} RunStrategy as a {@link GranularRetriableTestWrapper}
120      * attribute.
121      *
122      * @param skipTestCases whether the testcases should be skipped.
123      */
setMarkTestsSkipped(boolean skipTestCases)124     public void setMarkTestsSkipped(boolean skipTestCases) {
125         mMainGranularRunListener.setMarkTestsSkipped(skipTestCases);
126     }
127 
128     /**
129      * Set the {@link ModuleDefinition}'s runMetricCollector as a {@link
130      * GranularRetriableTestWrapper} attribute.
131      *
132      * @param runMetricCollectors A list of MetricCollector for the module.
133      */
setMetricCollectors(List<IMetricCollector> runMetricCollectors)134     public void setMetricCollectors(List<IMetricCollector> runMetricCollectors) {
135         mRunMetricCollectors = runMetricCollectors;
136     }
137 
138     /**
139      * Set the {@link ModuleDefinition}'s ModuleConfig as a {@link GranularRetriableTestWrapper}
140      * attribute.
141      *
142      * @param moduleConfiguration Provide the module metrics.
143      */
setModuleConfig(IConfiguration moduleConfiguration)144     public void setModuleConfig(IConfiguration moduleConfiguration) {
145         mModuleConfiguration = moduleConfiguration;
146     }
147 
148     /**
149      * Set the {@link IInvocationContext} as a {@link GranularRetriableTestWrapper} attribute.
150      *
151      * @param moduleInvocationContext The wrapper uses the InvocationContext to initialize the
152      *     MetricCollector when necessary.
153      */
setInvocationContext(IInvocationContext moduleInvocationContext)154     public void setInvocationContext(IInvocationContext moduleInvocationContext) {
155         mModuleInvocationContext = moduleInvocationContext;
156     }
157 
158     /**
159      * Set the Module's {@link ILogSaver} as a {@link GranularRetriableTestWrapper} attribute.
160      *
161      * @param logSaver The listeners for each test run should save the logs.
162      */
setLogSaver(ILogSaver logSaver)163     public void setLogSaver(ILogSaver logSaver) {
164         mLogSaver = logSaver;
165     }
166 
167     /**
168      * Initialize a new {@link ModuleListener} for each test run.
169      *
170      * @return a {@link ITestInvocationListener} listener which contains the new {@link
171      *     ModuleListener}, the main {@link ITestInvocationListener} and main {@link
172      *     TestFailureListener}, and wrapped by RunMetricsCollector and Module MetricCollector (if
173      *     not initialized).
174      */
initializeListeners()175     private ITestInvocationListener initializeListeners() {
176         List<ITestInvocationListener> currentTestListener = new ArrayList<>();
177         // Add all the module level listeners, including TestFailureListener
178         if (mModuleLevelListeners != null) {
179             currentTestListener.addAll(mModuleLevelListeners);
180         }
181         currentTestListener.add(mMainGranularRunListener);
182 
183         mRetryAttemptForwarder = new RetryLogSaverResultForwarder(mLogSaver, currentTestListener);
184         ITestInvocationListener runListener = mRetryAttemptForwarder;
185         if (mFailureListener != null) {
186             mFailureListener.setLogger(mRetryAttemptForwarder);
187             currentTestListener.add(mFailureListener);
188         }
189 
190         // The module collectors itself are added: this list will be very limited.
191         // We clone them since the configuration object is shared across shards.
192         for (IMetricCollector collector :
193                 CollectorHelper.cloneCollectors(mModuleConfiguration.getMetricCollectors())) {
194             if (collector.isDisabled()) {
195                 CLog.d("%s has been disabled. Skipping.", collector);
196             } else {
197                 runListener = collector.init(mModuleInvocationContext, runListener);
198             }
199         }
200 
201         return runListener;
202     }
203 
204     /**
205      * Schedule a series of {@link IRemoteTest#run(TestInformation, ITestInvocationListener)}.
206      *
207      * @param listener The ResultForwarder listener which contains a new moduleListener for each
208      *     run.
209      */
210     @Override
run(TestInformation testInfo, ITestInvocationListener listener)211     public void run(TestInformation testInfo, ITestInvocationListener listener)
212             throws DeviceNotAvailableException {
213         mMainGranularRunListener.setCollectTestsOnly(mCollectTestsOnly);
214         ITestInvocationListener allListeners = initializeListeners();
215         // First do the regular run, not retried.
216         intraModuleRun(testInfo, allListeners);
217 
218         if (mMaxRunLimit <= 1) {
219             return;
220         }
221 
222         if (mRetryDecision == null) {
223             CLog.e("RetryDecision is null. Something is misconfigured this shouldn't happen");
224             return;
225         }
226 
227         // Bail out early if there is no need to retry at all.
228         if (!mRetryDecision.shouldRetry(
229                 mTest, 0, mMainGranularRunListener.getTestRunForAttempts(0))) {
230             return;
231         }
232         // Avoid rechecking the shouldRetry below the first time as it could retrigger reboot.
233         boolean firstCheck = true;
234 
235         // Deal with retried attempted
236         long startTime = System.currentTimeMillis();
237         try {
238             CLog.d("Starting intra-module retry.");
239             for (int attemptNumber = 1; attemptNumber < mMaxRunLimit; attemptNumber++) {
240                 if (!firstCheck) {
241                     boolean retry =
242                             mRetryDecision.shouldRetry(
243                                     mTest,
244                                     attemptNumber - 1,
245                                     mMainGranularRunListener.getTestRunForAttempts(
246                                             attemptNumber - 1));
247                     if (!retry) {
248                         return;
249                     }
250                 }
251                 firstCheck = false;
252                 CLog.d("Intra-module retry attempt number %s", attemptNumber);
253                 // Run the tests again
254                 intraModuleRun(testInfo, allListeners);
255             }
256             // Feed the last attempt if we reached here.
257             mRetryDecision.addLastAttempt(
258                     mMainGranularRunListener.getTestRunForAttempts(mMaxRunLimit - 1));
259         } finally {
260             mRetryStats = mRetryDecision.getRetryStatistics();
261             // Track how long we spend in retry
262             mRetryStats.mRetryTime = System.currentTimeMillis() - startTime;
263         }
264     }
265 
266     /** The workflow for each individual {@link IRemoteTest} run. */
intraModuleRun(TestInformation testInfo, ITestInvocationListener runListener)267     private final void intraModuleRun(TestInformation testInfo, ITestInvocationListener runListener)
268             throws DeviceNotAvailableException {
269         try {
270             List<IMetricCollector> clonedCollectors = cloneCollectors(mRunMetricCollectors);
271             if (mTest instanceof IMetricCollectorReceiver) {
272                 ((IMetricCollectorReceiver) mTest).setMetricCollectors(clonedCollectors);
273                 // If test can receive collectors then let it handle how to set them up
274                 mTest.run(testInfo, runListener);
275             } else {
276                 // Module only init the collectors here to avoid triggering the collectors when
277                 // replaying the cached events at the end. This ensures metrics are capture at
278                 // the proper time in the invocation.
279                 for (IMetricCollector collector : clonedCollectors) {
280                     if (collector.isDisabled()) {
281                         CLog.d("%s has been disabled. Skipping.", collector);
282                     } else {
283                         runListener = collector.init(mModuleInvocationContext, runListener);
284                     }
285                 }
286                 mTest.run(testInfo, runListener);
287             }
288         } catch (RuntimeException | AssertionError re) {
289             CLog.e("Module '%s' - test '%s' threw exception:", mModuleId, mTest.getClass());
290             CLog.e(re);
291             CLog.e("Proceeding to the next test.");
292             runListener.testRunFailed(createFromException(re));
293         } catch (DeviceUnresponsiveException due) {
294             // being able to catch a DeviceUnresponsiveException here implies that recovery was
295             // successful, and test execution should proceed to next module.
296             CLog.w(
297                     "Ignored DeviceUnresponsiveException because recovery was "
298                             + "successful, proceeding with next module. Stack trace:");
299             CLog.w(due);
300             CLog.w("Proceeding to the next test.");
301             // If it already was marked as failure do not remark it.
302             if (!mMainGranularRunListener.hasLastAttemptFailed()) {
303                 runListener.testRunFailed(createFromException(due));
304             }
305         } catch (DeviceNotAvailableException dnae) {
306             // TODO: See if it's possible to report IReportNotExecuted
307             CLog.e("Run in progress was not completed due to:");
308             CLog.e(dnae);
309             // If it already was marked as failure do not remark it.
310             if (!mMainGranularRunListener.hasLastAttemptFailed()) {
311                 runListener.testRunFailed(createFromException(dnae));
312             }
313             // Device Not Available Exception are rethrown.
314             throw dnae;
315         } finally {
316             mRetryAttemptForwarder.incrementAttempt();
317         }
318     }
319 
320     /** Get the merged TestRunResults from each {@link IRemoteTest} run. */
getFinalTestRunResults()321     public final List<TestRunResult> getFinalTestRunResults() {
322         MergeStrategy strategy = MergeStrategy.getMergeStrategy(mRetryDecision.getRetryStrategy());
323         mMainGranularRunListener.setMergeStrategy(strategy);
324         return mMainGranularRunListener.getMergedTestRunResults();
325     }
326 
327     @VisibleForTesting
getTestRunResultCollected()328     Map<String, List<TestRunResult>> getTestRunResultCollected() {
329         Map<String, List<TestRunResult>> runResultMap = new LinkedHashMap<>();
330         for (String runName : mMainGranularRunListener.getTestRunNames()) {
331             runResultMap.put(runName, mMainGranularRunListener.getTestRunAttempts(runName));
332         }
333         return runResultMap;
334     }
335 
336     @VisibleForTesting
cloneCollectors(List<IMetricCollector> originalCollectors)337     List<IMetricCollector> cloneCollectors(List<IMetricCollector> originalCollectors) {
338         return CollectorHelper.cloneCollectors(originalCollectors);
339     }
340 
341     /**
342      * Calculate the number of testcases in the {@link IRemoteTest}. This value distincts the same
343      * testcases that are rescheduled multiple times.
344      */
getExpectedTestsCount()345     public final int getExpectedTestsCount() {
346         return mMainGranularRunListener.getExpectedTests();
347     }
348 
349     /** Returns the listener containing all the results. */
getResultListener()350     public ModuleListener getResultListener() {
351         return mMainGranularRunListener;
352     }
353 
354     @Override
setCollectTestsOnly(boolean shouldCollectTest)355     public void setCollectTestsOnly(boolean shouldCollectTest) {
356         mCollectTestsOnly = shouldCollectTest;
357     }
358 
createFromException(Throwable exception)359     private FailureDescription createFromException(Throwable exception) {
360         FailureDescription failure =
361                 CurrentInvocation.createFailure(exception.getMessage(), null).setCause(exception);
362         if (exception instanceof IHarnessException) {
363             ErrorIdentifier id = ((IHarnessException) exception).getErrorId();
364             failure.setErrorIdentifier(id);
365             if (id != null) {
366                 failure.setFailureStatus(id.status());
367             }
368             failure.setOrigin(((IHarnessException) exception).getOrigin());
369         }
370         return failure;
371     }
372 }
373