1 /*
2  * Copyright (C) 2017 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 package com.android.compatibility.common.tradefed.util;
18 
19 import com.android.compatibility.common.tradefed.build.CompatibilityBuildHelper;
20 import com.android.compatibility.common.tradefed.result.SubPlanHelper;
21 import com.android.compatibility.common.tradefed.testtype.ISubPlan;
22 import com.android.compatibility.common.tradefed.testtype.ModuleRepo;
23 import com.android.compatibility.common.util.IInvocationResult;
24 import com.android.compatibility.common.util.LightInvocationResult;
25 import com.android.compatibility.common.util.ResultHandler;
26 import com.android.compatibility.common.util.TestFilter;
27 import com.android.tradefed.build.IBuildInfo;
28 import com.android.tradefed.config.ArgsOptionParser;
29 import com.android.tradefed.config.ConfigurationException;
30 import com.android.tradefed.device.DeviceNotAvailableException;
31 import com.android.tradefed.device.ITestDevice;
32 import com.android.tradefed.util.ArrayUtil;
33 
34 import com.google.common.annotations.VisibleForTesting;
35 
36 import java.io.FileNotFoundException;
37 import java.util.HashSet;
38 import java.util.List;
39 import java.util.Set;
40 
41 /**
42  * Helper for generating --include-filter and --exclude-filter values on compatibility retry.
43  */
44 public class RetryFilterHelper {
45 
46     protected String mSubPlan;
47     protected Set<String> mIncludeFilters = new HashSet<>();
48     protected Set<String> mExcludeFilters = new HashSet<>();
49     protected String mAbiName = null;
50     protected String mModuleName = null;
51     protected String mTestName = null;
52     protected RetryType mRetryType = null;
53 
54     /* Instance variables handy for retreiving the result to be retried */
55     private CompatibilityBuildHelper mBuild = null;
56     private int mSessionId;
57 
58     /* Sets to be populated by retry logic and returned by getter methods */
59     private Set<String> mRetryIncludes;
60     private Set<String> mRetryExcludes;
61 
RetryFilterHelper()62     public RetryFilterHelper() {}
63 
64     /**
65      * Constructor for a {@link RetryFilterHelper}. Requires a CompatibilityBuildHelper for
66      * retrieving previous sessions and the ID of the session to retry.
67      */
RetryFilterHelper(CompatibilityBuildHelper build, int sessionId)68     public RetryFilterHelper(CompatibilityBuildHelper build, int sessionId) {
69         mBuild = build;
70         mSessionId = sessionId;
71     }
72 
73     /**
74      * Constructor for a {@link RetryFilterHelper}.
75      *
76      * @param build a {@link CompatibilityBuildHelper} describing the build.
77      * @param sessionId The ID of the session to retry.
78      * @param subPlan The name of a subPlan to be used. Can be null.
79      * @param includeFilters The include module filters to apply
80      * @param excludeFilters The exclude module filters to apply
81      * @param abiName The name of abi to use. Can be null.
82      * @param moduleName The name of the module to run. Can be null.
83      * @param testName The name of the test to run. Can be null.
84      * @param retryType The type of results to retry. Can be null.
85      */
RetryFilterHelper(CompatibilityBuildHelper build, int sessionId, String subPlan, Set<String> includeFilters, Set<String> excludeFilters, String abiName, String moduleName, String testName, RetryType retryType)86     public RetryFilterHelper(CompatibilityBuildHelper build, int sessionId, String subPlan,
87             Set<String> includeFilters, Set<String> excludeFilters, String abiName,
88             String moduleName, String testName, RetryType retryType) {
89         this(build, sessionId);
90         mSubPlan = subPlan;
91         mIncludeFilters.addAll(includeFilters);
92         mExcludeFilters.addAll(excludeFilters);
93         mAbiName = abiName;
94         mModuleName = moduleName;
95         mTestName = testName;
96         mRetryType = retryType;
97     }
98 
99     /**
100      * Throws an {@link IllegalArgumentException} if the device build fingerprint doesn't match
101      * the fingerprint recorded in the previous session's result.
102      */
validateBuildFingerprint(ITestDevice device)103     public void validateBuildFingerprint(ITestDevice device) throws DeviceNotAvailableException {
104         String oldBuildFingerprint = new LightInvocationResult(getResult()).getBuildFingerprint();
105         if (oldBuildFingerprint == null) {
106             throw new FingerprintComparisonException(
107                     "Could not find the build_fingerprint field in the result xml.");
108         }
109         String currentBuildFingerprint = device.getProperty("ro.build.fingerprint");
110         if (!oldBuildFingerprint.equals(currentBuildFingerprint)) {
111             throw new FingerprintComparisonException(String.format(
112                     "Device build fingerprint must match %s to retry session %d",
113                     oldBuildFingerprint, mSessionId));
114         }
115     }
116 
117     /**
118      * Copy all applicable options from an input object to this instance of RetryFilterHelper.
119      */
120     @VisibleForTesting
setAllOptionsFrom(RetryFilterHelper obj)121     void setAllOptionsFrom(RetryFilterHelper obj) {
122         clearOptions(); // Remove existing options first
123         mSubPlan = obj.mSubPlan;
124         mIncludeFilters.addAll(obj.mIncludeFilters);
125         mExcludeFilters.addAll(obj.mExcludeFilters);
126         mAbiName = obj.mAbiName;
127         mModuleName = obj.mModuleName;
128         mTestName = obj.mTestName;
129         mRetryType = obj.mRetryType;
130     }
131 
132     /**
133      * Clear all option values of this RetryFilterHelper.
134      */
clearOptions()135     public void clearOptions() {
136         mSubPlan = null;
137         mIncludeFilters.clear();
138         mExcludeFilters.clear();
139         mModuleName = null;
140         mTestName = null;
141         mRetryType = null;
142         mAbiName = null;
143     }
144 
145     /**
146      * Using command-line arguments from the previous session's result, set the input object's
147      * option values to the values applied in the previous session.
148      */
setCommandLineOptionsFor(Object obj)149     public void setCommandLineOptionsFor(Object obj) {
150         // only need light version to retrieve command-line args
151         IInvocationResult result = new LightInvocationResult(getResult());
152         String retryCommandLineArgs = result.getCommandLineArgs();
153         if (retryCommandLineArgs != null) {
154             try {
155                 // parse the command-line string from the result file and set options
156                 ArgsOptionParser parser = new ArgsOptionParser(obj);
157                 parser.parse(OptionHelper.getValidCliArgs(retryCommandLineArgs, obj));
158             } catch (ConfigurationException e) {
159                 throw new RuntimeException(e);
160             }
161         }
162     }
163 
164     /**
165      * Set the retry command line args on the {@link IBuildInfo} to carry the original command
166      * across retries.
167      */
setBuildInfoRetryCommand(IBuildInfo info)168     public void setBuildInfoRetryCommand(IBuildInfo info) {
169         IInvocationResult result = new LightInvocationResult(getResult());
170         String retryCommandLineArgs = result.getCommandLineArgs();
171         new CompatibilityBuildHelper(info).setRetryCommandLineArgs(retryCommandLineArgs);
172     }
173 
174     /**
175      * Retrieve an instance of the result to retry using the instance variables referencing
176      * the build and the desired session ID. While it is faster to load this result once and
177      * store it as an instance variable, {@link IInvocationResult} objects are large, and
178      * memory is of greater concern.
179      */
getResult()180     public IInvocationResult getResult() {
181         IInvocationResult result = null;
182         try {
183             result = ResultHandler.findResult(mBuild.getResultsDir(), mSessionId);
184         } catch (FileNotFoundException e) {
185             throw new RuntimeException(e);
186         }
187         if (result == null) {
188             throw new IllegalArgumentException(String.format(
189                     "Could not find session with id %d", mSessionId));
190         }
191         return result;
192     }
193 
194     /**
195      * Populate mRetryIncludes and mRetryExcludes based on the options and the result set for
196      * this instance of RetryFilterHelper.
197      */
populateRetryFilters()198     public void populateRetryFilters() {
199         mRetryIncludes = new HashSet<>(mIncludeFilters); // reset for each population
200         mRetryExcludes = new HashSet<>(mExcludeFilters); // reset for each population
201         if (RetryType.CUSTOM.equals(mRetryType)) {
202             Set<String> customIncludes = new HashSet<>(mIncludeFilters);
203             Set<String> customExcludes = new HashSet<>(mExcludeFilters);
204             if (mSubPlan != null) {
205                 ISubPlan retrySubPlan = SubPlanHelper.getSubPlanByName(mBuild, mSubPlan);
206                 customIncludes.addAll(retrySubPlan.getIncludeFilters());
207                 customExcludes.addAll(retrySubPlan.getExcludeFilters());
208             }
209             // If includes were added, only use those includes. Also use excludes added directly
210             // or by subplan. Otherwise, default to normal retry.
211             if (!customIncludes.isEmpty()) {
212                 mRetryIncludes.clear();
213                 mRetryIncludes.addAll(customIncludes);
214                 mRetryExcludes.addAll(customExcludes);
215                 return;
216             }
217         }
218         // remove any extra filtering options
219         // TODO(aaronholden) remove non-plan includes (e.g. those in cts-vendor-interface)
220         // TODO(aaronholden) remove non-known-failure excludes
221         mModuleName = null;
222         mTestName = null;
223         mSubPlan = null;
224         populateFiltersBySubPlan();
225         populatePreviousSessionFilters();
226     }
227 
228     /* Generation of filters based on previous sessions is implemented thoroughly in SubPlanHelper,
229      * and retry filter generation is just a subset of the use cases for the subplan retry logic.
230      * Use retry type to determine which result types SubPlanHelper targets. */
populateFiltersBySubPlan()231     public void populateFiltersBySubPlan() {
232         SubPlanHelper retryPlanCreator = new SubPlanHelper();
233         retryPlanCreator.setResult(getResult());
234         if (RetryType.FAILED.equals(mRetryType)) {
235             // retry only failed tests
236             retryPlanCreator.addResultType(SubPlanHelper.FAILED);
237         } else if (RetryType.NOT_EXECUTED.equals(mRetryType)){
238             // retry only not executed tests
239             retryPlanCreator.addResultType(SubPlanHelper.NOT_EXECUTED);
240         } else {
241             // retry both failed and not executed tests
242             retryPlanCreator.addResultType(SubPlanHelper.FAILED);
243             retryPlanCreator.addResultType(SubPlanHelper.NOT_EXECUTED);
244         }
245         try {
246             ISubPlan retryPlan = retryPlanCreator.createSubPlan(mBuild);
247             mRetryIncludes.addAll(retryPlan.getIncludeFilters());
248             mRetryExcludes.addAll(retryPlan.getExcludeFilters());
249         } catch (ConfigurationException e) {
250             throw new RuntimeException ("Failed to create subplan for retry", e);
251         }
252     }
253 
254     /* Retrieves the options set via command-line on the previous session, and generates/adds
255      * filters accordingly */
populatePreviousSessionFilters()256     private void populatePreviousSessionFilters() {
257         // Temporarily store options from this instance in another instance
258         RetryFilterHelper tmpHelper = new RetryFilterHelper(mBuild, mSessionId);
259         tmpHelper.setAllOptionsFrom(this);
260         // Copy command-line args from previous session to this RetryFilterHelper's options
261         setCommandLineOptionsFor(this);
262 
263         mRetryIncludes.addAll(mIncludeFilters);
264         mRetryExcludes.addAll(mExcludeFilters);
265         if (mSubPlan != null) {
266             ISubPlan retrySubPlan = SubPlanHelper.getSubPlanByName(mBuild, mSubPlan);
267             mRetryIncludes.addAll(retrySubPlan.getIncludeFilters());
268             mRetryExcludes.addAll(retrySubPlan.getExcludeFilters());
269         }
270         if (mModuleName != null) {
271             try {
272                 List<String> modules = ModuleRepo.getModuleNamesMatching(
273                         mBuild.getTestsDir(), mModuleName);
274                 if (modules.size() == 0) {
275                     throw new IllegalArgumentException(
276                             String.format("No modules found matching %s", mModuleName));
277                 } else if (modules.size() > 1) {
278                     throw new IllegalArgumentException(String.format(
279                             "Multiple modules found matching %s:\n%s\nWhich one did you mean?\n",
280                             mModuleName, ArrayUtil.join("\n", modules)));
281                 } else {
282                     String module = modules.get(0);
283                     cleanFilters(mRetryIncludes, module);
284                     cleanFilters(mRetryExcludes, module);
285                     mRetryIncludes.add(new TestFilter(mAbiName, module, mTestName).toString());
286                 }
287             } catch (FileNotFoundException e) {
288                 throw new RuntimeException(e);
289             }
290         } else if (mTestName != null) {
291             throw new IllegalArgumentException(
292                 "Test name given without module name. Add --module <module-name>");
293         }
294 
295         // Copy options for current session back to this instance
296         setAllOptionsFrom(tmpHelper);
297     }
298 
299     /* Helper method designed to remove filters in a list not applicable to the given module */
cleanFilters(Set<String> filters, String module)300     private static void cleanFilters(Set<String> filters, String module) {
301         Set<String> cleanedFilters = new HashSet<String>();
302         for (String filter : filters) {
303             if (module.equals(TestFilter.createFrom(filter).getName())) {
304                 cleanedFilters.add(filter); // Module name matches, filter passes
305             }
306         }
307         filters.clear();
308         filters.addAll(cleanedFilters);
309     }
310 
311     /** Retrieve include filters to be applied on retry */
getIncludeFilters()312     public Set<String> getIncludeFilters() {
313         return new HashSet<>(mRetryIncludes);
314     }
315 
316     /** Retrieve exclude filters to be applied on retry */
getExcludeFilters()317     public Set<String> getExcludeFilters() {
318         return new HashSet<>(mRetryExcludes);
319     }
320 
321     /** Clears retry filters and internal storage of options, except buildInfo and session ID */
tearDown()322     public void tearDown() {
323         clearOptions();
324         mRetryIncludes = null;
325         mRetryExcludes = null;
326         // keep references to buildInfo and session ID
327     }
328 }
329