1 /*
2  * Copyright (C) 2017 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 package com.android.compatibility.common.tradefed.testtype.retry;
17 
18 import com.android.compatibility.common.tradefed.build.CompatibilityBuildHelper;
19 import com.android.compatibility.common.tradefed.testtype.suite.CompatibilityTestSuite;
20 import com.android.compatibility.common.tradefed.util.RetryFilterHelper;
21 import com.android.compatibility.common.tradefed.util.RetryType;
22 import com.android.tradefed.build.IBuildInfo;
23 import com.android.tradefed.config.ConfigurationException;
24 import com.android.tradefed.config.IConfiguration;
25 import com.android.tradefed.config.IConfigurationReceiver;
26 import com.android.tradefed.config.Option;
27 import com.android.tradefed.config.Option.Importance;
28 import com.android.tradefed.config.OptionClass;
29 import com.android.tradefed.config.OptionSetter;
30 import com.android.tradefed.device.DeviceNotAvailableException;
31 import com.android.tradefed.device.ITestDevice;
32 import com.android.tradefed.invoker.IInvocationContext;
33 import com.android.tradefed.invoker.TestInformation;
34 import com.android.tradefed.log.LogUtil.CLog;
35 import com.android.tradefed.result.ITestInvocationListener;
36 import com.android.tradefed.suite.checker.ISystemStatusChecker;
37 import com.android.tradefed.suite.checker.ISystemStatusCheckerReceiver;
38 import com.android.tradefed.testtype.IBuildReceiver;
39 import com.android.tradefed.testtype.IDeviceTest;
40 import com.android.tradefed.testtype.IInvocationContextReceiver;
41 import com.android.tradefed.testtype.IRemoteTest;
42 import com.android.tradefed.testtype.IShardableTest;
43 import com.android.tradefed.testtype.suite.BaseTestSuite;
44 import com.android.tradefed.util.MultiMap;
45 
46 import com.google.common.annotations.VisibleForTesting;
47 
48 import java.util.ArrayList;
49 import java.util.Collection;
50 import java.util.HashSet;
51 import java.util.List;
52 import java.util.Set;
53 
54 /**
55  * Runner that creates a {@link CompatibilityTestSuite} to re-run some previous results. Only the
56  * 'cts' plan is supported. TODO: explore other new way to build the retry (instead of relying on
57  * one massive pair of include/exclude filters)
58  *
59  * @deprecated A new retry has been implemented.
60  */
61 @Deprecated
62 @OptionClass(alias = "compatibility")
63 public class RetryFactoryTest
64         implements IRemoteTest,
65                 IDeviceTest,
66                 IBuildReceiver,
67                 ISystemStatusCheckerReceiver,
68                 IInvocationContextReceiver,
69                 IShardableTest,
70                 IConfigurationReceiver {
71 
72     /**
73      * Mirror the {@link CompatibilityTestSuite} options in order to create it.
74      */
75     public static final String RETRY_OPTION = "retry";
76     public static final String RETRY_TYPE_OPTION = "retry-type";
77 
78     @Option(name = RETRY_OPTION,
79             shortName = 'r',
80             description = "retry a previous session's failed and not executed tests.",
81             mandatory = true)
82     private Integer mRetrySessionId = null;
83 
84     @Option(name = CompatibilityTestSuite.SUBPLAN_OPTION,
85             description = "the subplan to run",
86             importance = Importance.IF_UNSET)
87     protected String mSubPlan;
88 
89     @Option(name = CompatibilityTestSuite.INCLUDE_FILTER_OPTION,
90             description = "the include module filters to apply.",
91             importance = Importance.ALWAYS)
92     protected Set<String> mIncludeFilters = new HashSet<>();
93 
94     @Option(name = CompatibilityTestSuite.EXCLUDE_FILTER_OPTION,
95             description = "the exclude module filters to apply.",
96             importance = Importance.ALWAYS)
97     protected Set<String> mExcludeFilters = new HashSet<>();
98 
99     @Option(name = CompatibilityTestSuite.ABI_OPTION,
100             shortName = 'a',
101             description = "the abi to test.",
102             importance = Importance.IF_UNSET)
103     protected String mAbiName = null;
104 
105     @Option(name = CompatibilityTestSuite.PRIMARY_ABI_RUN,
106             description =
107             "Whether to run tests with only the device primary abi. "
108                   + "This will override the --abi option.")
109     protected boolean mPrimaryAbiRun = false;
110 
111     @Option(name = CompatibilityTestSuite.MODULE_OPTION,
112             shortName = 'm',
113             description = "the test module to run.",
114             importance = Importance.IF_UNSET)
115     protected String mModuleName = null;
116 
117     @Option(name = CompatibilityTestSuite.TEST_OPTION,
118             shortName = CompatibilityTestSuite.TEST_OPTION_SHORT_NAME,
119             description = "the test run.",
120             importance = Importance.IF_UNSET)
121     protected String mTestName = null;
122 
123     @Option(
124         name = "module-arg",
125         description =
126                 "the arguments to pass to a module. The expected format is"
127                         + "\"<module-name>:<arg-name>:[<arg-key>:=]<arg-value>\"",
128         importance = Importance.ALWAYS
129     )
130     private List<String> mModuleArgs = new ArrayList<>();
131 
132     @Option(name = CompatibilityTestSuite.TEST_ARG_OPTION,
133             description = "the arguments to pass to a test. The expected format is "
134                     + "\"<test-class>:<arg-name>:[<arg-key>:=]<arg-value>\"",
135             importance = Importance.ALWAYS)
136     private List<String> mTestArgs = new ArrayList<>();
137 
138     @Option(name = RETRY_TYPE_OPTION,
139             description = "used with " + RETRY_OPTION + ", retry tests"
140             + " of a certain status. Possible values include \"failed\" and \"not_executed\".",
141             importance = Importance.IF_UNSET)
142     protected RetryType mRetryType = null;
143 
144     @Option(
145         name = BaseTestSuite.CONFIG_PATTERNS_OPTION,
146         description =
147                 "The pattern(s) of the configurations that should be loaded from a directory."
148                         + " If none is explicitly specified, .*.xml and .*.config will be used."
149                         + " Can be repeated."
150     )
151     private List<String> mConfigPatterns = new ArrayList<>();
152 
153     @Option(
154         name = "module-metadata-include-filter",
155         description =
156                 "Include modules for execution based on matching of metadata fields: for any of "
157                         + "the specified filter name and value, if a module has a metadata field "
158                         + "with the same name and value, it will be included. When both module "
159                         + "inclusion and exclusion rules are applied, inclusion rules will be "
160                         + "evaluated first. Using this together with test filter inclusion rules "
161                         + "may result in no tests to execute if the rules don't overlap."
162     )
163     private MultiMap<String, String> mModuleMetadataIncludeFilter = new MultiMap<>();
164 
165     @Option(
166         name = "module-metadata-exclude-filter",
167         description =
168                 "Exclude modules for execution based on matching of metadata fields: for any of "
169                         + "the specified filter name and value, if a module has a metadata field "
170                         + "with the same name and value, it will be excluded. When both module "
171                         + "inclusion and exclusion rules are applied, inclusion rules will be "
172                         + "evaluated first."
173     )
174     private MultiMap<String, String> mModuleMetadataExcludeFilter = new MultiMap<>();
175 
176     private List<ISystemStatusChecker> mStatusCheckers;
177     private IBuildInfo mBuildInfo;
178     private ITestDevice mDevice;
179     private IInvocationContext mContext;
180     private IConfiguration mMainConfiguration;
181 
182     @Override
setSystemStatusChecker(List<ISystemStatusChecker> systemCheckers)183     public void setSystemStatusChecker(List<ISystemStatusChecker> systemCheckers) {
184         mStatusCheckers = systemCheckers;
185     }
186 
187     @Override
setBuild(IBuildInfo buildInfo)188     public void setBuild(IBuildInfo buildInfo) {
189         mBuildInfo = buildInfo;
190     }
191 
192     @Override
setDevice(ITestDevice device)193     public void setDevice(ITestDevice device) {
194         mDevice = device;
195     }
196 
197     @Override
getDevice()198     public ITestDevice getDevice() {
199         return mDevice;
200     }
201 
202     @Override
setInvocationContext(IInvocationContext invocationContext)203     public void setInvocationContext(IInvocationContext invocationContext) {
204         mContext = invocationContext;
205     }
206 
207     @Override
setConfiguration(IConfiguration configuration)208     public void setConfiguration(IConfiguration configuration) {
209         mMainConfiguration = configuration;
210     }
211 
212     /** Build a CompatibilityTest with appropriate filters to run only the tests of interests. */
213     @Override
run(TestInformation testInfo, ITestInvocationListener listener)214     public void run(TestInformation testInfo, ITestInvocationListener listener)
215             throws DeviceNotAvailableException {
216         CompatibilityTestSuite test = loadSuite();
217         // run the retry run.
218         test.run(testInfo, listener);
219     }
220 
221     @Override
split(int shardCountHint)222     public Collection<IRemoteTest> split(int shardCountHint) {
223         try {
224             CompatibilityTestSuite test = loadSuite();
225             return test.split(shardCountHint);
226         } catch (DeviceNotAvailableException e) {
227             CLog.e("Failed to shard the retry run.");
228             CLog.e(e);
229         }
230         return null;
231     }
232 
233     /**
234      * Helper to create a {@link CompatibilityTestSuite} from previous results.
235      */
loadSuite()236     private CompatibilityTestSuite loadSuite() throws DeviceNotAvailableException {
237         // Create a compatibility test and set it to run only what we want.
238         CompatibilityTestSuite test = createTest();
239 
240         CompatibilityBuildHelper buildHelper = new CompatibilityBuildHelper(mBuildInfo);
241         // Create the helper with all the options needed.
242         RetryFilterHelper helper = createFilterHelper(buildHelper);
243         // TODO: we have access to the original command line, we should accommodate more re-run
244         // scenario like when the original cts.xml config was not used.
245         helper.validateBuildFingerprint(mDevice);
246         // ResultReporter when creating the xml will use the retry command line
247         helper.setBuildInfoRetryCommand(mBuildInfo);
248         helper.setCommandLineOptionsFor(test);
249         helper.setCommandLineOptionsFor(this);
250         helper.populateRetryFilters();
251 
252         try {
253             OptionSetter setter = new OptionSetter(test);
254             for (String moduleArg : mModuleArgs) {
255                 setter.setOptionValue("compatibility:module-arg", moduleArg);
256             }
257             for (String testArg : mTestArgs) {
258                 setter.setOptionValue("compatibility:test-arg", testArg);
259             }
260         } catch (ConfigurationException e) {
261             throw new RuntimeException(e);
262         }
263 
264         test.setIncludeFilter(helper.getIncludeFilters());
265         test.setExcludeFilter(helper.getExcludeFilters());
266         test.setDevice(mDevice);
267         test.setBuild(mBuildInfo);
268         test.setAbiName(mAbiName);
269         test.setPrimaryAbiRun(mPrimaryAbiRun);
270         test.setSystemStatusChecker(mStatusCheckers);
271         test.setInvocationContext(mContext);
272         test.setConfiguration(mMainConfiguration);
273         test.addConfigPatterns(mConfigPatterns);
274         test.addModuleMetadataIncludeFilters(mModuleMetadataIncludeFilter);
275         test.addModuleMetadataExcludeFilters(mModuleMetadataExcludeFilter);
276         // reset the retry id - Ensure that retry of retry does not throw
277         test.resetRetryId();
278         // clean the helper
279         helper.tearDown();
280         return test;
281     }
282 
283     /**
284      * @return a {@link RetryFilterHelper} created from the attributes of this object.
285      */
createFilterHelper(CompatibilityBuildHelper buildHelper)286     protected RetryFilterHelper createFilterHelper(CompatibilityBuildHelper buildHelper) {
287         return new RetryFilterHelper(buildHelper, mRetrySessionId, mSubPlan, mIncludeFilters,
288                 mExcludeFilters, mAbiName, mModuleName, mTestName, mRetryType);
289     }
290 
291     @VisibleForTesting
createTest()292     CompatibilityTestSuite createTest() {
293         return new CompatibilityTestSuite();
294     }
295 
296     /**
297      * @return the ID of the session to be retried.
298      */
getRetrySessionId()299     protected Integer getRetrySessionId() {
300         return mRetrySessionId;
301     }
302 }
303