1 /*
2  * Copyright (C) 2019 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 package com.android.tradefed.testtype.binary;
17 
18 import com.google.common.annotations.VisibleForTesting;
19 import com.android.tradefed.config.Option;
20 import com.android.tradefed.config.OptionCopier;
21 import com.android.tradefed.device.DeviceNotAvailableException;
22 import com.android.tradefed.invoker.TestInformation;
23 import com.android.tradefed.metrics.proto.MetricMeasurement.Metric;
24 import com.android.tradefed.result.FailureDescription;
25 import com.android.tradefed.result.ITestInvocationListener;
26 import com.android.tradefed.result.TestDescription;
27 import com.android.tradefed.testtype.IAbi;
28 import com.android.tradefed.testtype.IAbiReceiver;
29 import com.android.tradefed.testtype.IRemoteTest;
30 import com.android.tradefed.testtype.IRuntimeHintProvider;
31 import com.android.tradefed.testtype.IShardableTest;
32 import com.android.tradefed.testtype.ITestCollector;
33 import com.android.tradefed.testtype.ITestFilterReceiver;
34 import com.android.tradefed.result.error.InfraErrorIdentifier;
35 import com.android.tradefed.result.proto.TestRecordProto.FailureStatus;
36 import com.android.tradefed.util.StreamUtil;
37 
38 import java.io.File;
39 import java.io.IOException;
40 import java.lang.reflect.InvocationTargetException;
41 import java.util.ArrayList;
42 import java.util.Collection;
43 import java.util.HashMap;
44 import java.util.LinkedHashMap;
45 import java.util.LinkedHashSet;
46 import java.util.List;
47 import java.util.Map;
48 import java.util.Set;
49 
50 /** Base class for executable style of tests. For example: binaries, shell scripts. */
51 public abstract class ExecutableBaseTest
52         implements IRemoteTest,
53                 IRuntimeHintProvider,
54                 ITestCollector,
55                 IShardableTest,
56                 IAbiReceiver,
57                 ITestFilterReceiver {
58 
59     public static final String NO_BINARY_ERROR = "Binary %s does not exist.";
60 
61     @Option(
62             name = "per-binary-timeout",
63             isTimeVal = true,
64             description = "Timeout applied to each binary for their execution.")
65     private long mTimeoutPerBinaryMs = 5 * 60 * 1000L;
66 
67     @Option(name = "binary", description = "Path to the binary to be run. Can be repeated.")
68     private List<String> mBinaryPaths = new ArrayList<>();
69 
70     @Option(
71             name = "test-command-line",
72             description = "The test commands of each test names.",
73             requiredForRerun = true)
74     private Map<String, String> mTestCommands = new LinkedHashMap<>();
75 
76     @Option(
77             name = "collect-tests-only",
78             description = "Only dry-run through the tests, do not actually run them.")
79     private boolean mCollectTestsOnly = false;
80 
81     @Option(
82         name = "runtime-hint",
83         description = "The hint about the test's runtime.",
84         isTimeVal = true
85     )
86     private long mRuntimeHintMs = 60000L; // 1 minute
87 
88     private IAbi mAbi;
89     private TestInformation mTestInfo;
90     private Set<String> mIncludeFilters = new LinkedHashSet<>();
91     private Set<String> mExcludeFilters = new LinkedHashSet<>();
92 
93     /**
94      * Get test commands.
95      *
96      * @return the test commands.
97      */
98     @VisibleForTesting
getTestCommands()99     Map<String, String> getTestCommands() {
100         return mTestCommands;
101     }
102 
103     /** @return the timeout applied to each binary for their execution. */
getTimeoutPerBinaryMs()104     protected long getTimeoutPerBinaryMs() {
105         return mTimeoutPerBinaryMs;
106     }
107 
108     /** {@inheritDoc} */
109     @Override
addIncludeFilter(String filter)110     public void addIncludeFilter(String filter) {
111         mIncludeFilters.add(filter);
112     }
113 
114     /** {@inheritDoc} */
115     @Override
addExcludeFilter(String filter)116     public void addExcludeFilter(String filter) {
117         mExcludeFilters.add(filter);
118     }
119 
120     /** {@inheritDoc} */
121     @Override
addAllIncludeFilters(Set<String> filters)122     public void addAllIncludeFilters(Set<String> filters) {
123         mIncludeFilters.addAll(filters);
124     }
125 
126     /** {@inheritDoc} */
127     @Override
addAllExcludeFilters(Set<String> filters)128     public void addAllExcludeFilters(Set<String> filters) {
129         mExcludeFilters.addAll(filters);
130     }
131 
132     /** {@inheritDoc} */
133     @Override
clearIncludeFilters()134     public void clearIncludeFilters() {
135         mIncludeFilters.clear();
136     }
137 
138     /** {@inheritDoc} */
139     @Override
clearExcludeFilters()140     public void clearExcludeFilters() {
141         mExcludeFilters.clear();
142     }
143 
144     /** {@inheritDoc} */
145     @Override
getIncludeFilters()146     public Set<String> getIncludeFilters() {
147         return mIncludeFilters;
148     }
149 
150     /** {@inheritDoc} */
151     @Override
getExcludeFilters()152     public Set<String> getExcludeFilters() {
153         return mExcludeFilters;
154     }
155 
156     @Override
run(TestInformation testInfo, ITestInvocationListener listener)157     public void run(TestInformation testInfo, ITestInvocationListener listener)
158             throws DeviceNotAvailableException {
159         mTestInfo = testInfo;
160         Map<String, String> testCommands = getAllTestCommands();
161         for (String testName : testCommands.keySet()) {
162             String cmd = testCommands.get(testName);
163             String path = findBinary(cmd);
164             TestDescription description = new TestDescription(testName, testName);
165             if (shouldSkipCurrentTest(description)) continue;
166             if (path == null) {
167                 listener.testRunStarted(testName, 0);
168                 FailureDescription failure =
169                         FailureDescription.create(
170                                         String.format(NO_BINARY_ERROR, cmd),
171                                         FailureStatus.TEST_FAILURE)
172                                 .setErrorIdentifier(InfraErrorIdentifier.ARTIFACT_NOT_FOUND);
173                 listener.testRunFailed(failure);
174                 listener.testRunEnded(0L, new HashMap<String, Metric>());
175             } else {
176                 listener.testRunStarted(testName, 1);
177                 long startTimeMs = System.currentTimeMillis();
178                 listener.testStarted(description);
179                 try {
180                     if (!mCollectTestsOnly) {
181                         // Do not actually run the test if we are dry running it.
182                         runBinary(path, listener, description);
183                     }
184                 } catch (IOException e) {
185                     listener.testFailed(
186                             description, FailureDescription.create(StreamUtil.getStackTrace(e)));
187                 } finally {
188                     listener.testEnded(description, new HashMap<String, Metric>());
189                     listener.testRunEnded(
190                             System.currentTimeMillis() - startTimeMs,
191                             new HashMap<String, Metric>());
192                 }
193             }
194         }
195     }
196 
197     /**
198      * Check if current test should be skipped.
199      *
200      * @param description The test in progress.
201      * @return true if the test should be skipped.
202      */
shouldSkipCurrentTest(TestDescription description)203     private boolean shouldSkipCurrentTest(TestDescription description) {
204         // Force to skip any test not listed in include filters, or listed in exclude filters.
205         // exclude filters have highest priority.
206         String testName = description.getTestName();
207         if (mExcludeFilters.contains(testName)
208                 || mExcludeFilters.contains(description.toString())) {
209             return true;
210         }
211         if (!mIncludeFilters.isEmpty()) {
212             return !mIncludeFilters.contains(testName)
213                     && !mIncludeFilters.contains(description.toString());
214         }
215         return false;
216     }
217 
218     /**
219      * Search for the binary to be able to run it.
220      *
221      * @param binary the path of the binary or simply the binary name.
222      * @return The path to the binary, or null if not found.
223      */
findBinary(String binary)224     public abstract String findBinary(String binary) throws DeviceNotAvailableException;
225 
226     /**
227      * Actually run the binary at the given path.
228      *
229      * @param binaryPath The path of the binary.
230      * @param listener The listener where to report the results.
231      * @param description The test in progress.
232      */
runBinary( String binaryPath, ITestInvocationListener listener, TestDescription description)233     public abstract void runBinary(
234             String binaryPath, ITestInvocationListener listener, TestDescription description)
235             throws DeviceNotAvailableException, IOException;
236 
237     /** {@inheritDoc} */
238     @Override
setCollectTestsOnly(boolean shouldCollectTest)239     public final void setCollectTestsOnly(boolean shouldCollectTest) {
240         mCollectTestsOnly = shouldCollectTest;
241     }
242 
243     /** {@inheritDoc} */
244     @Override
getRuntimeHint()245     public final long getRuntimeHint() {
246         return mRuntimeHintMs;
247     }
248 
249     /** {@inheritDoc} */
250     @Override
setAbi(IAbi abi)251     public final void setAbi(IAbi abi) {
252         mAbi = abi;
253     }
254 
255     /** {@inheritDoc} */
256     @Override
getAbi()257     public IAbi getAbi() {
258         return mAbi;
259     }
260 
getTestInfo()261     TestInformation getTestInfo() {
262         return mTestInfo;
263     }
264 
265     /** {@inheritDoc} */
266     @Override
split()267     public final Collection<IRemoteTest> split() {
268         int testCount = mBinaryPaths.size() + mTestCommands.size();
269         if (testCount <= 2) {
270             return null;
271         }
272         Collection<IRemoteTest> tests = new ArrayList<>();
273         for (String path : mBinaryPaths) {
274             tests.add(getTestShard(path, null, null));
275         }
276         Map<String, String> testCommands = new LinkedHashMap<>(mTestCommands);
277         for (String testName : testCommands.keySet()) {
278             String cmd = testCommands.get(testName);
279             tests.add(getTestShard(null, testName, cmd));
280         }
281         return tests;
282     }
283 
284     /**
285      * Get a testShard of ExecutableBaseTest.
286      *
287      * @param binaryPath the binary path for ExecutableHostTest.
288      * @param testName the test name for ExecutableTargetTest.
289      * @param cmd the test command for ExecutableTargetTest.
290      * @return a shard{@link IRemoteTest} of ExecutableBaseTest{@link ExecutableBaseTest}
291      */
getTestShard(String binaryPath, String testName, String cmd)292     private IRemoteTest getTestShard(String binaryPath, String testName, String cmd) {
293         ExecutableBaseTest shard = null;
294         try {
295             shard = this.getClass().getDeclaredConstructor().newInstance();
296             OptionCopier.copyOptionsNoThrow(this, shard);
297             shard.mBinaryPaths.clear();
298             shard.mTestCommands.clear();
299             if (binaryPath != null) {
300                 // Set one binary per shard
301                 shard.mBinaryPaths.add(binaryPath);
302             } else if (testName != null && cmd != null) {
303                 // Set one test command per shard
304                 shard.mTestCommands.put(testName, cmd);
305             }
306             // Copy the filters to each shard
307             shard.mExcludeFilters.addAll(mExcludeFilters);
308             shard.mIncludeFilters.addAll(mIncludeFilters);
309         } catch (InstantiationException
310                 | IllegalAccessException
311                 | InvocationTargetException
312                 | NoSuchMethodException e) {
313             // This cannot happen because the class was already created once at that point.
314             throw new RuntimeException(
315                     String.format(
316                             "%s (%s) when attempting to create shard object",
317                             e.getClass().getSimpleName(), e.getMessage()));
318         }
319         return shard;
320     }
321 
322     /**
323      * Convert mBinaryPaths to mTestCommands for consistency.
324      *
325      * @return a Map{@link LinkedHashMap}<String, String> of testCommands.
326      */
getAllTestCommands()327     private Map<String, String> getAllTestCommands() {
328         Map<String, String> testCommands = new LinkedHashMap<>(mTestCommands);
329         for (String binary : mBinaryPaths) {
330             testCommands.put(new File(binary).getName(), binary);
331         }
332         return testCommands;
333     }
334 }
335