1 /*
2  * Copyright (C) 2017 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 package com.android.tradefed.testtype.python;
17 
18 import com.android.annotations.VisibleForTesting;
19 import com.android.tradefed.build.IDeviceBuildInfo;
20 import com.android.tradefed.config.GlobalConfiguration;
21 import com.android.tradefed.config.Option;
22 import com.android.tradefed.config.OptionClass;
23 import com.android.tradefed.device.DeviceNotAvailableException;
24 import com.android.tradefed.device.StubDevice;
25 import com.android.tradefed.invoker.ExecutionFiles.FilesKey;
26 import com.android.tradefed.invoker.TestInformation;
27 import com.android.tradefed.log.LogUtil.CLog;
28 import com.android.tradefed.metrics.proto.MetricMeasurement.Metric;
29 import com.android.tradefed.result.ByteArrayInputStreamSource;
30 import com.android.tradefed.result.FailureDescription;
31 import com.android.tradefed.result.FileInputStreamSource;
32 import com.android.tradefed.result.ITestInvocationListener;
33 import com.android.tradefed.result.InputStreamSource;
34 import com.android.tradefed.result.LogDataType;
35 import com.android.tradefed.result.ResultForwarder;
36 import com.android.tradefed.result.proto.TestRecordProto.FailureStatus;
37 import com.android.tradefed.testtype.IRemoteTest;
38 import com.android.tradefed.testtype.ITestFilterReceiver;
39 import com.android.tradefed.testtype.PythonUnitTestResultParser;
40 import com.android.tradefed.util.CommandResult;
41 import com.android.tradefed.util.CommandStatus;
42 import com.android.tradefed.util.FileUtil;
43 import com.android.tradefed.util.IRunUtil;
44 import com.android.tradefed.util.RunUtil;
45 import com.android.tradefed.util.SubprocessTestResultsParser;
46 
47 import com.google.common.base.Joiner;
48 
49 import java.io.File;
50 import java.io.IOException;
51 import java.util.ArrayList;
52 import java.util.Arrays;
53 import java.util.Formatter;
54 import java.util.HashMap;
55 import java.util.HashSet;
56 import java.util.LinkedHashSet;
57 import java.util.List;
58 import java.util.Set;
59 
60 /**
61  * Host test meant to run a python binary file from the Android Build system (Soong)
62  *
63  * <p>The test runner supports include-filter and exclude-filter. Note that exclude-filter works by
64  * ignoring the test result, instead of skipping the actual test. The tests specified in the
65  * exclude-filter will still be executed.
66  */
67 @OptionClass(alias = "python-host")
68 public class PythonBinaryHostTest implements IRemoteTest, ITestFilterReceiver {
69 
70     protected static final String ANDROID_SERIAL_VAR = "ANDROID_SERIAL";
71     protected static final String LD_LIBRARY_PATH = "LD_LIBRARY_PATH";
72     protected static final String PATH_VAR = "PATH";
73     protected static final long PATH_TIMEOUT_MS = 60000L;
74 
75     @VisibleForTesting static final String USE_TEST_OUTPUT_FILE_OPTION = "use-test-output-file";
76     static final String TEST_OUTPUT_FILE_FLAG = "test-output-file";
77 
78     private static final String PYTHON_LOG_STDOUT_FORMAT = "%s-stdout";
79     private static final String PYTHON_LOG_STDERR_FORMAT = "%s-stderr";
80     private static final String PYTHON_LOG_TEST_OUTPUT_FORMAT = "%s-test-output";
81 
82     private Set<String> mIncludeFilters = new LinkedHashSet<>();
83     private Set<String> mExcludeFilters = new LinkedHashSet<>();
84     private String mLdLibraryPath = null;
85 
86     @Option(name = "par-file-name", description = "The binary names inside the build info to run.")
87     private Set<String> mBinaryNames = new HashSet<>();
88 
89     @Option(
90         name = "python-binaries",
91         description = "The full path to a runnable python binary. Can be repeated."
92     )
93     private Set<File> mBinaries = new HashSet<>();
94 
95     @Option(
96         name = "test-timeout",
97         description = "Timeout for a single par file to terminate.",
98         isTimeVal = true
99     )
100     private long mTestTimeout = 20 * 1000L;
101 
102     @Option(
103             name = "inject-serial-option",
104             description = "Whether or not to pass a -s <serialnumber> option to the binary")
105     private boolean mInjectSerial = false;
106 
107     @Option(
108             name = "inject-android-serial",
109             description = "Whether or not to pass a ANDROID_SERIAL variable to the process.")
110     private boolean mInjectAndroidSerialVar = true;
111 
112     @Option(
113         name = "python-options",
114         description = "Option string to be passed to the binary when running"
115     )
116     private List<String> mTestOptions = new ArrayList<>();
117 
118     @Option(
119             name = USE_TEST_OUTPUT_FILE_OPTION,
120             description =
121                     "Whether the test should write results to the file specified via the --"
122                             + TEST_OUTPUT_FILE_FLAG
123                             + " flag instead of stderr which could contain spurious messages that "
124                             + "break result parsing. Using this option requires that the Python "
125                             + "test have the necessary logic to accept the flag and write results "
126                             + "in the expected format.")
127     private boolean mUseTestOutputFile = false;
128 
129     private TestInformation mTestInfo;
130     private IRunUtil mRunUtil;
131 
132     /** {@inheritDoc} */
133     @Override
addIncludeFilter(String filter)134     public void addIncludeFilter(String filter) {
135         mIncludeFilters.add(filter);
136     }
137 
138     /** {@inheritDoc} */
139     @Override
addExcludeFilter(String filter)140     public void addExcludeFilter(String filter) {
141         mExcludeFilters.add(filter);
142     }
143 
144     /** {@inheritDoc} */
145     @Override
addAllIncludeFilters(Set<String> filters)146     public void addAllIncludeFilters(Set<String> filters) {
147         mIncludeFilters.addAll(filters);
148     }
149 
150     /** {@inheritDoc} */
151     @Override
addAllExcludeFilters(Set<String> filters)152     public void addAllExcludeFilters(Set<String> filters) {
153         mExcludeFilters.addAll(filters);
154     }
155 
156     /** {@inheritDoc} */
157     @Override
clearIncludeFilters()158     public void clearIncludeFilters() {
159         mIncludeFilters.clear();
160     }
161 
162     /** {@inheritDoc} */
163     @Override
clearExcludeFilters()164     public void clearExcludeFilters() {
165         mExcludeFilters.clear();
166     }
167 
168     /** {@inheritDoc} */
169     @Override
getIncludeFilters()170     public Set<String> getIncludeFilters() {
171         return mIncludeFilters;
172     }
173 
174     /** {@inheritDoc} */
175     @Override
getExcludeFilters()176     public Set<String> getExcludeFilters() {
177         return mExcludeFilters;
178     }
179 
180     @Override
run(TestInformation testInfo, ITestInvocationListener listener)181     public final void run(TestInformation testInfo, ITestInvocationListener listener)
182             throws DeviceNotAvailableException {
183         mTestInfo = testInfo;
184         File testDir = mTestInfo.executionFiles().get(FilesKey.HOST_TESTS_DIRECTORY);
185         if (testDir == null || !testDir.exists()) {
186             testDir = mTestInfo.executionFiles().get(FilesKey.TESTS_DIRECTORY);
187         }
188         if (testDir != null && testDir.exists()) {
189             File libDir = new File(testDir, "lib");
190             List<String> ldLibraryPath = new ArrayList<>();
191             if (libDir.exists()) {
192                 ldLibraryPath.add(libDir.getAbsolutePath());
193             }
194 
195             File lib64Dir = new File(testDir, "lib64");
196             if (lib64Dir.exists()) {
197                 ldLibraryPath.add(lib64Dir.getAbsolutePath());
198             }
199             if (!ldLibraryPath.isEmpty()) {
200                 mLdLibraryPath = Joiner.on(":").join(ldLibraryPath);
201             }
202         }
203         List<File> pythonFilesList = findParFiles();
204         for (File pyFile : pythonFilesList) {
205             if (!pyFile.exists()) {
206                 CLog.d(
207                         "ignoring %s which doesn't look like a test file.",
208                         pyFile.getAbsolutePath());
209                 continue;
210             }
211             pyFile.setExecutable(true);
212             runSinglePythonFile(listener, testInfo, pyFile);
213         }
214     }
215 
findParFiles()216     private List<File> findParFiles() {
217         File testsDir = null;
218         if (mTestInfo.getBuildInfo() instanceof IDeviceBuildInfo) {
219             testsDir = ((IDeviceBuildInfo) mTestInfo.getBuildInfo()).getTestsDir();
220         }
221         List<File> files = new ArrayList<>();
222         for (String parFileName : mBinaryNames) {
223             File res = null;
224             // search tests dir
225             if (testsDir != null) {
226                 res = FileUtil.findFile(testsDir, parFileName);
227             }
228 
229             // TODO: is there other places to search?
230             if (res == null) {
231                 throw new RuntimeException(
232                         String.format("Couldn't find a par file %s", parFileName));
233             }
234             files.add(res);
235         }
236         files.addAll(mBinaries);
237         return files;
238     }
239 
runSinglePythonFile( ITestInvocationListener listener, TestInformation testInfo, File pyFile)240     private void runSinglePythonFile(
241             ITestInvocationListener listener, TestInformation testInfo, File pyFile) {
242         List<String> commandLine = new ArrayList<>();
243         commandLine.add(pyFile.getAbsolutePath());
244         // If we have a physical device, pass it to the python test by serial
245         if (!(mTestInfo.getDevice().getIDevice() instanceof StubDevice) && mInjectSerial) {
246             // TODO: support multi-device python tests?
247             commandLine.add("-s");
248             commandLine.add(mTestInfo.getDevice().getSerialNumber());
249         }
250 
251         if (mLdLibraryPath != null) {
252             getRunUtil().setEnvVariable(LD_LIBRARY_PATH, mLdLibraryPath);
253         }
254         if (mInjectAndroidSerialVar) {
255             getRunUtil()
256                     .setEnvVariable(ANDROID_SERIAL_VAR, mTestInfo.getDevice().getSerialNumber());
257         }
258 
259         File tempTestOutputFile = null;
260         if (mUseTestOutputFile) {
261             try {
262                 tempTestOutputFile = FileUtil.createTempFile("python-test-output", ".txt");
263             } catch (IOException e) {
264                 throw new RuntimeException(e);
265             }
266 
267             commandLine.add("--" + TEST_OUTPUT_FILE_FLAG);
268             commandLine.add(tempTestOutputFile.getAbsolutePath());
269         }
270 
271         File updatedAdb = testInfo.executionFiles().get(FilesKey.ADB_BINARY);
272         if (updatedAdb == null) {
273             String adbPath = getAdbPath();
274             // Don't check if it's the adb on the $PATH
275             if (!adbPath.equals("adb")) {
276                 updatedAdb = new File(adbPath);
277                 if (!updatedAdb.exists()) {
278                     updatedAdb = null;
279                 }
280             }
281         }
282         if (updatedAdb != null) {
283             CLog.d("Testing with adb binary at: %s", updatedAdb);
284             // If a special adb version is used, pass it to the PATH
285             CommandResult pathResult =
286                     getRunUtil()
287                             .runTimedCmd(PATH_TIMEOUT_MS, "/bin/bash", "-c", "echo $" + PATH_VAR);
288             if (!CommandStatus.SUCCESS.equals(pathResult.getStatus())) {
289                 throw new RuntimeException(
290                         String.format(
291                                 "Failed to get the $PATH. status: %s, stdout: %s, stderr: %s",
292                                 pathResult.getStatus(),
293                                 pathResult.getStdout(),
294                                 pathResult.getStderr()));
295             }
296             // Include the directory of the adb on the PATH to be used.
297             String path =
298                     String.format(
299                             "%s:%s",
300                             updatedAdb.getParentFile().getAbsolutePath(),
301                             pathResult.getStdout().trim());
302             CLog.d("Using $PATH with updated adb: %s", path);
303             getRunUtil().setEnvVariable(PATH_VAR, path);
304             // Log the version of adb seen
305             CommandResult versionRes = getRunUtil().runTimedCmd(PATH_TIMEOUT_MS, "adb", "version");
306             CLog.d("%s", versionRes.getStdout());
307             CLog.d("%s", versionRes.getStderr());
308         }
309         // Add all the other options
310         commandLine.addAll(mTestOptions);
311 
312         CommandResult result =
313                 getRunUtil().runTimedCmd(mTestTimeout, commandLine.toArray(new String[0]));
314         String runName = pyFile.getName();
315         PythonForwarder forwarder = new PythonForwarder(listener, runName);
316         if (!CommandStatus.SUCCESS.equals(result.getStatus())) {
317             CLog.e(
318                     "Something went wrong when running the python binary:\nstdout: "
319                             + "%s\nstderr:%s",
320                     result.getStdout(), result.getStderr());
321         }
322         if (result.getStdout() != null) {
323             try (InputStreamSource data =
324                     new ByteArrayInputStreamSource(result.getStdout().getBytes())) {
325                 listener.testLog(
326                         String.format(PYTHON_LOG_STDOUT_FORMAT, runName), LogDataType.TEXT, data);
327             }
328         }
329         File stderrFile = null;
330         try {
331             // Note that we still log stderr when parsing results from a test-written output file
332             // since it most likely contains useful debugging information.
333             stderrFile = FileUtil.createTempFile("python-res", ".txt");
334             FileUtil.writeToFile(result.getStderr(), stderrFile);
335             testLogFile(listener, String.format(PYTHON_LOG_STDERR_FORMAT, runName), stderrFile);
336 
337             File testOutputFile = stderrFile;
338             String testOutput = result.getStderr();
339 
340             if (mUseTestOutputFile) {
341                 testOutputFile = tempTestOutputFile;
342                 // This assumes that the output file is encoded using the same charset as the
343                 // currently configured default.
344                 testOutput = FileUtil.readStringFromFile(testOutputFile);
345                 testLogFile(
346                         listener,
347                         String.format(PYTHON_LOG_TEST_OUTPUT_FORMAT, runName),
348                         testOutputFile);
349             }
350 
351             // If it doesn't have the std output TEST_RUN_STARTED, use regular parser.
352             if (!testOutput.contains("TEST_RUN_STARTED")) {
353                 // Attempt to parse the pure python output
354                 PythonUnitTestResultParser pythonParser =
355                         new PythonUnitTestResultParser(
356                                 Arrays.asList(forwarder),
357                                 "python-run",
358                                 mIncludeFilters,
359                                 mExcludeFilters);
360                 pythonParser.processNewLines(testOutput.split("\n"));
361             } else {
362                 if (!mIncludeFilters.isEmpty() || !mExcludeFilters.isEmpty()) {
363                     throw new RuntimeException(
364                             "Non-unittest python test does not support using filters in "
365                                     + "PythonBinaryHostTest. Please use test runner "
366                                     + "ExecutableHostTest instead.");
367                 }
368                 try (SubprocessTestResultsParser parser =
369                         new SubprocessTestResultsParser(forwarder, mTestInfo.getContext())) {
370                     parser.parseFile(testOutputFile);
371                 }
372             }
373         } catch (RuntimeException e) {
374             StringBuilder message = new StringBuilder();
375             Formatter formatter = new Formatter(message);
376 
377             formatter.format(
378                     "Failed to parse the python logs: %s. Please ensure that verbosity of "
379                             + "output is high enough to be parsed.",
380                     e.getMessage());
381 
382             if (mUseTestOutputFile) {
383                 formatter.format(
384                         " Make sure that your test writes its output to the file specified "
385                                 + "by the --%s flag and that its contents (%s) are in the format "
386                                 + "expected by the test runner.",
387                         TEST_OUTPUT_FILE_FLAG,
388                         String.format(PYTHON_LOG_TEST_OUTPUT_FORMAT, runName));
389             }
390 
391             reportFailure(listener, runName, message.toString());
392             CLog.e(e);
393         } catch (IOException e) {
394             throw new RuntimeException(e);
395         } finally {
396             FileUtil.deleteFile(stderrFile);
397             FileUtil.deleteFile(tempTestOutputFile);
398         }
399     }
400 
401     @VisibleForTesting
getRunUtil()402     IRunUtil getRunUtil() {
403         if (mRunUtil == null) {
404             mRunUtil = new RunUtil();
405         }
406         return mRunUtil;
407     }
408 
409     @VisibleForTesting
getAdbPath()410     String getAdbPath() {
411         return GlobalConfiguration.getDeviceManagerInstance().getAdbPath();
412     }
413 
reportFailure( ITestInvocationListener listener, String runName, String errorMessage)414     private void reportFailure(
415             ITestInvocationListener listener, String runName, String errorMessage) {
416         listener.testRunStarted(runName, 0);
417         FailureDescription description =
418                 FailureDescription.create(errorMessage, FailureStatus.TEST_FAILURE);
419         listener.testRunFailed(description);
420         listener.testRunEnded(0L, new HashMap<String, Metric>());
421     }
422 
testLogFile(ITestInvocationListener listener, String dataName, File f)423     private static void testLogFile(ITestInvocationListener listener, String dataName, File f) {
424         try (FileInputStreamSource data = new FileInputStreamSource(f)) {
425             listener.testLog(dataName, LogDataType.TEXT, data);
426         }
427     }
428 
429     /** Result forwarder to replace the run name by the binary name. */
430     public static class PythonForwarder extends ResultForwarder {
431 
432         private String mRunName;
433 
434         /** Ctor with the run name using the binary name. */
PythonForwarder(ITestInvocationListener listener, String name)435         public PythonForwarder(ITestInvocationListener listener, String name) {
436             super(listener);
437             mRunName = name;
438         }
439 
440         @Override
testRunStarted(String runName, int testCount)441         public void testRunStarted(String runName, int testCount) {
442             // Replace run name
443             testRunStarted(runName, testCount, 0);
444         }
445 
446         @Override
testRunStarted(String runName, int testCount, int attempt)447         public void testRunStarted(String runName, int testCount, int attempt) {
448             // Replace run name
449             testRunStarted(runName, testCount, attempt, System.currentTimeMillis());
450         }
451 
452         @Override
testRunStarted(String runName, int testCount, int attempt, long startTime)453         public void testRunStarted(String runName, int testCount, int attempt, long startTime) {
454             // Replace run name
455             super.testRunStarted(mRunName, testCount, attempt, startTime);
456         }
457     }
458 }
459