1 /*
2  * Copyright (C) 2019 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 package com.android.uicd.tests;
17 
18 import com.android.tradefed.build.IBuildInfo;
19 import com.android.tradefed.config.Option;
20 import com.android.tradefed.config.Option.Importance;
21 import com.android.tradefed.device.DeviceNotAvailableException;
22 import com.android.tradefed.device.ITestDevice;
23 import com.android.tradefed.invoker.TestInformation;
24 import com.android.tradefed.log.LogUtil.CLog;
25 import com.android.tradefed.result.FileInputStreamSource;
26 import com.android.tradefed.result.ITestInvocationListener;
27 import com.android.tradefed.result.InputStreamSource;
28 import com.android.tradefed.result.LogDataType;
29 import com.android.tradefed.result.TestDescription;
30 import com.android.tradefed.testtype.IRemoteTest;
31 import com.android.tradefed.util.CommandResult;
32 import com.android.tradefed.util.FileUtil;
33 import com.android.tradefed.util.MultiMap;
34 import com.android.tradefed.util.RunUtil;
35 
36 import org.json.JSONArray;
37 import org.json.JSONException;
38 import org.json.JSONObject;
39 
40 import java.io.File;
41 import java.io.IOException;
42 import java.nio.file.Paths;
43 import java.util.ArrayList;
44 import java.util.Collection;
45 import java.util.HashMap;
46 import java.util.List;
47 import java.util.Map;
48 import java.util.UUID;
49 
50 /**
51  * The class enables user to run their pre-recorded UICD tests on tradefed. Go to
52  * https://github.com/google/android-uiconductor/releases/tag/v0.1.1 to download the uicd_cli.tar.gz
53  * and extract the jar and apks required for the tests. Please look at the sample xmls in
54  * res/config/uicd to configure your tests.
55  */
56 public class UiConductorTest implements IRemoteTest {
57 
58     @Option(
59         name = "uicd-cli-jar",
60         description = "The cli jar that runs the user provided tests in commandline",
61         importance = Importance.IF_UNSET
62     )
63     private File cliJar;
64 
65     @Option(
66         name = "commandline-action-executable",
67         description =
68                 "the filesystem path of the binaries that are ran through command line actions on UICD. Can be repeated.",
69         importance = Importance.IF_UNSET
70     )
71     private Collection<File> binaries = new ArrayList<File>();
72 
73     @Option(
74         name = "global-variables",
75         description = "Global variable (uicd_key1=value1,uicd_key2=value2)",
76         importance = Importance.ALWAYS
77     )
78     private MultiMap<String, String> globalVariables = new MultiMap<>();
79 
80     @Option(
81         name = "play-mode",
82         description = "Play Mode (SINGLE|MULTIDEVICE|PLAYALL).",
83         importance = Importance.ALWAYS
84     )
85     private String playMode = "SINGLE";
86 
87     @Option(name = "test-name", description = "Name of the test.", importance = Importance.ALWAYS)
88     private String testName = "Your test results are here";
89 
90     // Same key can have multiple test files because global-variables can be referenced using the
91     // that particular key and shared across different tests.
92     // Refer res/config/uicd/uiconductor-globalvariable-sample.xml for more information.
93     @Option(
94         name = "uicd-test",
95         description =
96                 "the filesystem path of the json test files or directory of multiple json test files that needs to be run on devices. Can be repeated.",
97         importance = Importance.IF_UNSET
98     )
99     private MultiMap<String, File> uicdTest = new MultiMap<>();
100 
101     @Option(
102         name = "test-timeout",
103         description = "Time out for each test.",
104         importance = Importance.IF_UNSET
105     )
106     private int testTimeout = 1800000;
107 
108     private static final String BINARY_RELATIVE_PATH = "binary";
109 
110     private static final String OUTPUT_RELATIVE_PATH = "output";
111 
112     private static final String TESTS_RELATIVE_PATH = "tests";
113 
114     private static final String RESULTS_RELATIVE_PATH = "result";
115 
116     private static final String OPTION_SYMBOL = "-";
117     private static final String INPUT_OPTION_SHORT_NAME = "i";
118     private static final String OUTPUT_OPTION_SHORT_NAME = "o";
119     private static final String DEVICES_OPTION_SHORT_NAME = "d";
120     private static final String MODE_OPTION_SHORT_NAME = "m";
121     private static final String GLOBAL_VARIABLE_OPTION_SHORT_NAME = "g";
122 
123     private static final String CHILDRENRESULT_ATTRIBUTE = "childrenResult";
124     private static final String PLAYSTATUS_ATTRIBUTE = "playStatus";
125     private static final String VALIDATIONDETAILS_ATTRIBUTE = "validationDetails";
126 
127     private static final String EXECUTABLE = "u+x";
128 
129     private static String baseFilePath = System.getenv("HOME") + "/tmp/uicd-on-tf";
130 
131     Map<ITestDevice, IBuildInfo> deviceInfos;
132 
133     @Override
run(TestInformation testInfo, ITestInvocationListener listener)134     public void run(TestInformation testInfo, ITestInvocationListener listener)
135             throws DeviceNotAvailableException {
136         deviceInfos = testInfo.getContext().getDeviceBuildMap();
137         CLog.i("Starting the UIConductor tests:\n");
138         String runId = UUID.randomUUID().toString();
139         baseFilePath = Paths.get(baseFilePath, runId).toString();
140         String jarFileDir = Paths.get(baseFilePath, BINARY_RELATIVE_PATH).toString();
141         String testFilesDir = Paths.get(baseFilePath, TESTS_RELATIVE_PATH).toString();
142         String binaryFilesDir = Paths.get(baseFilePath).toString();
143         File jarFile;
144         MultiMap<String, File> copiedTestFileMap = new MultiMap<>();
145         if (cliJar == null || !cliJar.exists()) {
146             CLog.e("Unable to fetch provided binary.\n");
147             return;
148         }
149         try {
150             jarFile = copyFile(cliJar.getAbsolutePath(), jarFileDir);
151             FileUtil.chmod(jarFile, EXECUTABLE);
152 
153             for (Map.Entry<String, File> testFileOrDirEntry : uicdTest.entries()) {
154                 copiedTestFileMap.putAll(
155                         copyFile(
156                                 testFileOrDirEntry.getKey(),
157                                 testFileOrDirEntry.getValue().getAbsolutePath(),
158                                 testFilesDir));
159             }
160 
161             for (File binaryFile : binaries) {
162                 File binary = copyFile(binaryFile.getAbsolutePath(), binaryFilesDir);
163                 FileUtil.chmod(binary, EXECUTABLE);
164             }
165         } catch (IOException ex) {
166             throw new RuntimeException(ex);
167         }
168 
169         RunUtil rUtil = new RunUtil();
170         rUtil.setWorkingDir(new File(baseFilePath));
171         long runStartTime = System.currentTimeMillis();
172         listener.testRunStarted(testName, copiedTestFileMap.values().size());
173         for (Map.Entry<String, File> testFileEntry : copiedTestFileMap.entries()) {
174             runTest(
175                     listener,
176                     rUtil,
177                     jarFile,
178                     testFileEntry.getKey(),
179                     testFileEntry.getValue().getName());
180         }
181 
182         listener.testRunEnded(
183                 System.currentTimeMillis() - runStartTime, new HashMap<String, String>());
184         FileUtil.recursiveDelete(new File(baseFilePath));
185         CLog.i("Finishing the ui conductor tests\n");
186     }
187 
runTest( ITestInvocationListener listener, RunUtil rUtil, File jarFile, String key, String testFileName)188     public void runTest(
189             ITestInvocationListener listener,
190             RunUtil rUtil,
191             File jarFile,
192             String key,
193             String testFileName) {
194         TestDescription testDesc =
195                 new TestDescription(this.getClass().getSimpleName(), testFileName);
196         listener.testStarted(testDesc, System.currentTimeMillis());
197 
198         String testId = UUID.randomUUID().toString();
199         CommandResult cmndRes =
200                 rUtil.runTimedCmd(testTimeout, getCommand(jarFile, testFileName, testId, key));
201         logInfo(testId, "STD", cmndRes.getStdout());
202         logInfo(testId, "ERR", cmndRes.getStderr());
203 
204         File resultsFile =
205                 new File(
206                         Paths.get(
207                                         baseFilePath,
208                                         OUTPUT_RELATIVE_PATH,
209                                         testId,
210                                         RESULTS_RELATIVE_PATH,
211                                         "action_execution_result")
212                                 .toString());
213 
214         if (resultsFile.exists()) {
215             try {
216                 String content = FileUtil.readStringFromFile(resultsFile);
217                 JSONObject result = new JSONObject(content);
218                 List<String> errors = new ArrayList<>();
219                 errors = parseResult(errors, result);
220                 if (!errors.isEmpty()) {
221                     listener.testFailed(testDesc, errors.get(0));
222                     CLog.i("Test %s failed due to following errors: \n", testDesc.getTestName());
223                     for (String error : errors) {
224                         CLog.i(error + "\n");
225                     }
226                 }
227             } catch (IOException | JSONException e) {
228                 CLog.e(e);
229             }
230             String testResultFileName = testFileName + "_action_execution_result";
231             try (InputStreamSource iSSource = new FileInputStreamSource(resultsFile)) {
232                 listener.testLog(testResultFileName, LogDataType.TEXT, iSSource);
233             }
234         }
235         listener.testEnded(testDesc, System.currentTimeMillis(), new HashMap<String, String>());
236     }
237 
logInfo(String testId, String cmdOutputType, String content)238     private void logInfo(String testId, String cmdOutputType, String content) {
239         CLog.i(
240                 "==========================="
241                         + cmdOutputType
242                         + " logs for "
243                         + testId
244                         + " starts===========================\n");
245         CLog.i(content);
246         CLog.i(
247                 "==========================="
248                         + cmdOutputType
249                         + " logs for "
250                         + testId
251                         + " ends===========================\n");
252     }
253 
parseResult(List<String> errors, JSONObject result)254     private List<String> parseResult(List<String> errors, JSONObject result) throws JSONException {
255 
256         if (result != null) {
257             if (result.has(CHILDRENRESULT_ATTRIBUTE)) {
258                 JSONArray childResults = result.getJSONArray(CHILDRENRESULT_ATTRIBUTE);
259                 for (int i = 0; i < childResults.length(); i++) {
260                     errors = parseResult(errors, childResults.getJSONObject(i));
261                 }
262             }
263 
264             if (result.has(PLAYSTATUS_ATTRIBUTE)
265                     && result.getString(PLAYSTATUS_ATTRIBUTE).equalsIgnoreCase("FAIL")) {
266                 if (result.has(VALIDATIONDETAILS_ATTRIBUTE)) {
267                     errors.add(result.getString(VALIDATIONDETAILS_ATTRIBUTE));
268                 }
269             }
270         }
271         return errors;
272     }
273 
copyFile(String srcFilePath, String destDirPath)274     private File copyFile(String srcFilePath, String destDirPath) throws IOException {
275         File srcFile = new File(srcFilePath);
276         File destDir = new File(destDirPath);
277         if (srcFile.isDirectory()) {
278             for (File file : srcFile.listFiles()) {
279                 copyFile(file.getAbsolutePath(), Paths.get(destDirPath, file.getName()).toString());
280             }
281         }
282         if (!destDir.isDirectory() && !destDir.mkdirs()) {
283             throw new IOException(
284                     String.format("Could not create directory %s", destDir.getAbsolutePath()));
285         }
286         File destFile = new File(Paths.get(destDir.toString(), srcFile.getName()).toString());
287         FileUtil.copyFile(srcFile, destFile);
288         return destFile;
289     }
290 
291     // copy file to destDirPath while maintaining a map of key that refers to that src file
copyFile(String key, String srcFilePath, String destDirPath)292     private MultiMap<String, File> copyFile(String key, String srcFilePath, String destDirPath)
293             throws IOException {
294         MultiMap<String, File> copiedTestFileMap = new MultiMap<>();
295         File srcFile = new File(srcFilePath);
296         File destDir = new File(destDirPath);
297         if (srcFile.isDirectory()) {
298             for (File file : srcFile.listFiles()) {
299                 copiedTestFileMap.putAll(
300                         copyFile(
301                                 key,
302                                 file.getAbsolutePath(),
303                                 Paths.get(destDirPath, file.getName()).toString()));
304             }
305         }
306         if (!destDir.isDirectory() && !destDir.mkdirs()) {
307             throw new IOException(
308                     String.format("Could not create directory %s", destDir.getAbsolutePath()));
309         }
310         if (srcFile.isFile()) {
311             File destFile = new File(Paths.get(destDir.toString(), srcFile.getName()).toString());
312             FileUtil.copyFile(srcFile, destFile);
313             copiedTestFileMap.put(key, destFile);
314         }
315         return copiedTestFileMap;
316     }
317 
getTestFilesArgsForUicdBin(String testFilesDir, String filename)318     private String getTestFilesArgsForUicdBin(String testFilesDir, String filename) {
319         return (!testFilesDir.isEmpty() && !filename.isEmpty())
320                 ? Paths.get(testFilesDir, filename).toString()
321                 : "";
322     }
323 
getOutFilesArgsForUicdBin(String outFilesDir)324     private String getOutFilesArgsForUicdBin(String outFilesDir) {
325         return !outFilesDir.isEmpty() ? outFilesDir : "";
326     }
327 
getPlaymodeArgForUicdBin()328     private String getPlaymodeArgForUicdBin() {
329         return !playMode.isEmpty() ? playMode : "";
330     }
331 
getDevIdsArgsForUicdBin()332     private String getDevIdsArgsForUicdBin() {
333         List<String> devIds = new ArrayList<>();
334         for (ITestDevice device : deviceInfos.keySet()) {
335             devIds.add(device.getSerialNumber());
336         }
337         return String.join(",", devIds);
338     }
339 
getCommand(File jarFile, String testFileName, String testId, String key)340     private String[] getCommand(File jarFile, String testFileName, String testId, String key) {
341         List<String> command = new ArrayList<>();
342         command.add("java");
343         command.add("-jar");
344         command.add(jarFile.getAbsolutePath());
345         if (!getTestFilesArgsForUicdBin(TESTS_RELATIVE_PATH, testFileName).isEmpty()) {
346             command.add(OPTION_SYMBOL + INPUT_OPTION_SHORT_NAME);
347             command.add(getTestFilesArgsForUicdBin(TESTS_RELATIVE_PATH, testFileName));
348         }
349         if (!getOutFilesArgsForUicdBin(OUTPUT_RELATIVE_PATH + "/" + testId).isEmpty()) {
350             command.add(OPTION_SYMBOL + OUTPUT_OPTION_SHORT_NAME);
351             command.add(getOutFilesArgsForUicdBin(OUTPUT_RELATIVE_PATH + "/" + testId));
352         }
353         if (!getPlaymodeArgForUicdBin().isEmpty()) {
354             command.add(OPTION_SYMBOL + MODE_OPTION_SHORT_NAME);
355             command.add(getPlaymodeArgForUicdBin());
356         }
357         if (!getDevIdsArgsForUicdBin().isEmpty()) {
358             command.add(OPTION_SYMBOL + DEVICES_OPTION_SHORT_NAME);
359             command.add(getDevIdsArgsForUicdBin());
360         }
361         if (globalVariables.containsKey(key)) {
362             command.add(OPTION_SYMBOL + GLOBAL_VARIABLE_OPTION_SHORT_NAME);
363             command.add(String.join(",", globalVariables.get(key)));
364         }
365         return command.toArray(new String[] {});
366     }
367 }
368