1 /* 2 * Copyright (C) 2018 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package com.android.tradefed.testtype; 17 18 import static org.junit.Assert.assertEquals; 19 import static org.junit.Assert.assertFalse; 20 import static org.junit.Assert.assertNotEquals; 21 import static org.junit.Assert.assertTrue; 22 import static org.junit.Assert.fail; 23 import static org.mockito.ArgumentMatchers.any; 24 import static org.mockito.Mockito.when; 25 26 import com.android.tradefed.build.BuildInfoKey; 27 import com.android.tradefed.build.DeviceBuildInfo; 28 import com.android.tradefed.config.ConfigurationException; 29 import com.android.tradefed.config.OptionSetter; 30 import com.android.tradefed.device.DeviceNotAvailableException; 31 import com.android.tradefed.invoker.TestInformation; 32 import com.android.tradefed.result.ITestInvocationListener; 33 import com.android.tradefed.util.CommandResult; 34 import com.android.tradefed.util.CommandStatus; 35 import com.android.tradefed.util.FakeShellOutputReceiver; 36 import com.android.tradefed.util.FileUtil; 37 38 import org.junit.After; 39 import org.junit.Before; 40 import org.junit.Test; 41 import org.junit.runner.RunWith; 42 import org.junit.runners.JUnit4; 43 import org.mockito.Mockito; 44 45 import java.io.File; 46 import java.io.IOException; 47 import java.nio.charset.StandardCharsets; 48 import java.nio.file.Path; 49 import java.nio.file.Paths; 50 51 /** Unit tests for {@link HostGTest}. */ 52 @RunWith(JUnit4.class) 53 public class HostGTestTest { 54 private File mTestsDir; 55 private HostGTest mHostGTest; 56 private TestInformation mTestInfo; 57 private ITestInvocationListener mMockInvocationListener; 58 private FakeShellOutputReceiver mFakeReceiver; 59 private OptionSetter mSetter; 60 61 /** Helper to initialize the object or folder for unittest need. */ 62 @Before setUp()63 public void setUp() throws Exception { 64 mTestsDir = FileUtil.createTempDir("test_folder_for_unittest"); 65 mMockInvocationListener = Mockito.mock(ITestInvocationListener.class); 66 mFakeReceiver = new FakeShellOutputReceiver(); 67 68 mHostGTest = Mockito.spy(new HostGTest()); 69 GTestXmlResultParser mockXmlParser = Mockito.mock(GTestXmlResultParser.class); 70 when(mHostGTest.createXmlParser(any(), any())).thenReturn(mockXmlParser); 71 when(mHostGTest.createResultParser(any(), any())).thenReturn(mFakeReceiver); 72 73 mSetter = new OptionSetter(mHostGTest); 74 75 mTestInfo = TestInformation.newBuilder().build(); 76 } 77 78 @After afterMethod()79 public void afterMethod() { 80 FileUtil.recursiveDelete(mTestsDir); 81 } 82 83 /** 84 * Helper to create a executable script for use in these unit tests. 85 * 86 * <p>This method will create a executable file for unittest. This executable file is shell 87 * script file. It will output all arguments to a file like "file.called" when it has been 88 * called. It will also output to both stdout and stderr. Unit tests can check the .called file 89 * or the test output (or both) to determine test success or not. 90 * 91 * @param folderName The path where to create. 92 * @param fileName The file name you want to create. 93 * @return The file path of "file.called", it is used to check if the file has been called 94 * correctly or not. 95 */ createTestScript(String folderName, String fileName)96 private File createTestScript(String folderName, String fileName) throws IOException { 97 final Path outputPath = Paths.get(folderName, fileName + ".called"); 98 final String script = 99 String.format( 100 "echo \"$@\" > %s; echo \"stdout: %s\"; echo \"stderr: %s\" >&2", 101 outputPath, fileName, fileName); 102 103 final Path scriptPath = Paths.get(folderName, fileName); 104 createExecutableFile(scriptPath, script); 105 106 return outputPath.toFile(); 107 } 108 109 /** 110 * Helper to create an executable file with the given contents. 111 * 112 * @param outPath The path where to create. 113 * @param contents Contents to write to the file 114 */ createExecutableFile(Path outPath, String contents)115 private void createExecutableFile(Path outPath, String contents) throws IOException { 116 final File outFile = outPath.toFile(); 117 FileUtil.writeToFile(contents, outFile); 118 outFile.setExecutable(true); 119 } 120 121 /** 122 * Helper to create a sub folder in mTestsDir. 123 * 124 * @param folderName The path where to create. 125 * @return Sub folder File. 126 */ createSubFolder(String folderName)127 private File createSubFolder(String folderName) throws IOException { 128 return FileUtil.createTempDir(folderName, mTestsDir); 129 } 130 131 /** Test the executeHostCommand method. */ 132 @Test testExecuteHostCommand_success()133 public void testExecuteHostCommand_success() { 134 CommandResult lsResult = mHostGTest.executeHostCommand("ls"); 135 assertNotEquals("", lsResult.getStdout()); 136 assertEquals(CommandStatus.SUCCESS, lsResult.getStatus()); 137 } 138 139 /** Test the executeHostCommand method. */ 140 @Test testExecuteHostCommand_fail()141 public void testExecuteHostCommand_fail() { 142 CommandResult cmdResult = mHostGTest.executeHostCommand(""); 143 assertNotEquals(CommandStatus.SUCCESS, cmdResult.getStatus()); 144 } 145 146 /** Test the loadFilter method. */ 147 @Test testLoadFilter()148 public void testLoadFilter() throws ConfigurationException, IOException { 149 String moduleName = "hello_world_test"; 150 String testFilterKey = "presubmit"; 151 OptionSetter setter = new OptionSetter(mHostGTest); 152 setter.setOptionValue("test-filter-key", testFilterKey); 153 154 String filter = "LayerTransactionTest.*:LayerUpdateTest.*"; 155 String json_content = 156 "{\n" 157 + " \"" 158 + testFilterKey 159 + "\": {\n" 160 + " \"filter\": \"" 161 + filter 162 + "\"\n" 163 + " }\n" 164 + "}\n"; 165 Path path = Paths.get(mTestsDir.getAbsolutePath(), moduleName + GTestBase.FILTER_EXTENSION); 166 File filterFile = path.toFile(); 167 filterFile.createNewFile(); 168 filterFile.setReadable(true); 169 FileUtil.writeToFile(json_content, filterFile); 170 assertEquals( 171 mHostGTest.loadFilter(filterFile.getParent() + File.separator + moduleName), 172 filter); 173 } 174 175 /** Test runTest method. */ 176 @Test testRunTest()177 public void testRunTest() 178 throws ConfigurationException, IOException, DeviceNotAvailableException { 179 String moduleName = "hello_world_test"; 180 String dirPath = mTestsDir.getAbsolutePath(); 181 File cmd1 = createTestScript(dirPath, "cmd1"); 182 File cmd2 = createTestScript(dirPath, "cmd2"); 183 File cmd3 = createTestScript(dirPath, "cmd3"); 184 File cmd4 = createTestScript(dirPath, "cmd4"); 185 186 mSetter.setOptionValue("before-test-cmd", dirPath + File.separator + "cmd1"); 187 mSetter.setOptionValue("before-test-cmd", dirPath + File.separator + "cmd2"); 188 mSetter.setOptionValue("after-test-cmd", dirPath + File.separator + "cmd3"); 189 mSetter.setOptionValue("after-test-cmd", dirPath + File.separator + "cmd4"); 190 mSetter.setOptionValue("module-name", moduleName); 191 192 File hostLinkedFolder = createSubFolder("hosttestcases"); 193 createTestScript(hostLinkedFolder.getAbsolutePath(), moduleName); 194 195 DeviceBuildInfo buildInfo = new DeviceBuildInfo(); 196 buildInfo.setFile(BuildInfoKey.BuildInfoFileKey.HOST_LINKED_DIR, hostLinkedFolder, "0.0"); 197 mHostGTest.setBuild(buildInfo); 198 199 mHostGTest.run(mTestInfo, mMockInvocationListener); 200 201 assertTrue(cmd1.exists()); 202 assertTrue(cmd2.exists()); 203 assertTrue(cmd3.exists()); 204 assertTrue(cmd4.exists()); 205 assertNotEquals(0, mFakeReceiver.getReceivedOutput().length); 206 } 207 208 /** Test the run method for host linked folder is set. */ 209 @Test testRun_priority_get_testcase_from_hostlinked_folder()210 public void testRun_priority_get_testcase_from_hostlinked_folder() 211 throws IOException, ConfigurationException, DeviceNotAvailableException { 212 String moduleName = "hello_world_test"; 213 String hostLinkedFolderName = "hosttestcases"; 214 File hostLinkedFolder = createSubFolder(hostLinkedFolderName); 215 File hostTestcaseExecutedCheckFile = 216 createTestScript(hostLinkedFolder.getAbsolutePath(), moduleName); 217 218 String testFolderName = "testcases"; 219 File testcasesFolder = createSubFolder(testFolderName); 220 File testfolderTestcaseCheckExecuted = 221 createTestScript(testcasesFolder.getAbsolutePath(), moduleName); 222 223 mSetter.setOptionValue("module-name", moduleName); 224 DeviceBuildInfo buildInfo = new DeviceBuildInfo(); 225 buildInfo.setFile(BuildInfoKey.BuildInfoFileKey.HOST_LINKED_DIR, hostLinkedFolder, "0.0"); 226 buildInfo.setTestsDir(testcasesFolder, "0.0"); 227 mHostGTest.setBuild(buildInfo); 228 229 mHostGTest.run(mTestInfo, mMockInvocationListener); 230 assertTrue(hostTestcaseExecutedCheckFile.exists()); 231 assertFalse(testfolderTestcaseCheckExecuted.exists()); 232 assertNotEquals(0, mFakeReceiver.getReceivedOutput().length); 233 } 234 235 /** Test the run method for host linked folder is not set. */ 236 @Test testRun_get_testcase_from_testcases_folder_if_no_hostlinked_dir_set()237 public void testRun_get_testcase_from_testcases_folder_if_no_hostlinked_dir_set() 238 throws IOException, ConfigurationException, DeviceNotAvailableException { 239 String moduleName = "hello_world_test"; 240 String hostLinkedFolderName = "hosttestcases"; 241 File hostLinkedFolder = createSubFolder(hostLinkedFolderName); 242 File hostTestcaseExecutedCheckFile = 243 createTestScript(hostLinkedFolder.getAbsolutePath(), moduleName); 244 245 String testFolderName = "testcases"; 246 File testcasesFolder = createSubFolder(testFolderName); 247 File testfolderTestcaseCheckExecuted = 248 createTestScript(testcasesFolder.getAbsolutePath(), moduleName); 249 250 mSetter.setOptionValue("module-name", moduleName); 251 DeviceBuildInfo buildInfo = new DeviceBuildInfo(); 252 buildInfo.setTestsDir(testcasesFolder, "0.0"); 253 mHostGTest.setBuild(buildInfo); 254 255 mHostGTest.run(mTestInfo, mMockInvocationListener); 256 assertFalse(hostTestcaseExecutedCheckFile.exists()); 257 assertTrue(testfolderTestcaseCheckExecuted.exists()); 258 assertNotEquals(0, mFakeReceiver.getReceivedOutput().length); 259 } 260 261 /** Test can't find testcase. */ 262 @Test(expected = RuntimeException.class) testRun_can_not_find_testcase()263 public void testRun_can_not_find_testcase() 264 throws ConfigurationException, DeviceNotAvailableException { 265 String moduleName = "hello_world_test"; 266 mSetter.setOptionValue("module-name", moduleName); 267 DeviceBuildInfo buildInfo = new DeviceBuildInfo(); 268 mHostGTest.setBuild(buildInfo); 269 270 mHostGTest.run(mTestInfo, mMockInvocationListener); 271 assertNotEquals(0, mFakeReceiver.getReceivedOutput().length); 272 } 273 274 /** Test the run method for a binary with a suffix. */ 275 @Test testRun_withSuffix()276 public void testRun_withSuffix() throws Exception { 277 String moduleName = "hello_world_test"; 278 String hostLinkedFolderName = "hosttestcases"; 279 File hostLinkedFolder = createSubFolder(hostLinkedFolderName); 280 // The actual execution file has a suffix 281 File hostTestcaseExecutedCheckFile = 282 createTestScript(hostLinkedFolder.getAbsolutePath(), moduleName + "32"); 283 284 String testFolderName = "testcases"; 285 File testcasesFolder = createSubFolder(testFolderName); 286 File testfolderTestcaseCheckExecuted = 287 createTestScript(testcasesFolder.getAbsolutePath(), moduleName + "32"); 288 289 mSetter.setOptionValue("module-name", moduleName); 290 DeviceBuildInfo buildInfo = new DeviceBuildInfo(); 291 buildInfo.setFile(BuildInfoKey.BuildInfoFileKey.HOST_LINKED_DIR, hostLinkedFolder, "0.0"); 292 buildInfo.setTestsDir(testcasesFolder, "0.0"); 293 mHostGTest.setBuild(buildInfo); 294 295 mHostGTest.run(mTestInfo, mMockInvocationListener); 296 assertTrue(hostTestcaseExecutedCheckFile.exists()); 297 assertFalse(testfolderTestcaseCheckExecuted.exists()); 298 assertNotEquals(0, mFakeReceiver.getReceivedOutput().length); 299 } 300 301 /* Test that some command in the test run fails, an exception is thrown and the run stops. */ 302 @Test testBeforeCmdError()303 public void testBeforeCmdError() throws Exception { 304 String moduleName = "hello_world_test"; 305 String hostLinkedFolderName = "hosttestcases"; 306 File hostLinkedFolder = createSubFolder(hostLinkedFolderName); 307 File hostTestcaseExecutedCheckFile = 308 createTestScript(hostLinkedFolder.getAbsolutePath(), moduleName); 309 310 String testDir = mTestsDir.getAbsolutePath(); 311 Path errorScriptPath = Paths.get(testDir, "bad_cmd"); 312 createExecutableFile(errorScriptPath, "exit 1"); 313 314 DeviceBuildInfo buildInfo = new DeviceBuildInfo(); 315 buildInfo.setFile(BuildInfoKey.BuildInfoFileKey.HOST_LINKED_DIR, hostLinkedFolder, "0.0"); 316 mHostGTest.setBuild(buildInfo); 317 318 mSetter.setOptionValue("module-name", moduleName); 319 mSetter.setOptionValue("before-test-cmd", errorScriptPath.toString()); 320 321 try { 322 mHostGTest.run(mTestInfo, mMockInvocationListener); 323 fail("Didn't throw RuntimeException for before cmd with non-zero exit code"); 324 } catch (RuntimeException e) { 325 // Expected exception 326 } 327 assertFalse(hostTestcaseExecutedCheckFile.exists()); 328 } 329 330 /* Test that if the test module exits with code 1, no exception is thrown and the run completes 331 * normally. */ 332 @Test testTestFailureHandledCorrectly()333 public void testTestFailureHandledCorrectly() throws Exception { 334 String moduleName = "hello_world_test"; 335 String hostLinkedFolderName = "hosttestcases"; 336 File hostLinkedFolder = createSubFolder(hostLinkedFolderName); 337 Path errorScriptPath = Paths.get(hostLinkedFolder.getAbsolutePath(), moduleName); 338 createExecutableFile(errorScriptPath, "echo 'TEST FAILED'; exit 1"); 339 340 DeviceBuildInfo buildInfo = new DeviceBuildInfo(); 341 buildInfo.setFile(BuildInfoKey.BuildInfoFileKey.HOST_LINKED_DIR, hostLinkedFolder, "0.0"); 342 mHostGTest.setBuild(buildInfo); 343 344 mSetter.setOptionValue("module-name", moduleName); 345 346 mHostGTest.run(mTestInfo, mMockInvocationListener); 347 String testOutput = new String(mFakeReceiver.getReceivedOutput(), StandardCharsets.UTF_8); 348 assertEquals("TEST FAILED\n", testOutput); 349 } 350 351 /* Test that if the test module exits with a non-zero code other than 1, an exception is thrown 352 * and the run stops. */ 353 @Test testAbnormalTestCmdExitHandled()354 public void testAbnormalTestCmdExitHandled() throws Exception { 355 String moduleName = "hello_world_test"; 356 String hostLinkedFolderName = "hosttestcases"; 357 File hostLinkedFolder = createSubFolder(hostLinkedFolderName); 358 Path errorScriptPath = Paths.get(hostLinkedFolder.getAbsolutePath(), moduleName); 359 createExecutableFile(errorScriptPath, "echo 'TEST BLOWING UP'; exit 2"); 360 361 DeviceBuildInfo buildInfo = new DeviceBuildInfo(); 362 buildInfo.setFile(BuildInfoKey.BuildInfoFileKey.HOST_LINKED_DIR, hostLinkedFolder, "0.0"); 363 mHostGTest.setBuild(buildInfo); 364 365 mSetter.setOptionValue("module-name", moduleName); 366 367 try { 368 mHostGTest.run(mTestInfo, mMockInvocationListener); 369 fail("Didn't throw RuntimeException for test cmd with bad exit code"); 370 } catch (RuntimeException e) { 371 // Expected exception 372 } 373 assertNotEquals(0, mFakeReceiver.getReceivedOutput().length); 374 } 375 376 @Test testBothStdoutAndStderrCollected()377 public void testBothStdoutAndStderrCollected() throws Exception { 378 String moduleName = "hello_world_test"; 379 String hostLinkedFolderName = "hosttestcases"; 380 File hostLinkedFolder = createSubFolder(hostLinkedFolderName); 381 File hostTestcaseExecutedCheckFile = 382 createTestScript(hostLinkedFolder.getAbsolutePath(), moduleName); 383 384 DeviceBuildInfo buildInfo = new DeviceBuildInfo(); 385 buildInfo.setFile(BuildInfoKey.BuildInfoFileKey.HOST_LINKED_DIR, hostLinkedFolder, "0.0"); 386 mHostGTest.setBuild(buildInfo); 387 388 mSetter.setOptionValue("module-name", moduleName); 389 mHostGTest.run(mTestInfo, mMockInvocationListener); 390 assertTrue(hostTestcaseExecutedCheckFile.exists()); 391 392 String expected = String.format("stdout: %s\nstderr: %s\n", moduleName, moduleName); 393 String testOutput = new String(mFakeReceiver.getReceivedOutput(), StandardCharsets.UTF_8); 394 assertEquals(expected, testOutput); 395 } 396 } 397