1 /* 2 * Copyright (C) 2010 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package com.android.tradefed.testtype; 18 19 import static org.junit.Assert.assertEquals; 20 import static org.junit.Assert.assertNotNull; 21 import static org.junit.Assert.assertTrue; 22 23 import com.android.ddmlib.Log; 24 import com.android.ddmlib.testrunner.TestResult.TestStatus; 25 import com.android.tradefed.TestAppConstants; 26 import com.android.tradefed.config.OptionSetter; 27 import com.android.tradefed.device.DeviceNotAvailableException; 28 import com.android.tradefed.device.DeviceUnresponsiveException; 29 import com.android.tradefed.device.ITestDevice; 30 import com.android.tradefed.device.ITestDevice.RecoveryMode; 31 import com.android.tradefed.invoker.TestInformation; 32 import com.android.tradefed.metrics.proto.MetricMeasurement.Metric; 33 import com.android.tradefed.result.CollectingTestListener; 34 import com.android.tradefed.result.ITestInvocationListener; 35 import com.android.tradefed.result.TestDescription; 36 import com.android.tradefed.util.RunUtil; 37 38 import org.easymock.EasyMock; 39 import org.junit.Before; 40 import org.junit.Ignore; 41 import org.junit.Test; 42 import org.junit.runner.RunWith; 43 44 import java.io.IOException; 45 import java.util.HashMap; 46 47 /** Functional tests for {@link InstrumentationTest}. */ 48 @RunWith(DeviceJUnit4ClassRunner.class) 49 public class InstrumentationTestFuncTest implements IDeviceTest, ITestInformationReceiver { 50 51 private static final String LOG_TAG = "InstrumentationTestFuncTest"; 52 private static final long SHELL_TIMEOUT = 2500; 53 private static final int TEST_TIMEOUT = 2000; 54 private static final long WAIT_FOR_DEVICE_AVAILABLE = 5 * 60 * 1000; 55 56 private ITestDevice mDevice; 57 private TestInformation mTestInfo; 58 59 /** The {@link InstrumentationTest} under test */ 60 private InstrumentationTest mInstrumentationTest; 61 62 private ITestInvocationListener mMockListener; 63 64 @Override setTestInformation(TestInformation testInformation)65 public void setTestInformation(TestInformation testInformation) { 66 mTestInfo = testInformation; 67 } 68 69 @Override getTestInformation()70 public TestInformation getTestInformation() { 71 return mTestInfo; 72 } 73 74 @Override setDevice(ITestDevice device)75 public void setDevice(ITestDevice device) { 76 mDevice = device; 77 } 78 79 @Override getDevice()80 public ITestDevice getDevice() { 81 return mDevice; 82 } 83 84 @Before setUp()85 public void setUp() throws Exception { 86 mInstrumentationTest = new InstrumentationTest(); 87 mInstrumentationTest.setPackageName(TestAppConstants.TESTAPP_PACKAGE); 88 mInstrumentationTest.setDevice(getDevice()); 89 // use no timeout by default 90 mInstrumentationTest.setShellTimeout(-1); 91 // set to no rerun by default 92 mInstrumentationTest.setRerunMode(false); 93 mMockListener = EasyMock.createStrictMock(ITestInvocationListener.class); 94 getDevice().disableKeyguard(); 95 } 96 97 /** Test normal run scenario with a single passed test result. */ 98 @Test testRun()99 public void testRun() throws DeviceNotAvailableException { 100 Log.i(LOG_TAG, "testRun"); 101 TestDescription expectedTest = 102 new TestDescription( 103 TestAppConstants.TESTAPP_CLASS, TestAppConstants.PASSED_TEST_METHOD); 104 mInstrumentationTest.setClassName(TestAppConstants.TESTAPP_CLASS); 105 mInstrumentationTest.setMethodName(TestAppConstants.PASSED_TEST_METHOD); 106 mInstrumentationTest.setTestTimeout(TEST_TIMEOUT); 107 mInstrumentationTest.setShellTimeout(SHELL_TIMEOUT); 108 mMockListener.testRunStarted(TestAppConstants.TESTAPP_PACKAGE, 1); 109 mMockListener.testStarted(EasyMock.eq(expectedTest)); 110 mMockListener.testEnded( 111 EasyMock.eq(expectedTest), EasyMock.<HashMap<String, Metric>>anyObject()); 112 mMockListener.testRunEnded( 113 EasyMock.anyLong(), EasyMock.<HashMap<String, Metric>>anyObject()); 114 EasyMock.replay(mMockListener); 115 mInstrumentationTest.run(getTestInformation(), mMockListener); 116 EasyMock.verify(mMockListener); 117 } 118 119 /** Test normal run scenario with a single failed test result. */ 120 @Test testRun_testFailed()121 public void testRun_testFailed() throws DeviceNotAvailableException { 122 Log.i(LOG_TAG, "testRun_testFailed"); 123 mInstrumentationTest.setClassName(TestAppConstants.TESTAPP_CLASS); 124 mInstrumentationTest.setMethodName(TestAppConstants.FAILED_TEST_METHOD); 125 mInstrumentationTest.setTestTimeout(TEST_TIMEOUT); 126 mInstrumentationTest.setShellTimeout(SHELL_TIMEOUT); 127 String[] error = new String[1]; 128 error[0] = null; 129 mInstrumentationTest.run( 130 getTestInformation(), 131 new ITestInvocationListener() { 132 @Override 133 public void testFailed(TestDescription test, String trace) { 134 error[0] = trace; 135 } 136 }); 137 assertNotNull("testFailed was not called", error[0]); 138 assertTrue(error[0].contains("junit.framework.AssertionFailedError: test failed")); 139 } 140 141 /** Test run scenario where test process crashes. */ 142 @Test testRun_testCrash()143 public void testRun_testCrash() throws DeviceNotAvailableException { 144 Log.i(LOG_TAG, "testRun_testCrash"); 145 TestDescription expectedTest = 146 new TestDescription( 147 TestAppConstants.TESTAPP_CLASS, TestAppConstants.CRASH_TEST_METHOD); 148 mInstrumentationTest.setClassName(TestAppConstants.TESTAPP_CLASS); 149 mInstrumentationTest.setMethodName(TestAppConstants.CRASH_TEST_METHOD); 150 mInstrumentationTest.setTestTimeout(TEST_TIMEOUT); 151 mInstrumentationTest.setShellTimeout(SHELL_TIMEOUT); 152 mMockListener.testRunStarted(TestAppConstants.TESTAPP_PACKAGE, 1); 153 mMockListener.testStarted(EasyMock.eq(expectedTest)); 154 if (getDevice().getApiLevel() <= 23) { 155 // Before N handling of instrumentation crash is slightly different. 156 mMockListener.testFailed( 157 EasyMock.eq(expectedTest), EasyMock.contains("RuntimeException")); 158 mMockListener.testEnded( 159 EasyMock.eq(expectedTest), EasyMock.<HashMap<String, Metric>>anyObject()); 160 mMockListener.testRunFailed( 161 EasyMock.eq("Instrumentation run failed due to 'java.lang.RuntimeException'")); 162 } else { 163 mMockListener.testFailed( 164 EasyMock.eq(expectedTest), EasyMock.contains("Process crashed.")); 165 mMockListener.testEnded( 166 EasyMock.eq(expectedTest), EasyMock.<HashMap<String, Metric>>anyObject()); 167 mMockListener.testRunFailed( 168 EasyMock.eq("Instrumentation run failed due to 'Process crashed.'")); 169 } 170 mMockListener.testRunEnded( 171 EasyMock.anyLong(), EasyMock.<HashMap<String, Metric>>anyObject()); 172 try { 173 EasyMock.replay(mMockListener); 174 mInstrumentationTest.run(getTestInformation(), mMockListener); 175 EasyMock.verify(mMockListener); 176 } finally { 177 getDevice().waitForDeviceAvailable(); 178 } 179 } 180 181 /** Test run scenario where test run hangs indefinitely, and times out. */ 182 @Test testRun_testTimeout()183 public void testRun_testTimeout() throws DeviceNotAvailableException { 184 Log.i(LOG_TAG, "testRun_testTimeout"); 185 RecoveryMode initMode = getDevice().getRecoveryMode(); 186 getDevice().setRecoveryMode(RecoveryMode.NONE); 187 try { 188 mInstrumentationTest.setClassName(TestAppConstants.TESTAPP_CLASS); 189 mInstrumentationTest.setMethodName(TestAppConstants.TIMEOUT_TEST_METHOD); 190 mInstrumentationTest.setShellTimeout(SHELL_TIMEOUT); 191 mInstrumentationTest.setTestTimeout(TEST_TIMEOUT); 192 193 String[] error = new String[1]; 194 error[0] = null; 195 mInstrumentationTest.run( 196 getTestInformation(), 197 new ITestInvocationListener() { 198 @Override 199 public void testFailed(TestDescription test, String trace) { 200 error[0] = trace; 201 } 202 }); 203 assertEquals( 204 "Test failed to run to completion. Reason: 'Failed to receive adb shell test " 205 + "output within 2500 ms. Test may have timed out, or adb connection to device " 206 + "became unresponsive'. Check device logcat for details", 207 error[0]); 208 } finally { 209 getDevice().setRecoveryMode(initMode); 210 RunUtil.getDefault().sleep(500); 211 } 212 } 213 214 /** Test run scenario where device reboots during test run. */ 215 @Test testRun_deviceReboot()216 public void testRun_deviceReboot() throws Exception { 217 Log.i(LOG_TAG, "testRun_deviceReboot"); 218 mInstrumentationTest.setClassName(TestAppConstants.TESTAPP_CLASS); 219 mInstrumentationTest.setMethodName(TestAppConstants.TIMEOUT_TEST_METHOD); 220 mInstrumentationTest.setShellTimeout(0); 221 mInstrumentationTest.setTestTimeout(0); 222 // Set a max timeout to avoid hanging forever for safety 223 //OptionSetter setter = new OptionSetter(mInstrumentationTest); 224 //setter.setOptionValue("max-timeout", "600000"); 225 226 // fork off a thread to do the reboot 227 Thread rebootThread = 228 new Thread() { 229 @Override 230 public void run() { 231 // wait for test run to begin 232 try { 233 // Give time to the instrumentation to start 234 Thread.sleep(2000); 235 getDevice().reboot(); 236 } catch (InterruptedException e) { 237 Log.w(LOG_TAG, "interrupted"); 238 } catch (DeviceNotAvailableException dnae) { 239 Log.w(LOG_TAG, "Device did not come back online after reboot"); 240 } 241 } 242 }; 243 rebootThread.setName("InstrumentationTestFuncTest#testRun_deviceReboot"); 244 rebootThread.start(); 245 try { 246 String[] error = new String[1]; 247 error[0] = null; 248 mInstrumentationTest.run( 249 getTestInformation(), 250 new ITestInvocationListener() { 251 @Override 252 public void testRunFailed(String errorMessage) { 253 error[0] = errorMessage; 254 } 255 }); 256 assertEquals("Test run failed to complete. Expected 1 tests, received 0", error[0]); 257 } catch (DeviceUnresponsiveException expected) { 258 // expected 259 } finally { 260 rebootThread.join(WAIT_FOR_DEVICE_AVAILABLE); 261 getDevice().waitForDeviceAvailable(); 262 } 263 } 264 265 /** Test that when a max-timeout is set the instrumentation is stopped. */ 266 @Test testRun_maxTimeout()267 public void testRun_maxTimeout() throws Exception { 268 Log.i(LOG_TAG, "testRun_maxTimeout"); 269 mInstrumentationTest.setClassName(TestAppConstants.TESTAPP_CLASS); 270 mInstrumentationTest.setMethodName(TestAppConstants.TIMEOUT_TEST_METHOD); 271 mInstrumentationTest.setShellTimeout(0); 272 mInstrumentationTest.setTestTimeout(0); 273 OptionSetter setter = new OptionSetter(mInstrumentationTest); 274 setter.setOptionValue("max-timeout", "5000"); 275 final String[] called = new String[1]; 276 called[0] = null; 277 mInstrumentationTest.run( 278 getTestInformation(), 279 new ITestInvocationListener() { 280 @Override 281 public void testRunFailed(String errorMessage) { 282 called[0] = errorMessage; 283 } 284 }); 285 assertEquals( 286 "com.android.ddmlib.TimeoutException: executeRemoteCommand timed out after 5000ms", 287 called[0]); 288 } 289 290 /** Test run scenario where device runtime resets during test run. */ 291 @Test 292 @Ignore testRun_deviceRuntimeReset()293 public void testRun_deviceRuntimeReset() throws Exception { 294 Log.i(LOG_TAG, "testRun_deviceRuntimeReset"); 295 mInstrumentationTest.setShellTimeout(SHELL_TIMEOUT); 296 mInstrumentationTest.setTestTimeout(TEST_TIMEOUT); 297 mInstrumentationTest.setClassName(TestAppConstants.TESTAPP_CLASS); 298 mInstrumentationTest.setMethodName(TestAppConstants.TIMEOUT_TEST_METHOD); 299 300 // fork off a thread to do the runtime reset 301 Thread resetThread = 302 new Thread() { 303 @Override 304 public void run() { 305 // wait for test run to begin 306 try { 307 Thread.sleep(1000); 308 Runtime.getRuntime() 309 .exec( 310 String.format( 311 "adb -s %s shell stop", 312 getDevice().getIDevice().getSerialNumber())); 313 Thread.sleep(500); 314 Runtime.getRuntime() 315 .exec( 316 String.format( 317 "adb -s %s shell start", 318 getDevice().getIDevice().getSerialNumber())); 319 } catch (InterruptedException e) { 320 Log.w(LOG_TAG, "interrupted"); 321 } catch (IOException e) { 322 Log.w(LOG_TAG, "IOException when rebooting"); 323 } 324 } 325 }; 326 resetThread.setName("InstrumentationTestFuncTest#testRun_deviceRuntimeReset"); 327 resetThread.start(); 328 try { 329 String[] error = new String[1]; 330 error[0] = null; 331 mInstrumentationTest.run( 332 getTestInformation(), 333 new ITestInvocationListener() { 334 @Override 335 public void testRunFailed(String errorMessage) { 336 error[0] = errorMessage; 337 } 338 }); 339 assertEquals( 340 "Failed to receive adb shell test output within 120000 ms. Test may have " 341 + "timed out, or adb connection to device became unresponsive", 342 error[0]); 343 } finally { 344 resetThread.join(WAIT_FOR_DEVICE_AVAILABLE); 345 RunUtil.getDefault().sleep(5000); 346 getDevice().waitForDeviceAvailable(); 347 } 348 } 349 350 /** 351 * Test running all the tests with rerun on. At least one method will cause run to stop 352 * (currently TIMEOUT_TEST_METHOD and CRASH_TEST_METHOD). Verify that results are recorded for 353 * all tests in the suite. 354 */ 355 @Test testRun_rerun()356 public void testRun_rerun() throws Exception { 357 Log.i(LOG_TAG, "testRun_rerun"); 358 // run all tests in class 359 RecoveryMode initMode = getDevice().getRecoveryMode(); 360 getDevice().setRecoveryMode(RecoveryMode.NONE); 361 try { 362 OptionSetter setter = new OptionSetter(mInstrumentationTest); 363 setter.setOptionValue("collect-tests-timeout", Long.toString(SHELL_TIMEOUT)); 364 mInstrumentationTest.setClassName(TestAppConstants.TESTAPP_CLASS); 365 mInstrumentationTest.setRerunMode(true); 366 mInstrumentationTest.setShellTimeout(SHELL_TIMEOUT); 367 mInstrumentationTest.setTestTimeout(TEST_TIMEOUT); 368 CollectingTestListener listener = new CollectingTestListener(); 369 mInstrumentationTest.run(getTestInformation(), listener); 370 assertEquals(TestAppConstants.TOTAL_TEST_CLASS_TESTS, listener.getNumTotalTests()); 371 assertEquals( 372 TestAppConstants.TOTAL_TEST_CLASS_PASSED_TESTS, 373 listener.getNumTestsInState(TestStatus.PASSED)); 374 } finally { 375 getDevice().setRecoveryMode(initMode); 376 } 377 } 378 379 /** 380 * Test a run that crashes when collecting tests. 381 * 382 * <p>Expect run to proceed, but be reported as a run failure 383 */ 384 @Test testRun_rerunCrash()385 public void testRun_rerunCrash() throws Exception { 386 Log.i(LOG_TAG, "testRun_rerunCrash"); 387 mInstrumentationTest.setClassName(TestAppConstants.CRASH_ON_INIT_TEST_CLASS); 388 mInstrumentationTest.setMethodName(TestAppConstants.CRASH_ON_INIT_TEST_METHOD); 389 mInstrumentationTest.setRerunMode(false); 390 mInstrumentationTest.setShellTimeout(SHELL_TIMEOUT); 391 mInstrumentationTest.setTestTimeout(TEST_TIMEOUT); 392 CollectingTestListener listener = new CollectingTestListener(); 393 mInstrumentationTest.run(getTestInformation(), listener); 394 assertEquals(0, listener.getNumTotalTests()); 395 assertNotNull(listener.getCurrentRunResults()); 396 assertEquals(TestAppConstants.TESTAPP_PACKAGE, listener.getCurrentRunResults().getName()); 397 assertTrue(listener.getCurrentRunResults().isRunFailure()); 398 assertTrue(listener.getCurrentRunResults().isRunComplete()); 399 } 400 401 /** 402 * Test a run that hangs when collecting tests. 403 * 404 * <p>Expect a run failure to be reported 405 */ 406 @Test testRun_rerunHang()407 public void testRun_rerunHang() throws Exception { 408 Log.i(LOG_TAG, "testRun_rerunHang"); 409 RecoveryMode initMode = getDevice().getRecoveryMode(); 410 getDevice().setRecoveryMode(RecoveryMode.NONE); 411 try { 412 OptionSetter setter = new OptionSetter(mInstrumentationTest); 413 setter.setOptionValue("collect-tests-timeout", Long.toString(SHELL_TIMEOUT)); 414 mInstrumentationTest.setClassName(TestAppConstants.HANG_ON_INIT_TEST_CLASS); 415 mInstrumentationTest.setRerunMode(false); 416 mInstrumentationTest.setShellTimeout(SHELL_TIMEOUT); 417 mInstrumentationTest.setTestTimeout(TEST_TIMEOUT); 418 CollectingTestListener listener = new CollectingTestListener(); 419 mInstrumentationTest.run(getTestInformation(), listener); 420 assertEquals(0, listener.getNumTotalTests()); 421 assertEquals( 422 TestAppConstants.TESTAPP_PACKAGE, listener.getCurrentRunResults().getName()); 423 assertTrue(listener.getCurrentRunResults().isRunFailure()); 424 assertTrue(listener.getCurrentRunResults().isRunComplete()); 425 } finally { 426 getDevice().setRecoveryMode(initMode); 427 } 428 } 429 } 430