1 /* 2 * Copyright (C) 2017 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package com.android.tradefed.invoker.shard; 17 18 import static org.junit.Assert.assertEquals; 19 import static org.junit.Assert.assertFalse; 20 import static org.junit.Assert.assertNotEquals; 21 import static org.junit.Assert.assertSame; 22 import static org.junit.Assert.assertTrue; 23 24 import com.android.tradefed.build.BuildInfo; 25 import com.android.tradefed.build.StubBuildProvider; 26 import com.android.tradefed.command.CommandOptions; 27 import com.android.tradefed.config.Configuration; 28 import com.android.tradefed.config.ConfigurationDef; 29 import com.android.tradefed.config.ConfigurationException; 30 import com.android.tradefed.config.ConfigurationFactory; 31 import com.android.tradefed.config.DeviceConfigurationHolder; 32 import com.android.tradefed.config.GlobalConfiguration; 33 import com.android.tradefed.config.IConfiguration; 34 import com.android.tradefed.config.OptionSetter; 35 import com.android.tradefed.device.DeviceNotAvailableException; 36 import com.android.tradefed.device.ITestDevice; 37 import com.android.tradefed.invoker.IInvocationContext; 38 import com.android.tradefed.invoker.IRescheduler; 39 import com.android.tradefed.invoker.InvocationContext; 40 import com.android.tradefed.invoker.TestInformation; 41 import com.android.tradefed.result.ILogSaver; 42 import com.android.tradefed.result.ITestInvocationListener; 43 import com.android.tradefed.testtype.IInvocationContextReceiver; 44 import com.android.tradefed.testtype.IRemoteTest; 45 import com.android.tradefed.testtype.IShardableTest; 46 import com.android.tradefed.testtype.StubTest; 47 import com.android.tradefed.testtype.suite.ITestSuite; 48 import com.android.tradefed.util.FileUtil; 49 50 import org.easymock.EasyMock; 51 import org.junit.Assert; 52 import org.junit.Before; 53 import org.junit.Test; 54 import org.junit.runner.RunWith; 55 import org.junit.runners.JUnit4; 56 import org.mockito.ArgumentMatcher; 57 import org.mockito.Mockito; 58 59 import java.io.File; 60 import java.io.IOException; 61 import java.util.ArrayList; 62 import java.util.Collection; 63 import java.util.LinkedHashMap; 64 import java.util.List; 65 66 /** Unit tests for {@link StrictShardHelper}. */ 67 @RunWith(JUnit4.class) 68 public class StrictShardHelperTest { 69 70 private static final String TEST_CONFIG = 71 "<configuration description=\"shard config test\">\n" 72 + " <%s class=\"%s\" />\n" 73 + "</configuration>"; 74 75 private StrictShardHelper mHelper; 76 private IConfiguration mConfig; 77 private ILogSaver mMockLogSaver; 78 private TestInformation mTestInfo; 79 private IInvocationContext mContext; 80 private IRescheduler mRescheduler; 81 82 @Before setUp()83 public void setUp() { 84 mHelper = new StrictShardHelper(); 85 mConfig = new Configuration("fake_sharding_config", "desc"); 86 mContext = new InvocationContext(); 87 mContext.addAllocatedDevice( 88 ConfigurationDef.DEFAULT_DEVICE_NAME, Mockito.mock(ITestDevice.class)); 89 mContext.addDeviceBuildInfo(ConfigurationDef.DEFAULT_DEVICE_NAME, new BuildInfo()); 90 mTestInfo = TestInformation.newBuilder().setInvocationContext(mContext).build(); 91 mRescheduler = Mockito.mock(IRescheduler.class); 92 mMockLogSaver = Mockito.mock(ILogSaver.class); 93 mConfig.setLogSaver(mMockLogSaver); 94 } 95 96 /** Test sharding using Tradefed internal algorithm. */ 97 @Test testShardConfig_internal()98 public void testShardConfig_internal() throws Exception { 99 try { 100 GlobalConfiguration.createGlobalConfiguration(new String[] {"empty"}); 101 } catch (IllegalStateException ignore) { 102 // Ignore 103 } 104 File configFile = 105 createTmpConfig(Configuration.BUILD_PROVIDER_TYPE_NAME, new StubBuildProvider()); 106 try { 107 DeviceConfigurationHolder holder = 108 new DeviceConfigurationHolder(ConfigurationDef.DEFAULT_DEVICE_NAME); 109 holder.addSpecificConfig(new StubBuildProvider()); 110 mConfig.setDeviceConfig(holder); 111 CommandOptions options = new CommandOptions(); 112 OptionSetter setter = new OptionSetter(options); 113 setter.setOptionValue("shard-count", "5"); 114 mConfig.setCommandOptions(options); 115 mConfig.setCommandLine(new String[] {configFile.getAbsolutePath()}); 116 StubTest test = new StubTest(); 117 setter = new OptionSetter(test); 118 setter.setOptionValue("num-shards", "5"); 119 mConfig.setTest(test); 120 assertEquals(1, mConfig.getTests().size()); 121 assertTrue(mHelper.shardConfig(mConfig, mTestInfo, mRescheduler, null)); 122 // Ensure that we did split 1 tests per shard rescheduled. 123 Mockito.verify(mRescheduler, Mockito.times(5)) 124 .scheduleConfig( 125 Mockito.argThat( 126 new ArgumentMatcher<IConfiguration>() { 127 @Override 128 public boolean matches(IConfiguration argument) { 129 assertEquals(1, argument.getTests().size()); 130 return true; 131 } 132 })); 133 } finally { 134 FileUtil.deleteFile(configFile); 135 } 136 } 137 138 /** Test sharding using Tradefed internal algorithm. */ 139 @Test testShardConfig_internal_shardIndex()140 public void testShardConfig_internal_shardIndex() throws Exception { 141 CommandOptions options = new CommandOptions(); 142 OptionSetter setter = new OptionSetter(options); 143 setter.setOptionValue("shard-count", "5"); 144 setter.setOptionValue("shard-index", "2"); 145 mConfig.setCommandOptions(options); 146 mConfig.setCommandLine(new String[] {"empty"}); 147 StubTest test = new StubTest(); 148 setter = new OptionSetter(test); 149 setter.setOptionValue("num-shards", "5"); 150 mConfig.setTest(test); 151 assertEquals(1, mConfig.getTests().size()); 152 // We do not shard, we are relying on the current invocation to run. 153 assertFalse(mHelper.shardConfig(mConfig, mTestInfo, mRescheduler, null)); 154 // Rescheduled is NOT called because we use the current invocation to run the index. 155 Mockito.verify(mRescheduler, Mockito.times(0)).scheduleConfig(Mockito.any()); 156 assertEquals(1, mConfig.getTests().size()); 157 // Original IRemoteTest was replaced by the sharded one in the configuration. 158 assertNotEquals(test, mConfig.getTests().get(0)); 159 } 160 161 /** 162 * Test sharding using Tradefed internal algorithm. On a non shardable IRemoteTest and getting 163 * the shard 0. 164 */ 165 @Test testShardConfig_internal_shardIndex_notShardable_shard0()166 public void testShardConfig_internal_shardIndex_notShardable_shard0() throws Exception { 167 CommandOptions options = new CommandOptions(); 168 OptionSetter setter = new OptionSetter(options); 169 setter.setOptionValue("shard-count", "5"); 170 setter.setOptionValue("shard-index", "0"); 171 mConfig.setCommandOptions(options); 172 mConfig.setCommandLine(new String[] {"empty"}); 173 IRemoteTest test = 174 new IRemoteTest() { 175 @Override 176 public void run(TestInformation testInfo, ITestInvocationListener listener) 177 throws DeviceNotAvailableException { 178 // do nothing. 179 } 180 }; 181 mConfig.setTest(test); 182 assertEquals(1, mConfig.getTests().size()); 183 // We do not shard, we are relying on the current invocation to run. 184 assertFalse(mHelper.shardConfig(mConfig, mTestInfo, mRescheduler, null)); 185 // Rescheduled is NOT called because we use the current invocation to run the index. 186 Mockito.verify(mRescheduler, Mockito.times(0)).scheduleConfig(Mockito.any()); 187 assertEquals(1, mConfig.getTests().size()); 188 // Original IRemoteTest is the same since the test was not shardable 189 assertSame(test, mConfig.getTests().get(0)); 190 } 191 192 /** 193 * Test sharding using Tradefed internal algorithm. On a non shardable IRemoteTest and getting 194 * the shard 1. 195 */ 196 @Test testShardConfig_internal_shardIndex_notShardable_shard1()197 public void testShardConfig_internal_shardIndex_notShardable_shard1() throws Exception { 198 CommandOptions options = new CommandOptions(); 199 OptionSetter setter = new OptionSetter(options); 200 setter.setOptionValue("shard-count", "5"); 201 setter.setOptionValue("shard-index", "1"); 202 mConfig.setCommandOptions(options); 203 mConfig.setCommandLine(new String[] {"empty"}); 204 IRemoteTest test = 205 new IRemoteTest() { 206 @Override 207 public void run(TestInformation testInfo, ITestInvocationListener listener) 208 throws DeviceNotAvailableException { 209 // do nothing. 210 } 211 }; 212 mConfig.setTest(test); 213 assertEquals(1, mConfig.getTests().size()); 214 // We do not shard, we are relying on the current invocation to run. 215 assertFalse(mHelper.shardConfig(mConfig, mTestInfo, mRescheduler, null)); 216 // Rescheduled is NOT called because we use the current invocation to run the index. 217 Mockito.verify(mRescheduler, Mockito.times(0)).scheduleConfig(Mockito.any()); 218 // We have no tests to put in shard-index 1 so it's empty. 219 assertEquals(0, mConfig.getTests().size()); 220 } 221 222 /** Test class to simulate an ITestSuite getting split. */ 223 public static class SplitITestSuite extends ITestSuite { 224 225 private String mName; 226 private IRemoteTest mForceTest = null; 227 SplitITestSuite()228 public SplitITestSuite() {} 229 SplitITestSuite(String name)230 public SplitITestSuite(String name) { 231 mName = name; 232 } 233 SplitITestSuite(String name, IRemoteTest test)234 public SplitITestSuite(String name, IRemoteTest test) { 235 this(name); 236 mForceTest = test; 237 } 238 239 @Override loadTests()240 public LinkedHashMap<String, IConfiguration> loadTests() { 241 LinkedHashMap<String, IConfiguration> configs = new LinkedHashMap<>(); 242 IConfiguration configuration = null; 243 try { 244 configuration = 245 ConfigurationFactory.getInstance() 246 .createConfigurationFromArgs( 247 new String[] {"empty", "--num-shards", "2"}); 248 if (mForceTest != null) { 249 configuration.setTest(mForceTest); 250 } 251 } catch (ConfigurationException e) { 252 throw new RuntimeException(e); 253 } 254 configs.put(mName, configuration); 255 return configs; 256 } 257 } 258 createFakeSuite(String name)259 private ITestSuite createFakeSuite(String name) throws Exception { 260 ITestSuite suite = new SplitITestSuite(name); 261 return suite; 262 } 263 testShard(int shardIndex)264 private List<IRemoteTest> testShard(int shardIndex) throws Exception { 265 mContext.addAllocatedDevice("default", EasyMock.createMock(ITestDevice.class)); 266 List<IRemoteTest> test = new ArrayList<>(); 267 test.add(createFakeSuite("module2")); 268 test.add(createFakeSuite("module1")); 269 test.add(createFakeSuite("module3")); 270 test.add(createFakeSuite("module1")); 271 test.add(createFakeSuite("module1")); 272 test.add(createFakeSuite("module2")); 273 test.add(createFakeSuite("module3")); 274 CommandOptions options = new CommandOptions(); 275 OptionSetter setter = new OptionSetter(options); 276 setter.setOptionValue("shard-count", "3"); 277 setter.setOptionValue("shard-index", Integer.toString(shardIndex)); 278 mConfig.setCommandOptions(options); 279 mConfig.setCommandLine(new String[] {"empty"}); 280 mConfig.setTests(test); 281 mHelper.shardConfig(mConfig, mTestInfo, mRescheduler, null); 282 return mConfig.getTests(); 283 } 284 285 /** 286 * Total for all the _shardX test should be 14 tests (2 per modules). 6 for module1: 3 module1 287 * shard * 2 4 for module2: 2 module2 shard * 2 4 for module3: 2 module3 shard * 2 288 */ 289 @Test testMergeSuite_shard0()290 public void testMergeSuite_shard0() throws Exception { 291 List<IRemoteTest> res = testShard(0); 292 assertEquals(3, res.size()); 293 294 assertTrue(res.get(0) instanceof ITestSuite); 295 assertEquals("module3", ((ITestSuite) res.get(0)).getDirectModule().getId()); 296 assertEquals(2, ((ITestSuite) res.get(0)).getDirectModule().numTests()); 297 298 assertTrue(res.get(1) instanceof ITestSuite); 299 assertEquals("module1", ((ITestSuite) res.get(1)).getDirectModule().getId()); 300 assertEquals(1, ((ITestSuite) res.get(1)).getDirectModule().numTests()); 301 302 assertTrue(res.get(2) instanceof ITestSuite); 303 assertEquals("module2", ((ITestSuite) res.get(2)).getDirectModule().getId()); 304 assertEquals(1, ((ITestSuite) res.get(2)).getDirectModule().numTests()); 305 } 306 307 @Test testMergeSuite_shard1()308 public void testMergeSuite_shard1() throws Exception { 309 List<IRemoteTest> res = testShard(1); 310 assertEquals(3, res.size()); 311 312 assertTrue(res.get(0) instanceof ITestSuite); 313 assertEquals("module1", ((ITestSuite) res.get(0)).getDirectModule().getId()); 314 assertEquals(1, ((ITestSuite) res.get(0)).getDirectModule().numTests()); 315 316 assertTrue(res.get(1) instanceof ITestSuite); 317 assertEquals("module3", ((ITestSuite) res.get(1)).getDirectModule().getId()); 318 assertEquals(1, ((ITestSuite) res.get(1)).getDirectModule().numTests()); 319 320 assertTrue(res.get(2) instanceof ITestSuite); 321 assertEquals("module2", ((ITestSuite) res.get(2)).getDirectModule().getId()); 322 assertEquals(2, ((ITestSuite) res.get(2)).getDirectModule().numTests()); 323 } 324 325 @Test testMergeSuite_shard2()326 public void testMergeSuite_shard2() throws Exception { 327 List<IRemoteTest> res = testShard(2); 328 assertEquals(3, res.size()); 329 330 assertTrue(res.get(0) instanceof ITestSuite); 331 assertEquals("module1", ((ITestSuite) res.get(0)).getDirectModule().getId()); 332 assertEquals(4, ((ITestSuite) res.get(0)).getDirectModule().numTests()); 333 334 assertTrue(res.get(1) instanceof ITestSuite); 335 assertEquals("module2", ((ITestSuite) res.get(1)).getDirectModule().getId()); 336 assertEquals(1, ((ITestSuite) res.get(1)).getDirectModule().numTests()); 337 338 assertTrue(res.get(1) instanceof ITestSuite); 339 assertEquals("module3", ((ITestSuite) res.get(2)).getDirectModule().getId()); 340 assertEquals(1, ((ITestSuite) res.get(2)).getDirectModule().numTests()); 341 } 342 343 @Test testShardSuite()344 public void testShardSuite() throws Exception { 345 // mConfig 346 mHelper.shardConfig(mConfig, mTestInfo, mRescheduler, null); 347 } 348 349 /** 350 * Test class to ensure that when sharding interfaces are properly called and forwarded so the 351 * tests have all their information for sharding. 352 */ 353 public static class TestInterfaceClass implements IShardableTest, IInvocationContextReceiver { 354 355 @Override setInvocationContext(IInvocationContext invocationContext)356 public void setInvocationContext(IInvocationContext invocationContext) { 357 Assert.assertNotNull(invocationContext); 358 } 359 360 @Override run(TestInformation testInfo, ITestInvocationListener listener)361 public void run(TestInformation testInfo, ITestInvocationListener listener) 362 throws DeviceNotAvailableException { 363 // ignore 364 } 365 366 @Override split(int hintShard)367 public Collection<IRemoteTest> split(int hintShard) { 368 if (hintShard > 1) { 369 List<IRemoteTest> shards = new ArrayList<IRemoteTest>(hintShard); 370 for (int i = 0; i < hintShard; i++) { 371 shards.add(new TestInterfaceClass()); 372 } 373 return shards; 374 } 375 return null; 376 } 377 } 378 379 /** Test that no exception occurs when sharding for any possible interfaces. */ 380 @Test testSuite_withAllInterfaces()381 public void testSuite_withAllInterfaces() throws Exception { 382 mContext.addAllocatedDevice("default", EasyMock.createMock(ITestDevice.class)); 383 IRemoteTest forceTest = new TestInterfaceClass(); 384 IRemoteTest test = new SplitITestSuite("suite-interface", forceTest); 385 386 CommandOptions options = new CommandOptions(); 387 OptionSetter setter = new OptionSetter(options); 388 setter.setOptionValue("shard-count", "3"); 389 setter.setOptionValue("shard-index", Integer.toString(0)); 390 mConfig.setCommandOptions(options); 391 mConfig.setTest(test); 392 mHelper.shardConfig(mConfig, mTestInfo, mRescheduler, null); 393 394 List<IRemoteTest> res = mConfig.getTests(); 395 assertEquals(1, res.size()); 396 397 assertTrue(res.get(0) instanceof ITestSuite); 398 assertEquals("suite-interface", ((ITestSuite) res.get(0)).getDirectModule().getId()); 399 assertEquals(1, ((ITestSuite) res.get(0)).getDirectModule().numTests()); 400 } 401 402 /** Helper for distribution tests to simply populate a list of a given count. */ createFakeTestList(int count)403 private List<IRemoteTest> createFakeTestList(int count) { 404 List<IRemoteTest> testList = new ArrayList<>(); 405 for (int i = 0; i < count; i++) { 406 testList.add(new StubTest()); 407 } 408 return testList; 409 } 410 411 /** 412 * The distribution tests bellow expose an issue that arised with some combination of number of 413 * tests and shard-count. The number of tests allocated to each shard made us use the full list 414 * of tests before reaching the last shard, resulting in some OutOfBounds exception. Logic was 415 * added to detect these cases and properly handle them as well as ensuring a proper balancing. 416 */ 417 418 /** Test that the special ratio 130 tests for 20 shards is properly redistributed. */ 419 @Test testDistribution_hightests_highcount()420 public void testDistribution_hightests_highcount() { 421 List<IRemoteTest> testList = createFakeTestList(130); 422 int shardCount = 20; 423 List<List<IRemoteTest>> res = mHelper.splitTests(testList, shardCount); 424 assertEquals(7, res.get(0).size()); 425 assertEquals(7, res.get(1).size()); 426 assertEquals(7, res.get(2).size()); 427 assertEquals(7, res.get(3).size()); 428 assertEquals(7, res.get(4).size()); 429 assertEquals(7, res.get(5).size()); 430 assertEquals(7, res.get(6).size()); 431 assertEquals(7, res.get(7).size()); 432 assertEquals(7, res.get(8).size()); 433 assertEquals(7, res.get(9).size()); 434 assertEquals(6, res.get(10).size()); 435 assertEquals(6, res.get(11).size()); 436 assertEquals(6, res.get(12).size()); 437 assertEquals(6, res.get(13).size()); 438 assertEquals(6, res.get(14).size()); 439 assertEquals(6, res.get(15).size()); 440 assertEquals(6, res.get(16).size()); 441 assertEquals(6, res.get(17).size()); 442 assertEquals(6, res.get(18).size()); 443 assertEquals(6, res.get(19).size()); 444 } 445 446 /** Test that the special ratio 7 tests for 6 shards is properly redistributed. */ 447 @Test testDistribution_lowtests_lowcount()448 public void testDistribution_lowtests_lowcount() { 449 List<IRemoteTest> testList = createFakeTestList(7); 450 int shardCount = 6; 451 List<List<IRemoteTest>> res = mHelper.splitTests(testList, shardCount); 452 assertEquals(2, res.get(0).size()); 453 assertEquals(1, res.get(1).size()); 454 assertEquals(1, res.get(2).size()); 455 assertEquals(1, res.get(3).size()); 456 assertEquals(1, res.get(4).size()); 457 assertEquals(1, res.get(5).size()); 458 } 459 460 /** Test that the special ratio 13 tests for 6 shards is properly redistributed. */ 461 @Test testDistribution_lowtests_lowcount2()462 public void testDistribution_lowtests_lowcount2() { 463 List<IRemoteTest> testList = createFakeTestList(13); 464 int shardCount = 6; 465 List<List<IRemoteTest>> res = mHelper.splitTests(testList, shardCount); 466 assertEquals(3, res.get(0).size()); 467 assertEquals(2, res.get(1).size()); 468 assertEquals(2, res.get(2).size()); 469 assertEquals(2, res.get(3).size()); 470 assertEquals(2, res.get(4).size()); 471 assertEquals(2, res.get(5).size()); 472 } 473 createTmpConfig(String objType, Object obj)474 private File createTmpConfig(String objType, Object obj) throws IOException { 475 File configFile = FileUtil.createTempFile("shard-helper-test", ".xml"); 476 String content = String.format(TEST_CONFIG, objType, obj.getClass().getCanonicalName()); 477 FileUtil.writeToFile(content, configFile); 478 return configFile; 479 } 480 } 481