1 /* 2 * Copyright (C) 2018 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package com.android.compatibility.common.tradefed.result.suite; 17 18 import static org.junit.Assert.*; 19 20 import com.android.compatibility.common.tradefed.build.CompatibilityBuildHelper; 21 import com.android.compatibility.common.tradefed.targetprep.BuildFingerPrintPreparer; 22 import com.android.tradefed.build.BuildInfo; 23 import com.android.tradefed.build.DeviceBuildInfo; 24 import com.android.tradefed.build.IBuildInfo; 25 import com.android.tradefed.build.IBuildProvider; 26 import com.android.tradefed.config.Configuration; 27 import com.android.tradefed.config.ConfigurationDef; 28 import com.android.tradefed.config.ConfigurationDescriptor; 29 import com.android.tradefed.config.IConfiguration; 30 import com.android.tradefed.config.OptionSetter; 31 import com.android.tradefed.device.ITestDevice; 32 import com.android.tradefed.invoker.IInvocationContext; 33 import com.android.tradefed.invoker.InvocationContext; 34 import com.android.tradefed.invoker.TestInvocation; 35 import com.android.tradefed.result.CollectingTestListener; 36 import com.android.tradefed.result.proto.TestRecordProto.TestRecord; 37 import com.android.tradefed.targetprep.ITargetPreparer; 38 import com.android.tradefed.util.FileUtil; 39 40 import com.google.protobuf.Any; 41 42 import org.easymock.EasyMock; 43 import org.junit.After; 44 import org.junit.Before; 45 import org.junit.Test; 46 import org.junit.runner.RunWith; 47 import org.junit.runners.JUnit4; 48 49 import java.io.File; 50 import java.io.FileOutputStream; 51 52 /** 53 * Unit tests for {@link PreviousResultLoader}. 54 */ 55 @RunWith(JUnit4.class) 56 public class PreviousResultLoaderTest { 57 58 public static final String RUN_HISTORY_KEY = "run_history"; 59 60 private PreviousResultLoader mLoader; 61 private IInvocationContext mContext; 62 private File mRootDir; 63 private File mProtoFile; 64 65 private ITestDevice mMockDevice; 66 private IBuildProvider mMockProvider; 67 68 @Before setUp()69 public void setUp() throws Exception { 70 mMockProvider = EasyMock.createMock(IBuildProvider.class); 71 mLoader = new PreviousResultLoader(); 72 mLoader.setProvider(mMockProvider); 73 OptionSetter setter = new OptionSetter(mLoader); 74 setter.setOptionValue("retry", "0"); 75 mContext = new InvocationContext(); 76 mContext.setConfigurationDescriptor(new ConfigurationDescriptor()); 77 mContext.addInvocationAttribute(TestInvocation.COMMAND_ARGS_KEY, 78 "cts -m CtsGesture --skip-all-system-status-check"); 79 mContext.addDeviceBuildInfo(ConfigurationDef.DEFAULT_DEVICE_NAME, new BuildInfo()); 80 mMockDevice = EasyMock.createMock(ITestDevice.class); 81 } 82 83 @After tearDown()84 public void tearDown() throws Exception { 85 FileUtil.recursiveDelete(mRootDir); 86 } 87 88 /** 89 * Test the loader properly fails when the results are not loaded. 90 */ 91 @Test testReloadTests_failed()92 public void testReloadTests_failed() throws Exception { 93 EasyMock.expect(mMockProvider.getBuild()).andReturn(createFakeBuild("", false)); 94 // Delete the proto file 95 mProtoFile.delete(); 96 try { 97 EasyMock.replay(mMockProvider); 98 mLoader.init(); 99 fail("Should have thrown an exception."); 100 } catch (RuntimeException expected) { 101 // expected 102 assertEquals("Could not find any test-record.pb to load.", expected.getMessage()); 103 } 104 EasyMock.verify(mMockProvider); 105 } 106 107 /** 108 * Test that the loader can provide the results information back. 109 */ 110 @Test testReloadTests()111 public void testReloadTests() throws Exception { 112 final String EXPECTED_RUN_HISTORY = 113 "[{\"startTime\":1530218251501," 114 + "\"endTime\":1530218261061," 115 + "\"passedTests\":0," 116 + "\"failedTests\":0," 117 + "\"commandLineArgs\":\"cts -m CtsGesture " 118 + "--skip-all-system-status-check\"," 119 + "\"hostName\":\"user.android.com\"}]"; 120 EasyMock.expect(mMockProvider.getBuild()) 121 .andReturn(createFakeBuild(createBasicResults(), false)); 122 mContext.addAllocatedDevice(ConfigurationDef.DEFAULT_DEVICE_NAME, mMockDevice); 123 124 EasyMock.replay(mMockDevice, mMockProvider); 125 mLoader.init(); 126 assertEquals("cts -m CtsGesture --skip-all-system-status-check", mLoader.getCommandLine()); 127 IConfiguration config = new Configuration("name", "desc"); 128 assertEquals(0, config.getTargetPreparers().size()); 129 mLoader.customizeConfiguration(config); 130 // A special preparer was added for fingerprint 131 assertEquals(1, config.getTargetPreparers().size()); 132 ITargetPreparer preparer = config.getTargetPreparers().get(0); 133 assertTrue(preparer instanceof BuildFingerPrintPreparer); 134 assertEquals( 135 "testfingerprint", ((BuildFingerPrintPreparer) preparer).getExpectedFingerprint()); 136 String runHistory = 137 config.getCommandOptions().getInvocationData().getUniqueMap().get(RUN_HISTORY_KEY); 138 assertEquals(EXPECTED_RUN_HISTORY, runHistory); 139 EasyMock.verify(mMockDevice, mMockProvider); 140 } 141 142 @Test testReloadTests_withMultiProto()143 public void testReloadTests_withMultiProto() throws Exception { 144 final String EXPECTED_RUN_HISTORY = 145 "[{\"startTime\":1530218251501," 146 + "\"endTime\":1530218261061," 147 + "\"passedTests\":0," 148 + "\"failedTests\":0," 149 + "\"commandLineArgs\":\"cts -m CtsGesture " 150 + "--skip-all-system-status-check\"," 151 + "\"hostName\":\"user.android.com\"}]"; 152 EasyMock.expect(mMockProvider.getBuild()) 153 .andReturn(createFakeBuild(createBasicResults(), true)); 154 mContext.addAllocatedDevice(ConfigurationDef.DEFAULT_DEVICE_NAME, mMockDevice); 155 156 EasyMock.replay(mMockDevice, mMockProvider); 157 mLoader.init(); 158 assertEquals("cts -m CtsGesture --skip-all-system-status-check", mLoader.getCommandLine()); 159 CollectingTestListener listener = mLoader.loadPreviousResults(); 160 assertNotNull(listener); 161 IConfiguration config = new Configuration("name", "desc"); 162 assertEquals(0, config.getTargetPreparers().size()); 163 mLoader.customizeConfiguration(config); 164 // A special preparer was added for fingerprint 165 assertEquals(1, config.getTargetPreparers().size()); 166 ITargetPreparer preparer = config.getTargetPreparers().get(0); 167 assertTrue(preparer instanceof BuildFingerPrintPreparer); 168 assertEquals("testfingerprint", 169 ((BuildFingerPrintPreparer) preparer).getExpectedFingerprint()); 170 String runHistory = 171 config.getCommandOptions().getInvocationData().getUniqueMap().get(RUN_HISTORY_KEY); 172 assertEquals(EXPECTED_RUN_HISTORY, runHistory); 173 EasyMock.verify(mMockDevice, mMockProvider); 174 } 175 176 /** Test that the loader can correctly provide the run history back. */ 177 @Test testReloadTests_withRunHistory()178 public void testReloadTests_withRunHistory() throws Exception { 179 final String RUN_HISTORY_1 = 180 "{\"startTime\":1000000000000," 181 + "\"endTime\":1000000010000," 182 + "\"passedTests\":10," 183 + "\"failedTests\":5," 184 + "\"commandLineArgs\":\"cts -m CtsGesture --skip-all-system-status-check\"," 185 + "\"hostName\":\"user1.android.com\"}"; 186 final String RUN_HISTORY_2 = 187 "{\"startTime\":1530218251501," 188 + "\"endTime\":1530218261061," 189 + "\"passedTests\":0," 190 + "\"failedTests\":0," 191 + "\"commandLineArgs\":\"cts -m CtsGesture --skip-all-system-status-check " 192 + "--shard-count 5\"," 193 + "\"hostName\":\"user2.android.com\"}"; 194 final String OLD_RUN_HISTORY = String.format("[%s]", RUN_HISTORY_1); 195 final String EXPECTED_RUN_HISTORY = String.format("[%s,%s]", RUN_HISTORY_1, RUN_HISTORY_2); 196 mContext.addInvocationAttribute(RUN_HISTORY_KEY, OLD_RUN_HISTORY); 197 EasyMock.expect(mMockProvider.getBuild()) 198 .andReturn(createFakeBuild(createResultsWithRunHistory(), false)); 199 mContext.addAllocatedDevice(ConfigurationDef.DEFAULT_DEVICE_NAME, mMockDevice); 200 EasyMock.replay(mMockDevice, mMockProvider); 201 202 mLoader.init(); 203 IConfiguration config = new Configuration("name", "desc"); 204 mLoader.customizeConfiguration(config); 205 String runHistory = 206 config.getCommandOptions().getInvocationData().getUniqueMap().get(RUN_HISTORY_KEY); 207 208 assertEquals(EXPECTED_RUN_HISTORY, runHistory); 209 EasyMock.verify(mMockDevice, mMockProvider); 210 } 211 createFakeBuild(String resultContent, boolean index)212 private IBuildInfo createFakeBuild(String resultContent, boolean index) throws Exception { 213 DeviceBuildInfo build = new DeviceBuildInfo(); 214 build.addBuildAttribute(CompatibilityBuildHelper.SUITE_NAME, "CTS"); 215 mRootDir = FileUtil.createTempDir("cts-root-dir"); 216 new File(mRootDir, "android-cts/results/").mkdirs(); 217 build.addBuildAttribute(CompatibilityBuildHelper.ROOT_DIR, mRootDir.getAbsolutePath()); 218 // Create fake result dir 219 long time = System.currentTimeMillis(); 220 build.addBuildAttribute(CompatibilityBuildHelper.START_TIME_MS, Long.toString(time)); 221 new CompatibilityBuildHelper(build).getResultDir().mkdirs(); 222 // Populate a test_results.xml 223 File testResult = new File(new CompatibilityBuildHelper(build).getResultDir(), 224 "test_result.xml"); 225 testResult.createNewFile(); 226 // Populate a proto result 227 File protoDir = 228 new File( 229 new CompatibilityBuildHelper(build).getResultDir(), 230 CompatibilityProtoResultReporter.PROTO_DIR); 231 protoDir.mkdir(); 232 if (index) { 233 mProtoFile = new File(protoDir, CompatibilityProtoResultReporter.PROTO_FILE_NAME + "0"); 234 } else { 235 mProtoFile = new File(protoDir, CompatibilityProtoResultReporter.PROTO_FILE_NAME); 236 } 237 TestRecord.Builder builder = TestRecord.newBuilder(); 238 builder.setDescription(Any.pack(mContext.toProto())); 239 builder.build().writeDelimitedTo(new FileOutputStream(mProtoFile)); 240 FileUtil.writeToFile(resultContent, testResult); 241 return build; 242 } 243 createBasicResults()244 private String createBasicResults() { 245 StringBuilder sb = new StringBuilder(); 246 sb.append("<?xml version='1.0' encoding='UTF-8' standalone='no' ?>\n"); 247 sb.append("<?xml-stylesheet type=\"text/xsl\" href=\"compatibility_result.xsl\"?>\n"); 248 sb.append( 249 "<Result start=\"1530218251501\" end=\"1530218261061\" " 250 + "start_display=\"Thu Jun 28 13:37:31 PDT 2018\" " 251 + "end_display=\"Thu Jun 28 13:37:41 PDT 2018\" " 252 + "command_line_args=\"cts -m CtsGesture --skip-all-system-status-check\" " 253 + "suite_name=\"CTS\" suite_version=\"9.0_r1\" " 254 + "suite_plan=\"cts\" suite_build_number=\"8888\" report_version=\"5.0\" " 255 + "devices=\"HT6570300047\" " 256 + "host_name=\"user.android.com\">\n"); 257 sb.append( 258 " <Build command_line_args=\"cts -m CtsGesture --skip-all-system-status-check\"" 259 + " build_vendor_fingerprint=\"vendorFingerprint\" " 260 + " build_reference_fingerprint=\"\" " 261 + " build_fingerprint=\"testfingerprint\"/>\n"); 262 // Summary 263 sb.append(" <Summary pass=\"0\" failed=\"0\" modules_done=\"2\" modules_total=\"2\" />\n"); 264 // Each module results 265 sb.append(" <Module name=\"CtsGestureTestCases\" abi=\"arm64-v8a\" runtime=\"2776\" " 266 + "done=\"true\" pass=\"0\" total_tests=\"0\" />\n"); 267 sb.append(" <Module name=\"CtsGestureTestCases\" abi=\"armeabi-v7a\" runtime=\"2776\" " 268 + "done=\"true\" pass=\"0\" total_tests=\"0\" />\n"); 269 // End 270 sb.append("</Result>"); 271 return sb.toString(); 272 } 273 createResultsWithRunHistory()274 private String createResultsWithRunHistory() { 275 // This method is the same as createBasicResults() except that it contains run history. 276 StringBuilder sb = new StringBuilder(); 277 sb.append("<?xml version='1.0' encoding='UTF-8' standalone='no' ?>\n"); 278 sb.append("<?xml-stylesheet type=\"text/xsl\" href=\"compatibility_result.xsl\"?>\n"); 279 sb.append( 280 "<Result start=\"1530218251501\" end=\"1530218261061\" " 281 + "start_display=\"Thu Jun 28 13:37:31 PDT 2018\" " 282 + "end_display=\"Thu Jun 28 13:37:41 PDT 2018\" " 283 + "command_line_args=\"cts -m CtsGesture --skip-all-system-status-check " 284 + "--shard-count 5\" " 285 + "suite_name=\"CTS\" suite_version=\"9.0_r1\" " 286 + "suite_plan=\"cts\" suite_build_number=\"8888\" report_version=\"5.0\" " 287 + "devices=\"HT6570300047\" " 288 + "host_name=\"user2.android.com\" >\n"); 289 final String RUN_HISTORY_JSON = 290 "[{'startTime':1000000000000,'endTime':1000000010000," 291 + "'pass':10,'failed':5," 292 + "'commandLineArgs':'cts -m CtsGesture --skip-all-system-status-check'," 293 + "'hostName':'user1.android.com'}]"; 294 sb.append( 295 " <Build command_line_args=\"cts -m CtsGesture --skip-all-system-status-check\"" 296 + " build_vendor_fingerprint=\"vendorFingerprint\" " 297 + " build_reference_fingerprint=\"\" " 298 + " build_fingerprint=\"testfingerprint\"" 299 + " run_history=\"" 300 + RUN_HISTORY_JSON 301 + "\"/>\n"); 302 // Run history 303 sb.append( 304 " <RunHistory>\n" 305 + " <Run start=\"1000000000000\" end=\"1000000010000\" " 306 + "pass=\"10\" failed=\"5\" " 307 + "command_line_args=\"cts -m CtsGesture --skip-all-system-status-check\" " 308 + "hostName=\"user1.android.com\" />\n" 309 + " </RunHistory>\n"); 310 // Summary 311 sb.append(" <Summary pass=\"0\" failed=\"0\" modules_done=\"2\" modules_total=\"2\" />\n"); 312 // Each module results 313 sb.append( 314 " <Module name=\"CtsGestureTestCases\" abi=\"arm64-v8a\" runtime=\"2776\" " 315 + "done=\"true\" pass=\"0\" total_tests=\"0\" />\n"); 316 sb.append( 317 " <Module name=\"CtsGestureTestCases\" abi=\"armeabi-v7a\" runtime=\"2776\" " 318 + "done=\"true\" pass=\"0\" total_tests=\"0\" />\n"); 319 // End 320 sb.append("</Result>"); 321 return sb.toString(); 322 } 323 } 324