1 /* 2 * Copyright (C) 2018 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package android.platform.test.microbenchmark; 17 18 import static com.google.common.truth.Truth.assertThat; 19 20 import android.os.Bundle; 21 import android.platform.test.rule.TracePointRule; 22 23 import org.junit.After; 24 import org.junit.Before; 25 import org.junit.Test; 26 import org.junit.rules.TestRule; 27 import org.junit.runner.Description; 28 import org.junit.runner.JUnitCore; 29 import org.junit.runner.Result; 30 import org.junit.runner.RunWith; 31 import org.junit.runners.JUnit4; 32 import org.junit.runners.model.InitializationError; 33 import org.junit.runners.model.Statement; 34 35 import java.util.ArrayList; 36 import java.util.List; 37 38 /** 39 * Unit tests for the {@link Microbenchmark} runner. 40 */ 41 @RunWith(JUnit4.class) 42 public final class MicrobenchmarkTest { 43 /** 44 * Tests that iterations are respected for microbenchmark tests. 45 */ 46 @Test testIterationCount()47 public void testIterationCount() throws InitializationError { 48 Bundle args = new Bundle(); 49 args.putString("iterations", "10"); 50 Microbenchmark microbench = new Microbenchmark(BasicTest.class, args); 51 assertThat(microbench.testCount()).isEqualTo(10); 52 } 53 54 public static class BasicTest { 55 @Test doNothingTest()56 public void doNothingTest() { } 57 } 58 59 /** 60 * Tests that {@link TracePointRule} and {@link TightMethodRule}s are properly ordered. 61 * 62 * Before --> TightBefore --> Trace (begin) --> Test --> Trace(end) --> TightAfter --> After 63 */ 64 @Test testFeatureExecutionOrder()65 public void testFeatureExecutionOrder() throws InitializationError { 66 LoggingMicrobenchmark loggingRunner = new LoggingMicrobenchmark(LoggingTest.class); 67 loggingRunner.setOperationLog(new ArrayList<String>()); 68 Result result = new JUnitCore().run(loggingRunner); 69 assertThat(result.wasSuccessful()).isTrue(); 70 assertThat(loggingRunner.getOperationLog()).containsExactly( 71 "before", 72 "tight before", 73 "begin: testMethod(" 74 + "android.platform.test.microbenchmark.MicrobenchmarkTest$LoggingTest)", 75 "test", 76 "end", 77 "tight after", 78 "after") 79 .inOrder(); 80 } 81 82 /** 83 * Test iterations number are added to the test name with default suffix. 84 * 85 * Before --> TightBefore --> Trace (begin) with suffix @1 --> Test --> Trace(end) 86 * --> TightAfter --> After --> Before --> TightBefore --> Trace (begin) with suffix @2 87 * --> Test --> Trace(end) --> TightAfter --> After 88 */ 89 @Test testMultipleIterationsWithRename()90 public void testMultipleIterationsWithRename() throws InitializationError { 91 Bundle args = new Bundle(); 92 args.putString("iterations", "2"); 93 args.putString("rename-iterations", "true"); 94 LoggingMicrobenchmark loggingRunner = new LoggingMicrobenchmark(LoggingTest.class, args); 95 loggingRunner.setOperationLog(new ArrayList<String>()); 96 Result result = new JUnitCore().run(loggingRunner); 97 assertThat(result.wasSuccessful()).isTrue(); 98 assertThat(loggingRunner.getOperationLog()).containsExactly( 99 "before", 100 "tight before", 101 "begin: testMethod(" 102 + "android.platform.test.microbenchmark.MicrobenchmarkTest$LoggingTest$1)", 103 "test", 104 "end", 105 "tight after", 106 "after", 107 "before", 108 "tight before", 109 "begin: testMethod(" 110 + "android.platform.test.microbenchmark.MicrobenchmarkTest$LoggingTest$2)", 111 "test", 112 "end", 113 "tight after", 114 "after") 115 .inOrder(); 116 } 117 118 /** 119 * Test iterations number are added to the test name with custom suffix. 120 * 121 * Before --> TightBefore --> Trace (begin) with suffix --1 --> Test --> Trace(end) 122 * --> TightAfter --> After --> Before --> TightBefore --> Trace (begin) with suffix --2 123 * --> Test --> Trace(end) --> TightAfter --> After 124 */ 125 @Test testMultipleIterationsWithDifferentSuffix()126 public void testMultipleIterationsWithDifferentSuffix() throws InitializationError { 127 Bundle args = new Bundle(); 128 args.putString("iterations", "2"); 129 args.putString("rename-iterations", "true"); 130 args.putString("iteration-separator", "--"); 131 LoggingMicrobenchmark loggingRunner = new LoggingMicrobenchmark(LoggingTest.class, args); 132 loggingRunner.setOperationLog(new ArrayList<String>()); 133 Result result = new JUnitCore().run(loggingRunner); 134 assertThat(result.wasSuccessful()).isTrue(); 135 assertThat(loggingRunner.getOperationLog()).containsExactly( 136 "before", 137 "tight before", 138 "begin: testMethod(" 139 + "android.platform.test.microbenchmark.MicrobenchmarkTest$LoggingTest--1)", 140 "test", 141 "end", 142 "tight after", 143 "after", 144 "before", 145 "tight before", 146 "begin: testMethod(" 147 + "android.platform.test.microbenchmark.MicrobenchmarkTest$LoggingTest--2)", 148 "test", 149 "end", 150 "tight after", 151 "after") 152 .inOrder(); 153 } 154 155 /** 156 * Test iteration number are not added to the test name when explictly disabled. 157 * 158 * Before --> TightBefore --> Trace (begin) --> Test --> Trace(end) --> TightAfter 159 * --> After 160 */ 161 @Test testMultipleIterationsWithoutRename()162 public void testMultipleIterationsWithoutRename() throws InitializationError { 163 Bundle args = new Bundle(); 164 args.putString("iterations", "1"); 165 args.putString("rename-iterations", "false"); 166 LoggingMicrobenchmark loggingRunner = new LoggingMicrobenchmark(LoggingTest.class, args); 167 loggingRunner.setOperationLog(new ArrayList<String>()); 168 Result result = new JUnitCore().run(loggingRunner); 169 assertThat(result.wasSuccessful()).isTrue(); 170 assertThat(loggingRunner.getOperationLog()) 171 .containsExactly( 172 "before", 173 "tight before", 174 "begin: testMethod(" 175 + "android.platform.test.microbenchmark.MicrobenchmarkTest" 176 + "$LoggingTest)", 177 "test", 178 "end", 179 "tight after", 180 "after") 181 .inOrder(); 182 } 183 184 /** 185 * Test method iteration will iterate the inner-most test method N times. 186 * 187 * <p>Before --> TightBefore --> Trace (begin) --> Test x N --> Trace(end) --> TightAfter --> 188 * After 189 */ 190 @Test testMultipleMethodIterations()191 public void testMultipleMethodIterations() throws InitializationError { 192 Bundle args = new Bundle(); 193 args.putString("iterations", "1"); 194 args.putString("method-iterations", "10"); 195 args.putString("rename-iterations", "false"); 196 LoggingMicrobenchmark loggingRunner = new LoggingMicrobenchmark(LoggingTest.class, args); 197 loggingRunner.setOperationLog(new ArrayList<String>()); 198 Result result = new JUnitCore().run(loggingRunner); 199 assertThat(result.wasSuccessful()).isTrue(); 200 assertThat(loggingRunner.getOperationLog()) 201 .containsExactly( 202 "before", 203 "tight before", 204 "begin: testMethod(" 205 + "android.platform.test.microbenchmark.MicrobenchmarkTest" 206 + "$LoggingTest)", 207 "test", 208 "test", 209 "test", 210 "test", 211 "test", 212 "test", 213 "test", 214 "test", 215 "test", 216 "test", 217 "end", 218 "tight after", 219 "after") 220 .inOrder(); 221 } 222 223 /** 224 * An extensions of the {@link Microbenchmark} runner that logs the start and end of collecting 225 * traces. It also passes the operation log to the provided test {@code Class}, if it is a 226 * {@link LoggingTest}. This is used for ensuring the proper order for evaluating test {@link 227 * Statement}s. 228 */ 229 public static class LoggingMicrobenchmark extends Microbenchmark { 230 private List<String> mOperationLog; 231 LoggingMicrobenchmark(Class<?> klass)232 public LoggingMicrobenchmark(Class<?> klass) throws InitializationError { 233 super(klass); 234 } 235 LoggingMicrobenchmark(Class<?> klass, Bundle arguments)236 LoggingMicrobenchmark(Class<?> klass, Bundle arguments) throws InitializationError { 237 super(klass, arguments); 238 } 239 createTest()240 protected Object createTest() throws Exception { 241 Object test = super.createTest(); 242 if (test instanceof LoggingTest) { 243 ((LoggingTest)test).setOperationLog(mOperationLog); 244 } 245 return test; 246 } 247 setOperationLog(List<String> log)248 void setOperationLog(List<String> log) { 249 mOperationLog = log; 250 } 251 getOperationLog()252 List<String> getOperationLog() { 253 return mOperationLog; 254 } 255 256 @Override getTracePointRule()257 protected TracePointRule getTracePointRule() { 258 return new LoggingTracePointRule(); 259 } 260 261 class LoggingTracePointRule extends TracePointRule { 262 @Override beginSection(String sectionTag)263 protected void beginSection(String sectionTag) { 264 mOperationLog.add(String.format("begin: %s", sectionTag)); 265 } 266 267 @Override endSection()268 protected void endSection() { 269 mOperationLog.add("end"); 270 } 271 } 272 } 273 274 /** 275 * A test that logs {@link Before}, {@link After}, {@link Test}, and the logging {@link 276 * TightMethodRule} included, used in conjunction with {@link LoggingMicrobenchmark} to 277 * determine all {@link Statement}s are evaluated in the proper order. 278 */ 279 public static class LoggingTest { 280 @Microbenchmark.TightMethodRule 281 public TightRule orderRule = new TightRule(); 282 283 private List<String> mOperationLog; 284 setOperationLog(List<String> log)285 void setOperationLog(List<String> log) { 286 mOperationLog = log; 287 } 288 289 @Before beforeMethod()290 public void beforeMethod() { 291 mOperationLog.add("before"); 292 } 293 294 @Test testMethod()295 public void testMethod() { 296 mOperationLog.add("test"); 297 } 298 299 @After afterMethod()300 public void afterMethod() { 301 mOperationLog.add("after"); 302 } 303 304 class TightRule implements TestRule { 305 @Override apply(Statement base, Description description)306 public Statement apply(Statement base, Description description) { 307 return new Statement() { 308 @Override 309 public void evaluate() throws Throwable { 310 mOperationLog.add("tight before"); 311 base.evaluate(); 312 mOperationLog.add("tight after"); 313 } 314 }; 315 } 316 } 317 } 318 } 319