1 /* 2 * Copyright (C) 2014 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package com.android.graphics.tests; 18 19 import com.android.tradefed.config.Option; 20 import com.android.tradefed.config.Option.Importance; 21 import com.android.tradefed.device.CollectingOutputReceiver; 22 import com.android.tradefed.device.DeviceNotAvailableException; 23 import com.android.tradefed.device.ITestDevice; 24 import com.android.tradefed.log.LogUtil.CLog; 25 import com.android.tradefed.result.ITestInvocationListener; 26 import com.android.tradefed.testtype.IDeviceTest; 27 import com.android.tradefed.testtype.IRemoteTest; 28 import com.android.tradefed.util.AbiFormatter; 29 import com.android.tradefed.util.RunUtil; 30 import com.android.tradefed.util.proto.TfMetricProtoUtil; 31 32 import org.junit.Assert; 33 34 import java.util.HashMap; 35 import java.util.Map; 36 import java.util.concurrent.TimeUnit; 37 38 /** 39 * Test Runner for graphics Flatland Benchmark test. 40 * 41 * <p>Flatland test is a benchmark for measuring GPU performance in various 2D UI rendering and 42 * window composition scenarios. 43 * 44 * <p>Since it is measuring the hardware performance, the test should be executed in as consistent 45 * and static an environment as possible. 46 * 47 * <ul> 48 * <li>The display should be turned off and background service should be stopped before running 49 * the benchmark. Running 'adb shell stop' is probably sufficient for this, but if there are 50 * device specific background services that consume much CPU cycles, memory bandwidth, or 51 * might otherwise interfere with GPU rendering, those should be stopped as well 52 * <li>All relevant hardware clocks should be locked at particular frequency when running the 53 * test. 54 * </ul> 55 * 56 * <p>If running the benchmark with clocks locked causes thermal throttling, set option 57 * "--sleep-time" to 10 to 50 (ms) to insert sleep between each benchmark sample run. 58 * 59 * <p>Output interpretation: For each test case, the expected time in milliseconds that a single 60 * frame of the scenario takes to complete will be printed out. Four types of values could 61 * displayed: 62 * 63 * <ul> 64 * <li>fast - frames of the scenarios are completed too fast to be reliably benchmarked. This 65 * corresponds to frame time less than 3 ms. The scenario was skipped. "0" will be posted into 66 * the dashboard. 67 * <li>slow - frame time is too long, normally orver 50 ms. The scenario was skipped. "1000" will 68 * be posted into the dashboard. 69 * <li>varies - frame time was not stable. rerun the test to get a stable results. If that results 70 * show repeatedly, something is wrong with the environment, signal to file a bug. 71 * <li>decimal number - frame time for the scenarios are measured. 72 * </ul> 73 */ 74 public class FlatlandTest implements IDeviceTest, IRemoteTest { 75 76 private static final long SHELL_TIMEOUT = 30 * 60 * 1000; 77 private static final String COMMAND = "flatland|#ABI32#|"; 78 private static final String FIRST_LINE = "cmdline:"; 79 private static final String TITLE = "Scenario"; 80 private static final long START_TIMER = 2 * 60 * 1000; // 2 minutes 81 private static final String RESULT_FAST = "fast"; 82 private static final String RESULT_SLOW = "slow"; 83 private static final String RESULT_VARIES = "varies"; 84 85 private ITestDevice mTestDevice = null; 86 // HashMap to store results for 87 public Map<String, String> mResultMap = new HashMap<String, String>(); 88 89 @Option(name = "ru-key", description = "Reporting unit key to use when posting results") 90 private String mRuKey = "flatland"; 91 92 @Option(name = "run-path", description = "path for the binary") 93 private String mRunPath = "/data/local/tmp/"; 94 95 @Option( 96 name = "sleep-time", 97 description = 98 "sleep for N ms between samples, set to 10 - 50 ms if the locked CPU" 99 + " frequency causes thermal throttle.") 100 private int mSleepTime = 50; 101 102 @Option(name = "schema-map", description = "map a test case name to a schema key") 103 private Map<String, String> mSchemaMap = new HashMap<String, String>(); 104 105 @Option( 106 name = AbiFormatter.FORCE_ABI_STRING, 107 description = AbiFormatter.FORCE_ABI_DESCRIPTION, 108 importance = Importance.IF_UNSET) 109 private String mForceAbi = null; 110 111 @Override setDevice(ITestDevice testDevice)112 public void setDevice(ITestDevice testDevice) { 113 mTestDevice = testDevice; 114 } 115 116 @Override getDevice()117 public ITestDevice getDevice() { 118 return mTestDevice; 119 } 120 121 @Override run(ITestInvocationListener standardListener)122 public void run(ITestInvocationListener standardListener) throws DeviceNotAvailableException { 123 Assert.assertNotNull(mRunPath); 124 RunUtil.getDefault().sleep(START_TIMER); 125 126 // execute test 127 StringBuilder cmd = new StringBuilder(); 128 cmd.append(mRunPath); 129 cmd.append(COMMAND); 130 if (mSleepTime > 0) { 131 cmd.append(" -s "); 132 cmd.append(mSleepTime); 133 } 134 standardListener.testRunStarted(mRuKey, 1); 135 long start = System.currentTimeMillis(); 136 CollectingOutputReceiver receiver = new CollectingOutputReceiver(); 137 mTestDevice.executeShellCommand( 138 AbiFormatter.formatCmdForAbi(cmd.toString(), mForceAbi), 139 receiver, 140 SHELL_TIMEOUT, 141 TimeUnit.MILLISECONDS, 142 2); 143 String result = receiver.getOutput(); 144 if (result == null) { 145 CLog.v("no test results returned. Test failed?"); 146 return; 147 } 148 // parse results and report metrics 149 parseResult(result); 150 standardListener.testRunEnded( 151 (System.currentTimeMillis() - start), TfMetricProtoUtil.upgradeConvert(mResultMap)); 152 } 153 154 /** Parse results returned from running the benchmark */ parseResult(String result)155 public void parseResult(String result) { 156 String[] lines = result.split(System.getProperty("line.separator")); 157 if (lines.length <= 0) { 158 return; 159 } 160 for (int i = 0; i < lines.length; i++) { 161 if (!lines[i].contains(FIRST_LINE) && !lines[i].contains(TITLE)) { 162 // skip the first two lines 163 String[] items = lines[i].trim().split("\\|"); 164 if (items.length == 3) { 165 String schemaKey = String.format("%s %s", items[0].trim(), items[1].trim()); 166 if (mSchemaMap.get(schemaKey) != null) { 167 // get the mapped schema key if there is any 168 schemaKey = mSchemaMap.get(schemaKey); 169 } 170 String renderTime = items[2].trim(); 171 if (renderTime != null) { 172 if (renderTime.equals(RESULT_FAST)) { 173 mResultMap.put(schemaKey, "0"); 174 } else if (renderTime.equals(RESULT_SLOW)) { 175 mResultMap.put(schemaKey, "1000"); 176 } else if (renderTime.equals(RESULT_VARIES)) { 177 mResultMap.put(schemaKey, "-1"); 178 } else { 179 mResultMap.put(schemaKey, renderTime); 180 } 181 } 182 } 183 } 184 } 185 } 186 } 187