1 /*
2  * Copyright (C) 2011 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 package com.android.tradefed.testtype;
18 
19 import com.android.ddmlib.FileListingService;
20 import com.android.ddmlib.Log;
21 import com.android.tradefed.config.Option;
22 import com.android.tradefed.config.OptionClass;
23 import com.android.tradefed.device.DeviceNotAvailableException;
24 import com.android.tradefed.device.IFileEntry;
25 import com.android.tradefed.device.ITestDevice;
26 import com.android.tradefed.invoker.TestInformation;
27 import com.android.tradefed.result.ITestInvocationListener;
28 import com.android.tradefed.util.proto.TfMetricProtoUtil;
29 
30 import com.google.common.annotations.VisibleForTesting;
31 
32 import java.util.ArrayList;
33 import java.util.Collection;
34 import java.util.HashMap;
35 import java.util.Map;
36 import java.util.concurrent.TimeUnit;
37 
38 /**
39  * A Test that runs a native benchmark test executable on given device.
40  * <p/>
41  * It uses {@link NativeBenchmarkTestParser} to parse out the average operation time vs delay
42  * between operations those results to the {@link ITestInvocationListener}s.
43  */
44 @OptionClass(alias = "native-benchmark")
45 public class NativeBenchmarkTest implements IDeviceTest, IRemoteTest {
46 
47     private static final String LOG_TAG = "NativeStressTest";
48     static final String DEFAULT_TEST_PATH = "data/nativebenchmark";
49 
50     // The metrics key names to report to listeners
51     static final String AVG_OP_TIME_KEY_PREFIX = "avg-operation-time";
52     static final String ITERATION_KEY = "iterations";
53 
54     private ITestDevice mDevice = null;
55 
56     @Option(name = "native-benchmark-device-path",
57       description="The path on the device where native stress tests are located.")
58     private String mDeviceTestPath = DEFAULT_TEST_PATH;
59 
60     @Option(name = "benchmark-module-name",
61             description="The name of the native benchmark test module to run. " +
62             "If not specified all tests in --native-benchmark-device-path will be run.")
63     private String mTestModule = null;
64 
65     @Option(name = "benchmark-run-name",
66             description="Optional name to pass to test reporters. If unspecified, will use" +
67             "--benchmark-module-name.")
68     private String mReportRunName = null;
69 
70     @Option(name = "iterations",
71             description="The number of benchmark test iterations per run.")
72     private int mNumIterations = 1000;
73 
74     @Option(name = "delay-per-run",
75             description="The delay between each benchmark iteration, in micro seconds." +
76                 "Multiple values may be given to specify multiple runs with different delay values.")
77     // TODO: change units to seconds for consistency with native benchmark module input
78     private Collection<Integer> mDelays = new ArrayList<Integer>();
79 
80     @Option(name = "max-run-time", description =
81          "The maximum time to allow for one benchmark run in ms.")
82     private int mMaxRunTime = 5 * 60 * 1000;
83 
84     @Option(name = "server-cpu",
85             description="Optionally specify a server cpu.")
86     private int mServerCpu = 1;
87 
88     @Option(name = "client-cpu",
89             description="Optionally specify a client cpu.")
90     private int mClientCpu = 1;
91 
92     @Option(name = "max-cpu-freq",
93             description="Flag to force device cpu to run at maximum frequency.")
94     private boolean mMaxCpuFreq = false;
95 
96 
97     // TODO: consider sharing code with {@link GTest} and {@link NativeStressTest}
98 
99     /**
100      * {@inheritDoc}
101      */
102     @Override
setDevice(ITestDevice device)103     public void setDevice(ITestDevice device) {
104         mDevice = device;
105     }
106 
107     /**
108      * {@inheritDoc}
109      */
110     @Override
getDevice()111     public ITestDevice getDevice() {
112         return mDevice;
113     }
114 
115     /**
116      * Set the Android native benchmark test module to run.
117      *
118      * @param moduleName The name of the native test module to run
119      */
setModuleName(String moduleName)120     public void setModuleName(String moduleName) {
121         mTestModule = moduleName;
122     }
123 
124     /**
125      * Get the Android native benchmark test module to run.
126      *
127      * @return the name of the native test module to run, or null if not set
128      */
getModuleName()129     public String getModuleName() {
130         return mTestModule;
131     }
132 
133     /**
134      * Set the number of iterations to execute per run
135      */
setNumIterations(int iterations)136     void setNumIterations(int iterations) {
137         mNumIterations = iterations;
138     }
139 
140     /**
141      * Set the delay values per run
142      */
addDelaysPerRun(Collection<Integer> delays)143     void addDelaysPerRun(Collection<Integer> delays) {
144         mDelays.addAll(delays);
145     }
146 
147     /**
148      * Gets the path where native benchmark tests live on the device.
149      *
150      * @return The path on the device where the native tests live.
151      */
152     @VisibleForTesting
getTestPath()153     String getTestPath() {
154         StringBuilder testPath = new StringBuilder(mDeviceTestPath);
155         if (mTestModule != null) {
156             testPath.append(FileListingService.FILE_SEPARATOR);
157             testPath.append(mTestModule);
158         }
159         return testPath.toString();
160     }
161 
162     /**
163      * Executes all native benchmark tests in a folder as well as in all subfolders recursively.
164      *
165      * @param rootEntry The root folder to begin searching for native tests
166      * @param testDevice The device to run tests on
167      * @param listener the run listener
168      * @throws DeviceNotAvailableException
169      */
170     @VisibleForTesting
doRunAllTestsInSubdirectory( IFileEntry rootEntry, ITestDevice testDevice, ITestInvocationListener listener)171     void doRunAllTestsInSubdirectory(
172             IFileEntry rootEntry, ITestDevice testDevice, ITestInvocationListener listener)
173             throws DeviceNotAvailableException {
174 
175         if (rootEntry.isDirectory()) {
176             // recursively run tests in all subdirectories
177             for (IFileEntry childEntry : rootEntry.getChildren(true)) {
178                 doRunAllTestsInSubdirectory(childEntry, testDevice, listener);
179             }
180         } else {
181             // assume every file is a valid benchmark test binary.
182             // use name of file as run name
183             String runName = (mReportRunName == null ? rootEntry.getName() : mReportRunName);
184             String fullPath = rootEntry.getFullEscapedPath();
185             if (mDelays.size() == 0) {
186                 // default to one run with no delay
187                 mDelays.add(0);
188             }
189 
190             // force file to be executable
191             testDevice.executeShellCommand(String.format("chmod 755 %s", fullPath));
192             long startTime = System.currentTimeMillis();
193 
194             listener.testRunStarted(runName, 0);
195             Map<String, String> metricMap = new HashMap<String, String>();
196             metricMap.put(ITERATION_KEY, Integer.toString(mNumIterations));
197             try {
198                 for (Integer delay : mDelays) {
199                     NativeBenchmarkTestParser resultParser = createResultParser(runName);
200                     // convert delay to seconds
201                     double delayFloat = ((double)delay)/1000000;
202                     Log.i(LOG_TAG, String.format("Running %s for %d iterations with delay %f",
203                             rootEntry.getName(), mNumIterations, delayFloat));
204                     String cmd = String.format("%s -n %d -d %f -c %d -s %d", fullPath,
205                             mNumIterations, delayFloat, mClientCpu, mServerCpu);
206                     Log.i(LOG_TAG, String.format("Running native benchmark test on %s: %s",
207                             mDevice.getSerialNumber(), cmd));
208                     testDevice.executeShellCommand(cmd, resultParser,
209                             mMaxRunTime, TimeUnit.MILLISECONDS, 0);
210                     addMetric(metricMap, resultParser, delay);
211                 }
212                 // TODO: is catching exceptions, and reporting testRunFailed necessary?
213             } finally {
214                 final long elapsedTime = System.currentTimeMillis() - startTime;
215                 listener.testRunEnded(elapsedTime, TfMetricProtoUtil.upgradeConvert(metricMap));
216             }
217         }
218     }
219 
220     /**
221      * Adds the operation time metric for a run with given delay
222      *
223      * @param metricMap
224      * @param resultParser
225      * @param delay
226      */
addMetric(Map<String, String> metricMap, NativeBenchmarkTestParser resultParser, Integer delay)227     private void addMetric(Map<String, String> metricMap, NativeBenchmarkTestParser resultParser,
228             Integer delay) {
229         String metricKey = String.format("%s-delay%d", AVG_OP_TIME_KEY_PREFIX, delay);
230         // temporarily convert seconds to microseconds, as some reporters cannot handle small values
231         metricMap.put(metricKey, Double.toString(resultParser.getAvgOperationTime()*1000000));
232     }
233 
234     /**
235      * Factory method for creating a {@link NativeBenchmarkTestParser} that parses test output
236      * <p/>
237      * Exposed so unit tests can mock.
238      *
239      * @param runName
240      * @return a {@link NativeBenchmarkTestParser}
241      */
createResultParser(String runName)242     NativeBenchmarkTestParser createResultParser(String runName) {
243         return new NativeBenchmarkTestParser(runName);
244     }
245 
246     /** {@inheritDoc} */
247     @Override
run(TestInformation testInfo, ITestInvocationListener listener)248     public void run(TestInformation testInfo, ITestInvocationListener listener)
249             throws DeviceNotAvailableException {
250         if (mDevice == null) {
251             throw new IllegalArgumentException("Device has not been set");
252         }
253 
254         String testPath = getTestPath();
255         IFileEntry nativeTestDirectory = mDevice.getFileEntry(testPath);
256         if (nativeTestDirectory == null) {
257             Log.w(LOG_TAG, String.format("Could not find native benchmark test directory %s in %s!",
258                     testPath, mDevice.getSerialNumber()));
259             return;
260         }
261         if (mMaxCpuFreq) {
262             mDevice.executeShellCommand(
263                     "cat /sys/devices/system/cpu/cpu0/cpufreq/cpuinfo_max_freq > " +
264                     "/sys/devices/system/cpu/cpu0/cpufreq/scaling_min_freq");
265         }
266         doRunAllTestsInSubdirectory(nativeTestDirectory, mDevice, listener);
267         if (mMaxCpuFreq) {
268             // revert to normal
269             mDevice.executeShellCommand(
270                     "cat /sys/devices/system/cpu/cpu0/cpufreq/cpuinfo_min_freq > " +
271                     "/sys/devices/system/cpu/cpu0/cpufreq/scaling_min_freq");
272         }
273 
274     }
275 }
276