1 /*
2  * Copyright (C) 2010 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 package com.android.tradefed.testtype;
18 
19 import com.android.ddmlib.FileListingService;
20 import com.android.ddmlib.Log;
21 import com.android.tradefed.config.Option;
22 import com.android.tradefed.config.Option.Importance;
23 import com.android.tradefed.config.OptionClass;
24 import com.android.tradefed.device.DeviceNotAvailableException;
25 import com.android.tradefed.device.IFileEntry;
26 import com.android.tradefed.device.ITestDevice;
27 import com.android.tradefed.invoker.TestInformation;
28 import com.android.tradefed.result.ITestInvocationListener;
29 import com.android.tradefed.util.proto.TfMetricProtoUtil;
30 
31 import java.util.HashMap;
32 import java.util.Map;
33 import java.util.concurrent.TimeUnit;
34 
35 /**
36  * A Test that runs a native stress test executable on given device.
37  * <p/>
38  * It uses {@link NativeStressTestParser} to parse out number of iterations completed and report
39  * those results to the {@link ITestInvocationListener}s.
40  */
41 @OptionClass(alias = "native-stress")
42 public class NativeStressTest implements IDeviceTest, IRemoteTest {
43 
44     private static final String LOG_TAG = "NativeStressTest";
45     static final String DEFAULT_TEST_PATH = "data/nativestresstest";
46 
47     // The metrics key names to report to listeners
48     // TODO: these key names are temporary
49     static final String AVG_ITERATION_TIME_KEY = "avg-iteration-time";
50     static final String ITERATION_KEY = "iterations";
51 
52     private ITestDevice mDevice = null;
53 
54     @Option(name = "native-stress-device-path",
55       description="The path on the device where native stress tests are located.")
56     private String mDeviceTestPath = DEFAULT_TEST_PATH;
57 
58     @Option(name = "stress-module-name",
59             description="The name of the native test module to run. " +
60             "If not specified all tests in --native-stress-device-path will be run.")
61     private String mTestModule = null;
62 
63     @Option(name = "iterations",
64             description="The number of stress test iterations per run.",
65             importance = Importance.IF_UNSET)
66     private Integer mNumIterations = null;
67 
68     @Option(name = "runs",
69             description="The number of stress test runs to perform.")
70     private int mNumRuns = 1;
71 
72     @Option(name = "max-iteration-time", description =
73         "The maximum time to allow for one stress test iteration in ms.")
74     private int mMaxIterationTime = 5 * 60 * 1000;
75 
76     // TODO: consider sharing code with {@link GTest}
77 
78     /**
79      * {@inheritDoc}
80      */
81     @Override
setDevice(ITestDevice device)82     public void setDevice(ITestDevice device) {
83         mDevice = device;
84     }
85 
86     /**
87      * {@inheritDoc}
88      */
89     @Override
getDevice()90     public ITestDevice getDevice() {
91         return mDevice;
92     }
93 
94     /**
95      * Set the Android native stress test module to run.
96      *
97      * @param moduleName The name of the native test module to run
98      */
setModuleName(String moduleName)99     public void setModuleName(String moduleName) {
100         mTestModule = moduleName;
101     }
102 
103     /**
104      * Get the Android native test module to run.
105      *
106      * @return the name of the native test module to run, or null if not set
107      */
getModuleName()108     public String getModuleName() {
109         return mTestModule;
110     }
111 
112     /**
113      * Set the number of iterations to execute per run
114      */
setNumIterations(int iterations)115     void setNumIterations(int iterations) {
116         mNumIterations = iterations;
117     }
118 
119     /**
120      * Set the number of runs to execute
121      */
setNumRuns(int runs)122     void setNumRuns(int runs) {
123         mNumRuns = runs;
124     }
125 
126     /**
127      * Gets the path where native stress tests live on the device.
128      *
129      * @return The path on the device where the native tests live.
130      */
getTestPath()131     private String getTestPath() {
132         StringBuilder testPath = new StringBuilder(mDeviceTestPath);
133         if (mTestModule != null) {
134             testPath.append(FileListingService.FILE_SEPARATOR);
135             testPath.append(mTestModule);
136         }
137         return testPath.toString();
138     }
139 
140     /**
141      * Executes all native stress tests in a folder as well as in all subfolders recursively.
142      *
143      * @param rootEntry The root folder to begin searching for native tests
144      * @param testDevice The device to run tests on
145      * @param listener the run listener
146      * @throws DeviceNotAvailableException
147      */
doRunAllTestsInSubdirectory( IFileEntry rootEntry, ITestDevice testDevice, ITestInvocationListener listener)148     private void doRunAllTestsInSubdirectory(
149             IFileEntry rootEntry, ITestDevice testDevice, ITestInvocationListener listener)
150             throws DeviceNotAvailableException {
151 
152         if (rootEntry.isDirectory()) {
153             // recursively run tests in all subdirectories
154             for (IFileEntry childEntry : rootEntry.getChildren(true)) {
155                 doRunAllTestsInSubdirectory(childEntry, testDevice, listener);
156             }
157         } else {
158             // assume every file is a valid stress test binary.
159             // use name of file as run name
160             NativeStressTestParser resultParser = createResultParser(rootEntry.getName());
161             String fullPath = rootEntry.getFullEscapedPath();
162             Log.i(LOG_TAG, String.format("Running native stress test %s on %s", fullPath,
163                     mDevice.getSerialNumber()));
164             // force file to be executable
165             testDevice.executeShellCommand(String.format("chmod 755 %s", fullPath));
166             int startIteration = 0;
167             int endIteration = mNumIterations - 1;
168             long startTime = System.currentTimeMillis();
169             listener.testRunStarted(resultParser.getRunName(), 0);
170             try {
171                 for (int i = 0; i < mNumRuns; i++) {
172                     Log.i(LOG_TAG, String.format("Running %s for %d iterations",
173                             rootEntry.getName(), mNumIterations));
174                     // -s is start iteration, -e means end iteration
175                     // use maxShellOutputResponseTime to enforce the max iteration time
176                     // it won't be exact, but should be close
177                     testDevice.executeShellCommand(String.format("%s -s %d -e %d", fullPath,
178                             startIteration, endIteration), resultParser,
179                             mMaxIterationTime, TimeUnit.MILLISECONDS, 0);
180                     // iteration count is also used as a random seed value, so want use different
181                     // values for each run
182                     startIteration += mNumIterations;
183                     endIteration += mNumIterations;
184                 }
185                 // TODO: is catching exceptions, and reporting testRunFailed necessary?
186             } finally {
187                 reportTestCompleted(startTime, listener, resultParser);
188             }
189 
190         }
191     }
192 
reportTestCompleted( long startTime, ITestInvocationListener listener, NativeStressTestParser parser)193     private void reportTestCompleted(
194             long startTime, ITestInvocationListener listener, NativeStressTestParser parser) {
195         final long elapsedTime = System.currentTimeMillis() - startTime;
196         int iterationsComplete = parser.getIterationsCompleted();
197         float avgIterationTime = iterationsComplete > 0 ? elapsedTime / iterationsComplete : 0;
198         Map<String, String> metricMap = new HashMap<String, String>(2);
199         Log.i(LOG_TAG, String.format(
200                 "Stress test %s is finished. Num iterations %d, avg time %f ms",
201                 parser.getRunName(), iterationsComplete, avgIterationTime));
202         metricMap.put(ITERATION_KEY, Integer.toString(iterationsComplete));
203         metricMap.put(AVG_ITERATION_TIME_KEY, Float.toString(avgIterationTime));
204         listener.testRunEnded(elapsedTime, TfMetricProtoUtil.upgradeConvert(metricMap));
205     }
206 
207     /**
208      * Factory method for creating a {@link NativeStressTestParser} that parses test output
209      * <p/>
210      * Exposed so unit tests can mock.
211      *
212      * @param runName
213      * @return a {@link NativeStressTestParser}
214      */
createResultParser(String runName)215     NativeStressTestParser createResultParser(String runName) {
216         return new NativeStressTestParser(runName);
217     }
218 
219     /** {@inheritDoc} */
220     @Override
run(TestInformation testInfo, ITestInvocationListener listener)221     public void run(TestInformation testInfo, ITestInvocationListener listener)
222             throws DeviceNotAvailableException {
223         if (mDevice == null) {
224             throw new IllegalArgumentException("Device has not been set");
225         }
226         if (mNumIterations == null || mNumIterations <= 0) {
227             throw new IllegalArgumentException("number of iterations has not been set");
228         }
229 
230         String testPath = getTestPath();
231         IFileEntry nativeTestDirectory = mDevice.getFileEntry(testPath);
232         if (nativeTestDirectory == null) {
233             Log.w(LOG_TAG, String.format("Could not find native stress test directory %s in %s!",
234                     testPath, mDevice.getSerialNumber()));
235             return;
236         }
237         doRunAllTestsInSubdirectory(nativeTestDirectory, mDevice, listener);
238     }
239 }
240