1 /*
2  * Copyright (C) 2016 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 package android.perftests.utils;
18 
19 import android.app.Activity;
20 import android.app.Instrumentation;
21 import android.os.Bundle;
22 import android.os.Debug;
23 import android.util.Log;
24 
25 import androidx.test.InstrumentationRegistry;
26 
27 import java.io.File;
28 import java.util.ArrayList;
29 import java.util.concurrent.TimeUnit;
30 
31 /**
32  * Provides a benchmark framework.
33  *
34  * Example usage:
35  * // Executes the code while keepRunning returning true.
36  *
37  * public void sampleMethod() {
38  *     BenchmarkState state = new BenchmarkState();
39  *
40  *     int[] src = new int[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 };
41  *     while (state.keepRunning()) {
42  *         int[] dest = new int[src.length];
43  *         System.arraycopy(src, 0, dest, 0, src.length);
44  *     }
45  *     System.out.println(state.summaryLine());
46  * }
47  */
48 public final class BenchmarkState {
49 
50     private static final String TAG = "BenchmarkState";
51     private static final boolean ENABLE_PROFILING = false;
52 
53     private static final int NOT_STARTED = 0;  // The benchmark has not started yet.
54     private static final int WARMUP = 1; // The benchmark is warming up.
55     private static final int RUNNING = 2;  // The benchmark is running.
56     private static final int FINISHED = 3;  // The benchmark has stopped.
57 
58     private int mState = NOT_STARTED;  // Current benchmark state.
59 
60     private static final long WARMUP_DURATION_NS = ms2ns(250); // warm-up for at least 250ms
61     private static final int WARMUP_MIN_ITERATIONS = 16; // minimum iterations to warm-up for
62 
63     // TODO: Tune these values.
64     private static final long TARGET_TEST_DURATION_NS = ms2ns(500); // target testing for 500 ms
65     private static final int MAX_TEST_ITERATIONS = 1000000;
66     private static final int MIN_TEST_ITERATIONS = 10;
67     private static final int REPEAT_COUNT = 5;
68 
69     private long mStartTimeNs = 0;  // Previously captured System.nanoTime().
70     private boolean mPaused;
71     private long mPausedTimeNs = 0; // The System.nanoTime() when the pauseTiming() is called.
72     private long mPausedDurationNs = 0;  // The duration of paused state in nano sec.
73 
74     private int mIteration = 0;
75     private int mMaxIterations = 0;
76 
77     private int mRepeatCount = 0;
78 
79     // Statistics. These values will be filled when the benchmark has finished.
80     // The computation needs double precision, but long int is fine for final reporting.
81     private Stats mStats;
82 
83     // Individual duration in nano seconds.
84     private ArrayList<Long> mResults = new ArrayList<>();
85 
ms2ns(long ms)86     private static final long ms2ns(long ms) {
87         return TimeUnit.MILLISECONDS.toNanos(ms);
88     }
89 
90     // Stops the benchmark timer.
91     // This method can be called only when the timer is running.
pauseTiming()92     public void pauseTiming() {
93         if (mPaused) {
94             throw new IllegalStateException(
95                     "Unable to pause the benchmark. The benchmark has already paused.");
96         }
97         mPausedTimeNs = System.nanoTime();
98         mPaused = true;
99     }
100 
101     // Starts the benchmark timer.
102     // This method can be called only when the timer is stopped.
resumeTiming()103     public void resumeTiming() {
104         if (!mPaused) {
105             throw new IllegalStateException(
106                     "Unable to resume the benchmark. The benchmark is already running.");
107         }
108         mPausedDurationNs += System.nanoTime() - mPausedTimeNs;
109         mPausedTimeNs = 0;
110         mPaused = false;
111     }
112 
beginWarmup()113     private void beginWarmup() {
114         mStartTimeNs = System.nanoTime();
115         mIteration = 0;
116         mState = WARMUP;
117     }
118 
beginBenchmark(long warmupDuration, int iterations)119     private void beginBenchmark(long warmupDuration, int iterations) {
120         if (ENABLE_PROFILING) {
121             File f = new File(InstrumentationRegistry.getContext().getDataDir(), "benchprof");
122             Log.d(TAG, "Tracing to: " + f.getAbsolutePath());
123             Debug.startMethodTracingSampling(f.getAbsolutePath(), 16 * 1024 * 1024, 100);
124         }
125         mMaxIterations = (int) (TARGET_TEST_DURATION_NS / (warmupDuration / iterations));
126         mMaxIterations = Math.min(MAX_TEST_ITERATIONS,
127                 Math.max(mMaxIterations, MIN_TEST_ITERATIONS));
128         mPausedDurationNs = 0;
129         mIteration = 0;
130         mRepeatCount = 0;
131         mState = RUNNING;
132         mStartTimeNs = System.nanoTime();
133     }
134 
startNextTestRun()135     private boolean startNextTestRun() {
136         final long currentTime = System.nanoTime();
137         mResults.add((currentTime - mStartTimeNs - mPausedDurationNs) / mMaxIterations);
138         mRepeatCount++;
139         if (mRepeatCount >= REPEAT_COUNT) {
140             if (ENABLE_PROFILING) {
141                 Debug.stopMethodTracing();
142             }
143             mStats = new Stats(mResults);
144             mState = FINISHED;
145             return false;
146         }
147         mPausedDurationNs = 0;
148         mIteration = 0;
149         mStartTimeNs = System.nanoTime();
150         return true;
151     }
152 
153     /**
154      * Judges whether the benchmark needs more samples.
155      *
156      * For the usage, see class comment.
157      */
keepRunning()158     public boolean keepRunning() {
159         switch (mState) {
160             case NOT_STARTED:
161                 beginWarmup();
162                 return true;
163             case WARMUP:
164                 mIteration++;
165                 // Only check nanoTime on every iteration in WARMUP since we
166                 // don't yet have a target iteration count.
167                 final long duration = System.nanoTime() - mStartTimeNs;
168                 if (mIteration >= WARMUP_MIN_ITERATIONS && duration >= WARMUP_DURATION_NS) {
169                     beginBenchmark(duration, mIteration);
170                 }
171                 return true;
172             case RUNNING:
173                 mIteration++;
174                 if (mIteration >= mMaxIterations) {
175                     return startNextTestRun();
176                 }
177                 if (mPaused) {
178                     throw new IllegalStateException(
179                             "Benchmark step finished with paused state. " +
180                             "Resume the benchmark before finishing each step.");
181                 }
182                 return true;
183             case FINISHED:
184                 throw new IllegalStateException("The benchmark has finished.");
185             default:
186                 throw new IllegalStateException("The benchmark is in unknown state.");
187         }
188     }
189 
mean()190     private long mean() {
191         if (mState != FINISHED) {
192             throw new IllegalStateException("The benchmark hasn't finished");
193         }
194         return (long) mStats.getMean();
195     }
196 
median()197     private long median() {
198         if (mState != FINISHED) {
199             throw new IllegalStateException("The benchmark hasn't finished");
200         }
201         return mStats.getMedian();
202     }
203 
min()204     private long min() {
205         if (mState != FINISHED) {
206             throw new IllegalStateException("The benchmark hasn't finished");
207         }
208         return mStats.getMin();
209     }
210 
standardDeviation()211     private long standardDeviation() {
212         if (mState != FINISHED) {
213             throw new IllegalStateException("The benchmark hasn't finished");
214         }
215         return (long) mStats.getStandardDeviation();
216     }
217 
summaryLine()218     private String summaryLine() {
219         StringBuilder sb = new StringBuilder();
220         sb.append("Summary: ");
221         sb.append("median=").append(median()).append("ns, ");
222         sb.append("mean=").append(mean()).append("ns, ");
223         sb.append("min=").append(min()).append("ns, ");
224         sb.append("sigma=").append(standardDeviation()).append(", ");
225         sb.append("iteration=").append(mResults.size()).append(", ");
226         // print out the first few iterations' number for double checking.
227         int sampleNumber = Math.min(mResults.size(), 16);
228         for (int i = 0; i < sampleNumber; i++) {
229             sb.append("No ").append(i).append(" result is ").append(mResults.get(i)).append(", ");
230         }
231         return sb.toString();
232     }
233 
sendFullStatusReport(Instrumentation instrumentation, String key)234     public void sendFullStatusReport(Instrumentation instrumentation, String key) {
235         Log.i(TAG, key + summaryLine());
236         Bundle status = new Bundle();
237         status.putLong(key + "_median", median());
238         status.putLong(key + "_mean", mean());
239         status.putLong(key + "_min", min());
240         status.putLong(key + "_standardDeviation", standardDeviation());
241         instrumentation.sendStatus(Activity.RESULT_OK, status);
242     }
243 }
244