1 /*
2  * Copyright (C) 2015 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the License);
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an AS IS BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 package com.android.benchmark.results;
18 
19 import android.content.ContentValues;
20 import android.content.Context;
21 import android.database.Cursor;
22 import android.database.sqlite.SQLiteDatabase;
23 import android.database.sqlite.SQLiteOpenHelper;
24 import android.view.FrameMetrics;
25 import android.widget.Toast;
26 
27 import org.apache.commons.math.stat.descriptive.DescriptiveStatistics;
28 
29 import java.io.FileWriter;
30 import java.io.IOException;
31 import java.text.DateFormat;
32 import java.util.ArrayList;
33 import java.util.Date;
34 import java.util.HashMap;
35 
36 public class GlobalResultsStore extends SQLiteOpenHelper {
37     private static final int VERSION = 2;
38 
39     private static GlobalResultsStore sInstance;
40     private static final String UI_RESULTS_TABLE = "ui_results";
41 
42     private final Context mContext;
43 
GlobalResultsStore(Context context)44     private GlobalResultsStore(Context context) {
45         super(context, "BenchmarkResults", null, VERSION);
46         mContext = context;
47     }
48 
getInstance(Context context)49     public static GlobalResultsStore getInstance(Context context) {
50         if (sInstance == null) {
51             sInstance = new GlobalResultsStore(context.getApplicationContext());
52         }
53 
54         return sInstance;
55     }
56 
57     @Override
onCreate(SQLiteDatabase sqLiteDatabase)58     public void onCreate(SQLiteDatabase sqLiteDatabase) {
59         sqLiteDatabase.execSQL("CREATE TABLE " + UI_RESULTS_TABLE + " (" +
60                 " _id INTEGER PRIMARY KEY AUTOINCREMENT," +
61                 " name TEXT," +
62                 " run_id INTEGER," +
63                 " iteration INTEGER," +
64                 " timestamp TEXT,"  +
65                 " unknown_delay REAL," +
66                 " input REAL," +
67                 " animation REAL," +
68                 " layout REAL," +
69                 " draw REAL," +
70                 " sync REAL," +
71                 " command_issue REAL," +
72                 " swap_buffers REAL," +
73                 " total_duration REAL," +
74                 " jank_frame BOOLEAN, " +
75                 " device_charging INTEGER);");
76     }
77 
storeRunResults(String testName, int runId, int iteration, UiBenchmarkResult result)78     public void storeRunResults(String testName, int runId, int iteration,
79                                 UiBenchmarkResult result) {
80         SQLiteDatabase db = getWritableDatabase();
81         db.beginTransaction();
82 
83         try {
84             String date = DateFormat.getDateTimeInstance().format(new Date());
85             int jankIndexIndex = 0;
86             int[] sortedJankIndices = result.getSortedJankFrameIndices();
87             int totalFrameCount = result.getTotalFrameCount();
88             for (int frameIdx = 0; frameIdx < totalFrameCount; frameIdx++) {
89                 ContentValues cv = new ContentValues();
90                 cv.put("name", testName);
91                 cv.put("run_id", runId);
92                 cv.put("iteration", iteration);
93                 cv.put("timestamp", date);
94                 cv.put("unknown_delay",
95                         result.getMetricAtIndex(frameIdx, FrameMetrics.UNKNOWN_DELAY_DURATION));
96                 cv.put("input",
97                         result.getMetricAtIndex(frameIdx, FrameMetrics.INPUT_HANDLING_DURATION));
98                 cv.put("animation",
99                         result.getMetricAtIndex(frameIdx, FrameMetrics.ANIMATION_DURATION));
100                 cv.put("layout",
101                         result.getMetricAtIndex(frameIdx, FrameMetrics.LAYOUT_MEASURE_DURATION));
102                 cv.put("draw",
103                         result.getMetricAtIndex(frameIdx, FrameMetrics.DRAW_DURATION));
104                 cv.put("sync",
105                         result.getMetricAtIndex(frameIdx, FrameMetrics.SYNC_DURATION));
106                 cv.put("command_issue",
107                         result.getMetricAtIndex(frameIdx, FrameMetrics.COMMAND_ISSUE_DURATION));
108                 cv.put("swap_buffers",
109                         result.getMetricAtIndex(frameIdx, FrameMetrics.SWAP_BUFFERS_DURATION));
110                 cv.put("total_duration",
111                         result.getMetricAtIndex(frameIdx, FrameMetrics.TOTAL_DURATION));
112                 if (jankIndexIndex < sortedJankIndices.length &&
113                         sortedJankIndices[jankIndexIndex] == frameIdx) {
114                     jankIndexIndex++;
115                     cv.put("jank_frame", true);
116                 } else {
117                     cv.put("jank_frame", false);
118                 }
119                 db.insert(UI_RESULTS_TABLE, null, cv);
120             }
121             db.setTransactionSuccessful();
122             Toast.makeText(mContext, "Score: " + result.getScore()
123                     + " Jank: " + (100 * sortedJankIndices.length) / (float) totalFrameCount + "%",
124                     Toast.LENGTH_LONG).show();
125         } finally {
126             db.endTransaction();
127         }
128 
129     }
130 
loadTestResults(String testName, int runId)131     public ArrayList<UiBenchmarkResult> loadTestResults(String testName, int runId) {
132         SQLiteDatabase db = getReadableDatabase();
133         ArrayList<UiBenchmarkResult> resultList = new ArrayList<>();
134         try {
135             String[] columnsToQuery = new String[] {
136                     "name",
137                     "run_id",
138                     "iteration",
139                     "unknown_delay",
140                     "input",
141                     "animation",
142                     "layout",
143                     "draw",
144                     "sync",
145                     "command_issue",
146                     "swap_buffers",
147                     "total_duration",
148             };
149 
150             Cursor cursor = db.query(
151                     UI_RESULTS_TABLE, columnsToQuery, "run_id=? AND name=?",
152                     new String[] { Integer.toString(runId), testName }, null, null, "iteration");
153 
154             double[] values = new double[columnsToQuery.length - 3];
155 
156             while (cursor.moveToNext()) {
157                 int iteration = cursor.getInt(cursor.getColumnIndexOrThrow("iteration"));
158 
159                 values[0] = cursor.getDouble(
160                         cursor.getColumnIndexOrThrow("unknown_delay"));
161                 values[1] = cursor.getDouble(
162                         cursor.getColumnIndexOrThrow("input"));
163                 values[2] = cursor.getDouble(
164                         cursor.getColumnIndexOrThrow("animation"));
165                 values[3] = cursor.getDouble(
166                         cursor.getColumnIndexOrThrow("layout"));
167                 values[4] = cursor.getDouble(
168                         cursor.getColumnIndexOrThrow("draw"));
169                 values[5] = cursor.getDouble(
170                         cursor.getColumnIndexOrThrow("sync"));
171                 values[6] = cursor.getDouble(
172                         cursor.getColumnIndexOrThrow("command_issue"));
173                 values[7] = cursor.getDouble(
174                         cursor.getColumnIndexOrThrow("swap_buffers"));
175                 values[8] = cursor.getDouble(
176                         cursor.getColumnIndexOrThrow("total_duration"));
177 
178                 UiBenchmarkResult iterationResult;
179                 if (resultList.size() == iteration) {
180                     iterationResult = new UiBenchmarkResult(values);
181                     resultList.add(iteration, iterationResult);
182                 } else {
183                     iterationResult = resultList.get(iteration);
184                     iterationResult.update(values);
185                 }
186             }
187 
188             cursor.close();
189         } finally {
190             db.close();
191         }
192 
193         int total = resultList.get(0).getTotalFrameCount();
194         for (int i = 0; i < total; i++) {
195             System.out.println(""+ resultList.get(0).getMetricAtIndex(0, FrameMetrics.TOTAL_DURATION));
196         }
197 
198         return resultList;
199     }
200 
loadDetailedResults(int runId)201     public HashMap<String, ArrayList<UiBenchmarkResult>> loadDetailedResults(int runId) {
202         SQLiteDatabase db = getReadableDatabase();
203         HashMap<String, ArrayList<UiBenchmarkResult>> results = new HashMap<>();
204         try {
205             String[] columnsToQuery = new String[] {
206                     "name",
207                     "run_id",
208                     "iteration",
209                     "unknown_delay",
210                     "input",
211                     "animation",
212                     "layout",
213                     "draw",
214                     "sync",
215                     "command_issue",
216                     "swap_buffers",
217                     "total_duration",
218             };
219 
220             Cursor cursor = db.query(
221                     UI_RESULTS_TABLE, columnsToQuery, "run_id=?",
222                     new String[] { Integer.toString(runId) }, null, null, "name, iteration");
223 
224             double[] values = new double[columnsToQuery.length - 3];
225             while (cursor.moveToNext()) {
226                 int iteration = cursor.getInt(cursor.getColumnIndexOrThrow("iteration"));
227                 String name = cursor.getString(cursor.getColumnIndexOrThrow("name"));
228                 ArrayList<UiBenchmarkResult> resultList = results.get(name);
229                 if (resultList == null) {
230                     resultList = new ArrayList<>();
231                     results.put(name, resultList);
232                 }
233 
234                 values[0] = cursor.getDouble(
235                         cursor.getColumnIndexOrThrow("unknown_delay"));
236                 values[1] = cursor.getDouble(
237                         cursor.getColumnIndexOrThrow("input"));
238                 values[2] = cursor.getDouble(
239                         cursor.getColumnIndexOrThrow("animation"));
240                 values[3] = cursor.getDouble(
241                         cursor.getColumnIndexOrThrow("layout"));
242                 values[4] = cursor.getDouble(
243                         cursor.getColumnIndexOrThrow("draw"));
244                 values[5] = cursor.getDouble(
245                         cursor.getColumnIndexOrThrow("sync"));
246                 values[6] = cursor.getDouble(
247                         cursor.getColumnIndexOrThrow("command_issue"));
248                 values[7] = cursor.getDouble(
249                         cursor.getColumnIndexOrThrow("swap_buffers"));
250                 values[8] = cursor.getDouble(
251                         cursor.getColumnIndexOrThrow("total_duration"));
252                 values[8] = cursor.getDouble(
253                         cursor.getColumnIndexOrThrow("total_duration"));
254 
255                 UiBenchmarkResult iterationResult;
256                 if (resultList.size() == iteration) {
257                     iterationResult = new UiBenchmarkResult(values);
258                     resultList.add(iterationResult);
259                 } else {
260                     iterationResult = resultList.get(iteration);
261                     iterationResult.update(values);
262                 }
263             }
264 
265             cursor.close();
266         } finally {
267             db.close();
268         }
269 
270         return results;
271     }
272 
exportToCsv()273     public void exportToCsv() throws IOException {
274         String path = mContext.getFilesDir() + "/results-" + System.currentTimeMillis() + ".csv";
275         SQLiteDatabase db = getReadableDatabase();
276 
277         // stats across metrics for each run and each test
278         HashMap<String, DescriptiveStatistics> stats = new HashMap<>();
279 
280         Cursor runIdCursor = db.query(
281                 UI_RESULTS_TABLE, new String[] { "run_id" }, null, null, "run_id", null, null);
282 
283         while (runIdCursor.moveToNext()) {
284 
285             int runId = runIdCursor.getInt(runIdCursor.getColumnIndexOrThrow("run_id"));
286             HashMap<String, ArrayList<UiBenchmarkResult>> detailedResults =
287                     loadDetailedResults(runId);
288 
289             writeRawResults(runId, detailedResults);
290 
291             DescriptiveStatistics overall = new DescriptiveStatistics();
292             try (FileWriter writer = new FileWriter(path, true)) {
293                 writer.write("Run ID, " + runId + "\n");
294                 writer.write("Test, Iteration, Score, Jank Penalty, Consistency Bonus, 95th, " +
295                         "90th\n");
296                 for (String testName : detailedResults.keySet()) {
297                     ArrayList<UiBenchmarkResult> results = detailedResults.get(testName);
298                     DescriptiveStatistics scoreStats = new DescriptiveStatistics();
299                     DescriptiveStatistics jankPenalty = new DescriptiveStatistics();
300                     DescriptiveStatistics consistencyBonus = new DescriptiveStatistics();
301                     for (int i = 0; i < results.size(); i++) {
302                         UiBenchmarkResult result = results.get(i);
303                         int score = result.getScore();
304                         scoreStats.addValue(score);
305                         overall.addValue(score);
306                         jankPenalty.addValue(result.getJankPenalty());
307                         consistencyBonus.addValue(result.getConsistencyBonus());
308 
309                         writer.write(testName);
310                         writer.write(",");
311                         writer.write("" + i);
312                         writer.write(",");
313                         writer.write("" + score);
314                         writer.write(",");
315                         writer.write("" + result.getJankPenalty());
316                         writer.write(",");
317                         writer.write("" + result.getConsistencyBonus());
318                         writer.write(",");
319                         writer.write(Double.toString(
320                                 result.getPercentile(FrameMetrics.TOTAL_DURATION, 95)));
321                         writer.write(",");
322                         writer.write(Double.toString(
323                                 result.getPercentile(FrameMetrics.TOTAL_DURATION, 90)));
324                         writer.write("\n");
325                     }
326 
327                     writer.write("Score CV," +
328                             (100 * scoreStats.getStandardDeviation()
329                                     / scoreStats.getMean()) + "%\n");
330                     writer.write("Jank Penalty CV, " +
331                             (100 * jankPenalty.getStandardDeviation()
332                                     / jankPenalty.getMean()) + "%\n");
333                     writer.write("Consistency Bonus CV, " +
334                             (100 * consistencyBonus.getStandardDeviation()
335                                     / consistencyBonus.getMean()) + "%\n");
336                     writer.write("\n");
337                 }
338 
339                 writer.write("Overall Score CV,"  +
340                         (100 * overall.getStandardDeviation() / overall.getMean()) + "%\n");
341                 writer.flush();
342             }
343         }
344 
345         runIdCursor.close();
346     }
347 
writeRawResults(int runId, HashMap<String, ArrayList<UiBenchmarkResult>> detailedResults)348     private void writeRawResults(int runId,
349                                  HashMap<String, ArrayList<UiBenchmarkResult>> detailedResults) {
350         StringBuilder path = new StringBuilder();
351         path.append(mContext.getFilesDir());
352         path.append("/");
353         path.append(Integer.toString(runId));
354         path.append(".csv");
355         try (FileWriter writer = new FileWriter(path.toString())) {
356             for (String test : detailedResults.keySet()) {
357                 writer.write("Test, " + test + "\n");
358                 writer.write("iteration, unknown delay, input, animation, layout, draw, sync, " +
359                         "command issue, swap buffers\n");
360                 ArrayList<UiBenchmarkResult> runs = detailedResults.get(test);
361                 for (int i = 0; i < runs.size(); i++) {
362                     UiBenchmarkResult run = runs.get(i);
363                     for (int j = 0; j < run.getTotalFrameCount(); j++) {
364                         writer.write(Integer.toString(i) + "," +
365                                 run.getMetricAtIndex(j, FrameMetrics.UNKNOWN_DELAY_DURATION) + "," +
366                                 run.getMetricAtIndex(j, FrameMetrics.INPUT_HANDLING_DURATION) + "," +
367                                 run.getMetricAtIndex(j, FrameMetrics.ANIMATION_DURATION) + "," +
368                                 run.getMetricAtIndex(j, FrameMetrics.LAYOUT_MEASURE_DURATION) + "," +
369                                 run.getMetricAtIndex(j, FrameMetrics.DRAW_DURATION) + "," +
370                                 run.getMetricAtIndex(j, FrameMetrics.SYNC_DURATION) + "," +
371                                 run.getMetricAtIndex(j, FrameMetrics.COMMAND_ISSUE_DURATION) + "," +
372                                 run.getMetricAtIndex(j, FrameMetrics.SWAP_BUFFERS_DURATION) + "," +
373                                 run.getMetricAtIndex(j, FrameMetrics.TOTAL_DURATION) + "\n");
374                     }
375                 }
376             }
377         } catch (IOException e) {
378             e.printStackTrace();
379         }
380     }
381 
382     @Override
onUpgrade(SQLiteDatabase sqLiteDatabase, int oldVersion, int currentVersion)383     public void onUpgrade(SQLiteDatabase sqLiteDatabase, int oldVersion, int currentVersion) {
384         if (oldVersion < VERSION) {
385             sqLiteDatabase.execSQL("ALTER TABLE "
386                     + UI_RESULTS_TABLE + " ADD COLUMN timestamp TEXT;");
387         }
388     }
389 }
390