1 /*
2  * Copyright (C) 2018 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 package com.android.nn.benchmark.app;
18 
19 import android.os.Environment;
20 import android.test.suitebuilder.annotation.LargeTest;
21 
22 import com.android.nn.benchmark.core.TestModels;
23 import com.android.nn.benchmark.util.CSVWriter;
24 import com.android.nn.benchmark.util.TestExternalStorageActivity;
25 
26 import org.junit.AfterClass;
27 import org.junit.BeforeClass;
28 import org.junit.Test;
29 import org.junit.runner.RunWith;
30 import org.junit.runners.Parameterized;
31 import org.junit.runners.Parameterized.Parameters;
32 
33 import java.io.File;
34 import java.io.IOException;
35 import java.nio.file.Files;
36 import java.util.ArrayList;
37 import java.util.Collections;
38 import java.util.List;
39 
40 /**
41  * Tests that run all models/datasets/backend that are required for scoring the device.
42  * Produces a CSV file with benchmark results.
43  *
44  * To use, please run build_and_run_benchmark.sh
45  */
46 // TODO(pszczepaniak): Make it an activity, so it's possible to start from UI
47 @RunWith(Parameterized.class)
48 public class NNScoringTest extends BenchmarkTestBase {
49     private static final String RESULT_FILENAME = "mlts_benchmark.csv";
50     private static final String TAG = NNScoringTest.class.getSimpleName();
51 
52     private static File csvPath;
53     private static CSVWriter csvWriter;
54 
NNScoringTest(TestModels.TestModelEntry model)55     public NNScoringTest(TestModels.TestModelEntry model) {
56         super(model);
57     }
58 
59     @Override
prepareTest()60     protected void prepareTest() {
61         super.prepareTest();
62     }
63 
test(boolean useNnapi, boolean useCompleteInputSet)64     private void test(boolean useNnapi, boolean useCompleteInputSet) throws IOException {
65         if (!TestExternalStorageActivity.testWriteExternalStorage(getActivity(), false)) {
66             throw new IOException("No permission to store results in external storage");
67         }
68 
69         setUseNNApi(useNnapi);
70         setCompleteInputSet(useCompleteInputSet);
71         enableCompilationCachingBenchmarks();
72         TestAction ta = new TestAction(mModel, WARMUP_REPEATABLE_SECONDS,
73                 RUNTIME_REPEATABLE_SECONDS);
74         runTest(ta, mModel.getTestName());
75 
76         try (CSVWriter writer = new CSVWriter(getLocalCSVFile())) {
77             writer.write(ta.getBenchmark());
78         }
79     }
80 
81     @Test
82     @LargeTest
testTFLite()83     public void testTFLite() throws IOException {
84         test(false, false);
85     }
86 
87     @Test
88     @LargeTest
testNNAPI()89     public void testNNAPI() throws IOException {
90         test(true, true);
91     }
92 
getLocalCSVFile()93     public static File getLocalCSVFile() {
94         return new File("/data/data/com.android.nn.benchmark.app", RESULT_FILENAME);
95     }
96 
97     @BeforeClass
beforeClass()98     public static void beforeClass() throws IOException {
99         // Clear up CSV file in data directory for result storage
100         File localResults = getLocalCSVFile();
101         localResults.delete();
102         localResults.createNewFile();
103         try (CSVWriter writer = new CSVWriter(localResults)) {
104             writer.writeHeader();
105         }
106     }
107 
108     @AfterClass
afterClass()109     public static void afterClass() throws IOException {
110         // Copy results to external storage.
111         // We can't dump result straight there, due to append mode not working on external storage.
112         // And we need to store results in external storage for easy adb pull retreival on
113         // non user-debug devices.
114         File externalStorageCSVFile = new File(Environment.getExternalStorageDirectory(),
115                 RESULT_FILENAME);
116         externalStorageCSVFile.delete();
117         Files.copy(getLocalCSVFile().toPath(), externalStorageCSVFile.toPath());
118     }
119 
120 }
121