1 /*
2  * Copyright (C) 2020 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 package com.android.nn.crashtest.app;
18 
19 import android.app.Activity;
20 import android.content.Intent;
21 import android.os.Bundle;
22 import android.test.ActivityInstrumentationTestCase2;
23 import android.test.UiThreadTest;
24 import android.test.suitebuilder.annotation.LargeTest;
25 import android.util.Log;
26 
27 import androidx.test.InstrumentationRegistry;
28 
29 import com.android.nn.benchmark.app.BenchmarkTestBase;
30 import com.android.nn.benchmark.core.NnApiDelegationFailure;
31 import com.android.nn.benchmark.core.TestModels;
32 
33 import org.junit.After;
34 import org.junit.Before;
35 import org.junit.Rule;
36 import org.junit.Test;
37 import org.junit.rules.TestName;
38 import org.junit.runners.Parameterized.Parameters;
39 
40 import java.time.Duration;
41 import java.util.Arrays;
42 import java.util.stream.IntStream;
43 
44 abstract class NNParallelInferenceTest
45         extends ActivityInstrumentationTestCase2<NNParallelTestActivity>
46         implements AcceleratorSpecificTestSupport {
47     static final String TAG = "NNParallelInferenceTest";
48 
49     @Rule
50     public TestName mTestName = new TestName();
51 
52     private final int mThreadCount;
53     private final Duration mTestDuration;
54     private final String mAcceleratorName;
55 
runTestsInSeparateProcess()56     protected abstract boolean runTestsInSeparateProcess();
57 
NNParallelInferenceTest(int threadCount, Duration testDuration, String acceleratorName)58     protected NNParallelInferenceTest(int threadCount, Duration testDuration,
59             String acceleratorName) {
60         super(NNParallelTestActivity.class);
61         mThreadCount = threadCount;
62         mTestDuration = testDuration;
63         mAcceleratorName = acceleratorName;
64     }
65 
66     @Before
67     @Override
setUp()68     public void setUp() {
69         injectInstrumentation(InstrumentationRegistry.getInstrumentation());
70         BenchmarkTestBase.waitUntilCharged(getInstrumentation().getTargetContext(), 90);
71         try {
72             setActivityIntent(
73                     runAllModelsOnNThreadsForOnAccelerator(mThreadCount, mTestDuration,
74                             mAcceleratorName));
75         } catch (NnApiDelegationFailure nnApiDelegationFailure) {
76             throw new RuntimeException(
77                     "Cannot initialize test, failure looking for supported models, please check "
78                             + "the driver status",
79                     nnApiDelegationFailure);
80         }
81     }
82 
83     @Test
84     @LargeTest
85     @UiThreadTest
shouldNotFailWithParallelThreads()86     public void shouldNotFailWithParallelThreads() {
87         Bundle testData = new Bundle();
88         testData.putString("Test name", mTestName.getMethodName());
89         testData.putString("Test status", "Started");
90         getInstrumentation().sendStatus(Activity.RESULT_FIRST_USER, testData);
91 
92         CrashTestStatus.TestResult testResult = getActivity().testResult();
93         assertEquals("Test didn't complete successfully", CrashTestStatus.TestResult.SUCCESS,
94                 testResult);
95 
96         testData.putString("Test status", "Completed");
97         getInstrumentation().sendStatus(Activity.RESULT_OK, testData);
98     }
99 
100     @After
101     @Override
tearDown()102     public void tearDown() throws Exception {
103         Log.i(TAG, "Tearing down test");
104         super.tearDown();
105     }
106 
107     @Parameters(name = "{0} threads for {1} on accelerator {2}")
threadCountValues()108     public static Iterable<Object[]> threadCountValues() {
109         return AcceleratorSpecificTestSupport.perAcceleratorTestConfig(
110                 Arrays.asList(
111                         new Object[]{8, Duration.ofMinutes(30)},
112                         new Object[]{12, Duration.ofMinutes(20)}));
113     }
114 
runAllModelsOnNThreadsForOnAccelerator(int threadCount, Duration testDuration, String acceleratorName)115     private Intent runAllModelsOnNThreadsForOnAccelerator(int threadCount, Duration testDuration,
116             String acceleratorName) throws NnApiDelegationFailure {
117         Intent intent = new Intent();
118 
119         int modelsCount = TestModels.modelsList().size();
120         intent.putExtra(
121                 NNParallelTestActivity.EXTRA_TEST_LIST, IntStream.range(0, modelsCount).toArray());
122         intent.putExtra(NNParallelTestActivity.EXTRA_THREAD_COUNT, threadCount);
123         intent.putExtra(NNParallelTestActivity.EXTRA_TEST_DURATION_MILLIS, testDuration.toMillis());
124         intent.putExtra(NNParallelTestActivity.EXTRA_RUN_IN_SEPARATE_PROCESS,
125                 runTestsInSeparateProcess());
126         intent.putExtra(NNParallelTestActivity.EXTRA_TEST_NAME, mTestName.getMethodName());
127         if (acceleratorName != null) {
128             intent.putExtra(NNParallelTestActivity.EXTRA_ACCELERATOR_NAME, acceleratorName);
129             intent.putExtra(NNParallelTestActivity.EXTRA_IGNORE_UNSUPPORTED_MODELS, true);
130         }
131         return intent;
132     }
133 }
134