1 /*
2  * Copyright (C) 2020 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 package com.android.nn.crashtest.app;
18 
19 import android.content.Intent;
20 import android.os.RemoteException;
21 import android.test.ActivityInstrumentationTestCase2;
22 import android.test.UiThreadTest;
23 import android.test.suitebuilder.annotation.LargeTest;
24 
25 import androidx.test.InstrumentationRegistry;
26 
27 import com.android.nn.benchmark.core.NnApiDelegationFailure;
28 import com.android.nn.benchmark.core.TestModels;
29 
30 import org.junit.Before;
31 import org.junit.Rule;
32 import org.junit.Test;
33 import org.junit.rules.TestName;
34 import org.junit.runner.RunWith;
35 import org.junit.runners.Parameterized;
36 import org.junit.runners.Parameterized.Parameters;
37 
38 import java.time.Duration;
39 import java.util.Optional;
40 import java.util.concurrent.ExecutionException;
41 import java.util.concurrent.ExecutorService;
42 import java.util.concurrent.Executors;
43 import java.util.concurrent.Future;
44 import java.util.stream.IntStream;
45 
46 @RunWith(Parameterized.class)
47 public class NNClientEarlyTerminationTest extends
48         ActivityInstrumentationTestCase2<NNParallelTestActivity> implements
49         AcceleratorSpecificTestSupport {
50 
51     private static final String TAG = "NNClientEarlyTermination";
52     private static final Duration MAX_SEPARATE_PROCESS_EXECUTION_TIME = Duration.ofSeconds(70);
53     public static final int NNAPI_CLIENTS_COUNT = 4;
54 
55     private final ExecutorService mDriverLivenessValidationExecutor =
56             Executors.newSingleThreadExecutor();
57     private final String mAcceleratorName;
58 
59     @Rule
60     public TestName mTestName = new TestName();
61 
NNClientEarlyTerminationTest(String acceleratorName)62     public NNClientEarlyTerminationTest(String acceleratorName) {
63         super(NNParallelTestActivity.class);
64         mAcceleratorName = acceleratorName;
65     }
66 
67     @Parameters(name = "Accelerator({0})")
targetAccelerators()68     public static Iterable<String> targetAccelerators() {
69         return AcceleratorSpecificTestSupport.getTargetAcceleratorNames();
70     }
71 
72     @Before
73     @Override
setUp()74     public void setUp() {
75         injectInstrumentation(InstrumentationRegistry.getInstrumentation());
76         try {
77             final Intent runSomeInferencesInASeparateProcess = compileSupportedModelsOnNThreadsFor(
78                     NNAPI_CLIENTS_COUNT, MAX_SEPARATE_PROCESS_EXECUTION_TIME);
79             setActivityIntent(runSomeInferencesInASeparateProcess);
80         } catch (NnApiDelegationFailure nnApiDelegationFailure) {
81             throw new RuntimeException(
82                     "Cannot initialize test, failure looking for supported models, please check "
83                             + "the driver status",
84                     nnApiDelegationFailure);
85         }
86     }
87 
88     @Test
89     @LargeTest
90     @UiThreadTest
testDriverDoesNotFailWithParallelThreads()91     public void testDriverDoesNotFailWithParallelThreads()
92             throws ExecutionException, InterruptedException, RemoteException,
93             NnApiDelegationFailure {
94         final NNParallelTestActivity activity = getActivity();
95 
96         Optional<TestModels.TestModelEntry> modelForLivenessTest =
97                 AcceleratorSpecificTestSupport.findTestModelRunningOnAccelerator(activity, mAcceleratorName);
98         assertTrue("No model available to be run on accelerator " + mAcceleratorName,
99                 modelForLivenessTest.isPresent());
100 
101         final DriverLivenessChecker driverLivenessChecker = new DriverLivenessChecker(activity,
102                 mAcceleratorName, modelForLivenessTest.get());
103         Future<Boolean> driverDidNotCrash = mDriverLivenessValidationExecutor.submit(
104                 driverLivenessChecker);
105 
106         // Causing failure before tests would end on their own.
107         final long maxTerminationTime = MAX_SEPARATE_PROCESS_EXECUTION_TIME.toMillis() / 2;
108         final long minTerminationTime = MAX_SEPARATE_PROCESS_EXECUTION_TIME.toMillis() / 4;
109         Thread.sleep(ramdomInRange(minTerminationTime, maxTerminationTime));
110 
111         try {
112             activity.killTestProcess();
113         } catch (RemoteException e) {
114             driverLivenessChecker.stop();
115             throw e;
116         }
117 
118         CrashTestStatus.TestResult testResult = activity.testResult();
119         driverLivenessChecker.stop();
120 
121         assertEquals("Remote process is expected to be killed",
122                 CrashTestStatus.TestResult.CRASH,
123                 testResult);
124 
125         assertTrue("Driver shouldn't crash if a client process is terminated",
126                 driverDidNotCrash.get());
127     }
128 
compileSupportedModelsOnNThreadsFor(int threadCount, Duration testDuration)129     private Intent compileSupportedModelsOnNThreadsFor(int threadCount, Duration testDuration)
130             throws NnApiDelegationFailure {
131         Intent intent = new Intent();
132         intent.putExtra(
133                 NNParallelTestActivity.EXTRA_TEST_LIST, IntStream.range(0,
134                         TestModels.modelsList().size()).toArray());
135         intent.putExtra(NNParallelTestActivity.EXTRA_THREAD_COUNT, threadCount);
136         intent.putExtra(NNParallelTestActivity.EXTRA_TEST_DURATION_MILLIS, testDuration.toMillis());
137         intent.putExtra(NNParallelTestActivity.EXTRA_RUN_IN_SEPARATE_PROCESS, true);
138         intent.putExtra(NNParallelTestActivity.EXTRA_TEST_NAME, mTestName.getMethodName());
139         intent.putExtra(NNParallelTestActivity.EXTRA_ACCELERATOR_NAME, mAcceleratorName);
140         intent.putExtra(NNParallelTestActivity.EXTRA_IGNORE_UNSUPPORTED_MODELS, true);
141         intent.putExtra(NNParallelTestActivity.EXTRA_RUN_MODEL_COMPILATION_ONLY, true);
142         return intent;
143     }
144 }
145