1 /*
2  * Copyright (C) 2017 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 package com.android.collectors;
17 
18 import static org.junit.Assert.assertEquals;
19 import static org.junit.Assert.assertFalse;
20 import static org.junit.Assert.assertTrue;
21 import static org.mockito.Mockito.doReturn;
22 import static org.mockito.Mockito.mock;
23 
24 import com.android.ddmlib.testrunner.RemoteAndroidTestRunner;
25 import com.android.tradefed.invoker.IInvocationContext;
26 import com.android.tradefed.result.CollectingTestListener;
27 import com.android.tradefed.result.TestDescription;
28 import com.android.tradefed.result.TestResult;
29 import com.android.tradefed.result.TestRunResult;
30 import com.android.tradefed.testtype.AndroidJUnitTest;
31 import com.android.tradefed.testtype.DeviceJUnit4ClassRunner;
32 import com.android.tradefed.testtype.junit4.BaseHostJUnit4Test;
33 import com.android.tradefed.testtype.junit4.DeviceTestRunOptions;
34 
35 import org.junit.Before;
36 import org.junit.Test;
37 import org.junit.runner.RunWith;
38 
39 import java.util.Arrays;
40 import java.util.Collection;
41 import java.util.Map.Entry;
42 
43 /**
44  * Host side tests for the core device collectors, this ensure that we are able to use the
45  * collectors in a similar way as the infra.
46  *
47  * Command:
48  * mm CollectorHostsideLibTest CollectorDeviceLibTest -j16
49  * tradefed.sh run commandAndExit template/local_min --template:map test=CollectorHostsideLibTest
50  */
51 @RunWith(DeviceJUnit4ClassRunner.class)
52 public class DeviceCollectorsTest extends BaseHostJUnit4Test {
53     private static final String TEST_APK = "CollectorDeviceLibTest.apk";
54     private static final String PACKAGE_NAME = "android.device.collectors";
55     private static final String AJUR_RUNNER = "androidx.test.runner.AndroidJUnitRunner";
56 
57     private static final String STUB_BASE_COLLECTOR =
58             "android.device.collectors.StubTestMetricListener";
59     private static final String SCHEDULED_COLLECTOR =
60             "android.device.collectors.StubScheduledRunMetricListener";
61 
62     private RemoteAndroidTestRunner mTestRunner;
63     private IInvocationContext mContext;
64 
65     @Before
setUp()66     public void setUp() throws Exception {
67         installPackage(TEST_APK);
68         assertTrue(isPackageInstalled(PACKAGE_NAME));
69         mTestRunner =
70                 new RemoteAndroidTestRunner(PACKAGE_NAME, AJUR_RUNNER, getDevice().getIDevice());
71         // Set the new runListener order to ensure test cases can show their metrics.
72         mTestRunner.addInstrumentationArg(AndroidJUnitTest.NEW_RUN_LISTENER_ORDER_KEY, "true");
73         mTestRunner.addInstrumentationArg("notClass", "android.device.tests.TestEvents");
74         mContext = mock(IInvocationContext.class);
75         doReturn(Arrays.asList(getDevice())).when(mContext).getDevices();
76         doReturn(Arrays.asList(getBuild())).when(mContext).getBuildInfos();
77     }
78 
79     /**
80      * Test that our base metric listener can output metrics.
81      */
82     @Test
testBaseListenerRuns()83     public void testBaseListenerRuns() throws Exception {
84         mTestRunner.addInstrumentationArg("listener", STUB_BASE_COLLECTOR);
85         mTestRunner.setClassName("android.device.collectors.BaseMetricListenerInstrumentedTest");
86         CollectingTestListener listener = new CollectingTestListener();
87         assertTrue(getDevice().runInstrumentationTests(mTestRunner, listener));
88         Collection<TestRunResult> results = listener.getRunResults();
89         assertEquals(1, results.size());
90         TestRunResult result = results.iterator().next();
91         assertFalse(result.isRunFailure());
92         assertFalse(result.hasFailedTests());
93         // Ensure the listener added a metric at test run start and end.
94         assertTrue(result.getRunMetrics().containsKey("run_start"));
95         assertTrue(result.getRunMetrics().containsKey("run_end"));
96         // Check that each test case has results with the metrics associated.
97         for (TestResult res : result.getTestResults().values()) {
98             assertTrue(res.getMetrics().containsKey("test_start"));
99             assertTrue(res.getMetrics().containsKey("test_end"));
100         }
101     }
102 
103     /**
104      * Test that our base metric listener can filter metrics to run only against some groups tagged
105      * with an annotation. All annotation of BaseMetricListenerInstrumentedTest are annotated with
106      * the group 'testGroup'.
107      */
108     @Test
testBaseListenerRuns_withExcludeFilters()109     public void testBaseListenerRuns_withExcludeFilters() throws Exception {
110         mTestRunner.addInstrumentationArg("listener", STUB_BASE_COLLECTOR);
111         mTestRunner.addInstrumentationArg("exclude-filter-group", "testGroup");
112         mTestRunner.setClassName("android.device.collectors.BaseMetricListenerInstrumentedTest");
113         CollectingTestListener listener = new CollectingTestListener();
114         assertTrue(getDevice().runInstrumentationTests(mTestRunner, listener));
115         Collection<TestRunResult> results = listener.getRunResults();
116         assertEquals(1, results.size());
117         TestRunResult result = results.iterator().next();
118         assertFalse(result.isRunFailure());
119         assertFalse(result.hasFailedTests());
120         // Ensure the listener added a metric at test run start and end.
121         assertTrue(result.getRunMetrics().containsKey("run_start"));
122         assertTrue(result.getRunMetrics().containsKey("run_end"));
123         // We did run some tests
124         assertTrue(!result.getTestResults().isEmpty());
125         // After filtering none of the test case should contain any of the metrics since it was
126         // filtered.
127         for (TestResult testCaseResult : result.getTestResults().values()) {
128             assertFalse(testCaseResult.getMetrics().containsKey("test_start"));
129             assertFalse(testCaseResult.getMetrics().containsKey("test_fail"));
130             assertFalse(testCaseResult.getMetrics().containsKey("test_end"));
131         }
132     }
133 
134     /**
135      * Test that if an include and exclude filters are provided, the exclude filters has priority.
136      */
137     @Test
testBaseListenerRuns_withIncludeAndExcludeFilters()138     public void testBaseListenerRuns_withIncludeAndExcludeFilters() throws Exception {
139         mTestRunner.addInstrumentationArg("listener", STUB_BASE_COLLECTOR);
140         mTestRunner.addInstrumentationArg("include-filter-group", "testGroup");
141         mTestRunner.addInstrumentationArg("exclude-filter-group", "testGroup");
142         mTestRunner.setClassName("android.device.collectors.BaseMetricListenerInstrumentedTest");
143         CollectingTestListener listener = new CollectingTestListener();
144         assertTrue(getDevice().runInstrumentationTests(mTestRunner, listener));
145         Collection<TestRunResult> results = listener.getRunResults();
146         assertEquals(1, results.size());
147         TestRunResult result = results.iterator().next();
148         assertFalse(result.isRunFailure());
149         assertFalse(result.hasFailedTests());
150         // Ensure the listener added a metric at test run start and end.
151         assertTrue(result.getRunMetrics().containsKey("run_start"));
152         assertTrue(result.getRunMetrics().containsKey("run_end"));
153         // We did run some tests
154         assertTrue(!result.getTestResults().isEmpty());
155         // After filtering none of the test case should contain any of the metrics since it was
156         // filtered. Exclusion has priority over inclusion.
157         for (TestResult testCaseResult : result.getTestResults().values()) {
158             assertFalse(testCaseResult.getMetrics().containsKey("test_start"));
159             assertFalse(testCaseResult.getMetrics().containsKey("test_fail"));
160             assertFalse(testCaseResult.getMetrics().containsKey("test_end"));
161         }
162     }
163 
164     /**
165      * Test that if an include filter is provided, only method part of the included group will
166      * run the collection.
167      */
168     @Test
testBaseListenerRuns_withIncludeFilters()169     public void testBaseListenerRuns_withIncludeFilters() throws Exception {
170         mTestRunner.addInstrumentationArg("listener", STUB_BASE_COLLECTOR);
171         mTestRunner.addInstrumentationArg("include-filter-group", "testGroup1");
172         mTestRunner.setClassName("android.device.collectors.BaseMetricListenerInstrumentedTest");
173         CollectingTestListener listener = new CollectingTestListener();
174         assertTrue(getDevice().runInstrumentationTests(mTestRunner, listener));
175         Collection<TestRunResult> results = listener.getRunResults();
176         assertEquals(1, results.size());
177         TestRunResult result = results.iterator().next();
178         assertFalse(result.isRunFailure());
179         assertFalse(result.hasFailedTests());
180         // Ensure the listener added a metric at test run start and end.
181         assertTrue(result.getRunMetrics().containsKey("run_start"));
182         assertTrue(result.getRunMetrics().containsKey("run_end"));
183         // We did run some tests
184         assertTrue(!result.getTestResults().isEmpty());
185         // After filtering none of the test case should contain any of the metrics since it was
186         // filtered. Exclusion has priority over inclusion.
187         for (Entry<TestDescription, TestResult> testResult : result.getTestResults().entrySet()) {
188             // testReportMetrics method is the only one annotated with 'testGroup1' it should be the
189             // only one collecting metrics.
190             if ("testReportMetrics".equals(testResult.getKey().getTestName())) {
191                 assertTrue(testResult.getValue().getMetrics().containsKey("test_start"));
192                 assertTrue(testResult.getValue().getMetrics().containsKey("test_end"));
193             } else {
194                 assertFalse(testResult.getValue().getMetrics().containsKey("test_start"));
195                 assertFalse(testResult.getValue().getMetrics().containsKey("test_fail"));
196                 assertFalse(testResult.getValue().getMetrics().containsKey("test_end"));
197             }
198         }
199     }
200 
201     /**
202      * Test that our base scheduled listener can output metrics periodically.
203      */
204     @Test
testScheduledListenerRuns()205     public void testScheduledListenerRuns() throws Exception {
206         mTestRunner.addInstrumentationArg("listener", SCHEDULED_COLLECTOR);
207         mTestRunner.addInstrumentationArg("interval", "100");
208         mTestRunner.setClassName("android.device.collectors.BaseMetricListenerInstrumentedTest");
209         CollectingTestListener listener = new CollectingTestListener();
210         assertTrue(getDevice().runInstrumentationTests(mTestRunner, listener));
211         Collection<TestRunResult> results = listener.getRunResults();
212         assertEquals(1, results.size());
213         TestRunResult result = results.iterator().next();
214         assertFalse(result.isRunFailure());
215         assertFalse(result.hasFailedTests());
216         // There is time during the test to output at least a handful of periodic metrics.
217         assertTrue(result.getRunMetrics().containsKey("collect0"));
218         assertTrue(result.getRunMetrics().containsKey("collect1"));
219         assertTrue(result.getRunMetrics().containsKey("collect2"));
220     }
221 
222     /**
223      * Test that our base scheduled listener can use its default period to run when the interval
224      * given is not valid.
225      */
226     @Test
testScheduledListenerRuns_defaultValue()227     public void testScheduledListenerRuns_defaultValue() throws Exception {
228         mTestRunner.addInstrumentationArg("listener", SCHEDULED_COLLECTOR);
229         // Invalid interval will results in the default period to be used.
230         mTestRunner.addInstrumentationArg("interval", "-100");
231         mTestRunner.setClassName("android.device.collectors.BaseMetricListenerInstrumentedTest");
232         CollectingTestListener listener = new CollectingTestListener();
233         assertTrue(getDevice().runInstrumentationTests(mTestRunner, listener));
234         Collection<TestRunResult> results = listener.getRunResults();
235         assertEquals(1, results.size());
236         TestRunResult result = results.iterator().next();
237         assertFalse(result.isRunFailure());
238         assertFalse(result.hasFailedTests());
239         // The default interval value is one minute so it will only have time to run once.
240         assertEquals(1, result.getRunMetrics().size());
241         assertTrue(result.getRunMetrics().containsKey("collect0"));
242     }
243 
244     /**
245      * Test that when using -e log true. Nothing gets collected.
246      */
247     @Test
testLogOnly()248     public void testLogOnly() throws Exception {
249         DeviceTestRunOptions options = new DeviceTestRunOptions(PACKAGE_NAME);
250         options.addInstrumentationArg("listener", STUB_BASE_COLLECTOR);
251         options.addInstrumentationArg("log", "true");
252         options.setTestClassName("android.device.collectors.BaseMetricListenerInstrumentedTest");
253         boolean res = runDeviceTests(options);
254         assertTrue(res);
255         TestRunResult result = getLastDeviceRunResults();
256         assertTrue(result.getRunMetrics().isEmpty());
257         for (Entry<TestDescription, TestResult> testResult : result.getTestResults().entrySet()) {
258             assertTrue(testResult.getValue().getMetrics().isEmpty());
259         }
260     }
261 }
262