1 /*
2  * Copyright (C) 2019 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 package com.android.tradefed.testtype.binary;
17 
18 import static org.junit.Assert.fail;
19 import static org.mockito.ArgumentMatchers.any;
20 import static org.mockito.ArgumentMatchers.anyInt;
21 import static org.mockito.ArgumentMatchers.eq;
22 import static org.mockito.Mockito.doAnswer;
23 import static org.mockito.Mockito.doReturn;
24 import static org.mockito.Mockito.doThrow;
25 import static org.mockito.Mockito.times;
26 import static org.mockito.Mockito.verify;
27 
28 import com.android.tradefed.build.BuildInfo;
29 import com.android.tradefed.build.DeviceBuildInfo;
30 import com.android.tradefed.build.IDeviceBuildInfo;
31 import com.android.tradefed.config.OptionSetter;
32 import com.android.tradefed.device.DeviceNotAvailableException;
33 import com.android.tradefed.device.ITestDevice;
34 import com.android.tradefed.invoker.InvocationContext;
35 import com.android.tradefed.invoker.TestInformation;
36 import com.android.tradefed.metrics.proto.MetricMeasurement.Metric;
37 import com.android.tradefed.result.FailureDescription;
38 import com.android.tradefed.result.ITestInvocationListener;
39 import com.android.tradefed.result.error.InfraErrorIdentifier;
40 import com.android.tradefed.result.error.DeviceErrorIdentifier;
41 import com.android.tradefed.result.proto.TestRecordProto.FailureStatus;
42 import com.android.tradefed.util.CommandResult;
43 import com.android.tradefed.util.CommandStatus;
44 import com.android.tradefed.util.FileUtil;
45 import com.android.tradefed.util.IRunUtil;
46 
47 import org.junit.Before;
48 import org.junit.Test;
49 import org.junit.runner.RunWith;
50 import org.junit.runners.JUnit4;
51 import org.mockito.Mockito;
52 import org.mockito.invocation.InvocationOnMock;
53 import org.mockito.stubbing.Answer;
54 
55 import java.io.File;
56 import java.io.OutputStream;
57 import java.util.HashMap;
58 
59 /** Unit tests for {@link ExecutableHostTest}. */
60 @RunWith(JUnit4.class)
61 public class ExecutableHostTestTest {
62 
63     private ExecutableHostTest mExecutableTest;
64     private ITestInvocationListener mMockListener;
65     private ITestDevice mMockDevice;
66     private IRunUtil mMockRunUtil;
67     private TestInformation mTestInfo;
68 
69     @Before
setUp()70     public void setUp() {
71         mMockListener = Mockito.mock(ITestInvocationListener.class);
72         mMockDevice = Mockito.mock(ITestDevice.class);
73         mMockRunUtil = Mockito.mock(IRunUtil.class);
74         mExecutableTest =
75                 new ExecutableHostTest() {
76                     @Override
77                     IRunUtil createRunUtil() {
78                         return mMockRunUtil;
79                     }
80                 };
81         InvocationContext context = new InvocationContext();
82         context.addAllocatedDevice("device", mMockDevice);
83         mTestInfo = TestInformation.newBuilder().setInvocationContext(context).build();
84     }
85 
86     @Test
testRunHostExecutable_noBinaries()87     public void testRunHostExecutable_noBinaries() throws Exception {
88         mExecutableTest.run(mTestInfo, mMockListener);
89 
90         verify(mMockListener, times(0)).testRunStarted(any(), anyInt());
91     }
92 
93     @Test
testRunHostExecutable_doesNotExists()94     public void testRunHostExecutable_doesNotExists() throws Exception {
95         String path = "/does/not/exists/path/bin/test";
96         OptionSetter setter = new OptionSetter(mExecutableTest);
97         setter.setOptionValue("binary", path);
98         mTestInfo.getContext().addDeviceBuildInfo("device", new BuildInfo());
99         mExecutableTest.run(mTestInfo, mMockListener);
100 
101         verify(mMockListener, Mockito.times(1)).testRunStarted(eq("test"), eq(0));
102         FailureDescription failure =
103                 FailureDescription.create(
104                                 String.format(ExecutableBaseTest.NO_BINARY_ERROR, path),
105                                 FailureStatus.TEST_FAILURE)
106                         .setErrorIdentifier(InfraErrorIdentifier.ARTIFACT_NOT_FOUND);
107         verify(mMockListener, Mockito.times(1)).testRunFailed(failure);
108         verify(mMockListener, Mockito.times(1))
109                 .testRunEnded(eq(0L), Mockito.<HashMap<String, Metric>>any());
110     }
111 
112     @Test
testRunHostExecutable()113     public void testRunHostExecutable() throws Exception {
114         File tmpBinary = FileUtil.createTempFile("test-executable", "");
115         try {
116             OptionSetter setter = new OptionSetter(mExecutableTest);
117             setter.setOptionValue("binary", tmpBinary.getAbsolutePath());
118 
119             CommandResult result = new CommandResult(CommandStatus.SUCCESS);
120             doReturn(result)
121                     .when(mMockRunUtil)
122                     .runTimedCmd(
123                             Mockito.anyLong(),
124                             (OutputStream) Mockito.any(),
125                             Mockito.any(),
126                             Mockito.eq(tmpBinary.getAbsolutePath()));
127 
128             mExecutableTest.run(mTestInfo, mMockListener);
129 
130             verify(mMockListener, Mockito.times(1)).testRunStarted(eq(tmpBinary.getName()), eq(1));
131             verify(mMockListener, Mockito.times(0)).testRunFailed((String) any());
132             verify(mMockListener, Mockito.times(0)).testFailed(any(), (String) any());
133             verify(mMockListener, Mockito.times(1))
134                     .testRunEnded(Mockito.anyLong(), Mockito.<HashMap<String, Metric>>any());
135         } finally {
136             FileUtil.recursiveDelete(tmpBinary);
137         }
138     }
139 
140     @Test
testRunHostExecutable_relativePath()141     public void testRunHostExecutable_relativePath() throws Exception {
142         File tmpBinary = FileUtil.createTempFile("test-executable", "");
143         try {
144             OptionSetter setter = new OptionSetter(mExecutableTest);
145             setter.setOptionValue("binary", tmpBinary.getAbsolutePath());
146             setter.setOptionValue("relative-path-execution", "true");
147 
148             CommandResult result = new CommandResult(CommandStatus.SUCCESS);
149             doReturn(result)
150                     .when(mMockRunUtil)
151                     .runTimedCmd(
152                             Mockito.anyLong(),
153                             (OutputStream) Mockito.any(),
154                             Mockito.any(),
155                             Mockito.eq("bash"),
156                             Mockito.eq("-c"),
157                             Mockito.eq(
158                                     String.format(
159                                             "pushd %s; ./%s;",
160                                             tmpBinary.getParent(), tmpBinary.getName())));
161 
162             mExecutableTest.run(mTestInfo, mMockListener);
163 
164             verify(mMockListener, Mockito.times(1)).testRunStarted(eq(tmpBinary.getName()), eq(1));
165             verify(mMockListener, Mockito.times(0)).testRunFailed((String) any());
166             verify(mMockListener, Mockito.times(0)).testFailed(any(), (String) any());
167             verify(mMockListener, Mockito.times(1))
168                     .testRunEnded(Mockito.anyLong(), Mockito.<HashMap<String, Metric>>any());
169         } finally {
170             FileUtil.recursiveDelete(tmpBinary);
171         }
172     }
173 
174     @Test
testRunHostExecutable_dnae()175     public void testRunHostExecutable_dnae() throws Exception {
176         File tmpBinary = FileUtil.createTempFile("test-executable", "");
177         try {
178             OptionSetter setter = new OptionSetter(mExecutableTest);
179             setter.setOptionValue("binary", tmpBinary.getAbsolutePath());
180 
181             CommandResult result = new CommandResult(CommandStatus.SUCCESS);
182             doReturn(result)
183                     .when(mMockRunUtil)
184                     .runTimedCmd(
185                             Mockito.anyLong(),
186                             (OutputStream) Mockito.any(),
187                             Mockito.any(),
188                             Mockito.eq(tmpBinary.getAbsolutePath()));
189 
190             doThrow(new DeviceNotAvailableException("test", "serial"))
191                     .when(mMockDevice)
192                     .waitForDeviceAvailable();
193             DeviceNotAvailableException dnae = null;
194             try {
195                 mExecutableTest.run(mTestInfo, mMockListener);
196                 fail("Should have thrown an exception.");
197             } catch (DeviceNotAvailableException expected) {
198                 // Expected
199                 dnae = expected;
200             }
201 
202             verify(mMockListener, Mockito.times(1)).testRunStarted(eq(tmpBinary.getName()), eq(1));
203             FailureDescription failure =
204                     FailureDescription.create(
205                                     String.format(
206                                             "Device became unavailable after %s.",
207                                             tmpBinary.getAbsolutePath()),
208                                     FailureStatus.LOST_SYSTEM_UNDER_TEST)
209                             .setErrorIdentifier(DeviceErrorIdentifier.DEVICE_UNAVAILABLE)
210                             .setCause(dnae);
211             verify(mMockListener, Mockito.times(1)).testRunFailed(eq(failure));
212             verify(mMockListener, Mockito.times(0)).testFailed(any(), (String) any());
213             verify(mMockListener, Mockito.times(1))
214                     .testRunEnded(Mockito.anyLong(), Mockito.<HashMap<String, Metric>>any());
215         } finally {
216             FileUtil.recursiveDelete(tmpBinary);
217         }
218     }
219 
220     /** If the binary is available from the tests directory we can find it and run it. */
221     @Test
testRunHostExecutable_search()222     public void testRunHostExecutable_search() throws Exception {
223         File testsDir = FileUtil.createTempDir("executable-tests-dir");
224         File tmpBinary = FileUtil.createTempFile("test-executable", "", testsDir);
225         try {
226             IDeviceBuildInfo info = new DeviceBuildInfo();
227             info.setTestsDir(testsDir, "testversion");
228             mTestInfo.getContext().addDeviceBuildInfo("device", info);
229             OptionSetter setter = new OptionSetter(mExecutableTest);
230             setter.setOptionValue("binary", tmpBinary.getName());
231 
232             CommandResult result = new CommandResult(CommandStatus.SUCCESS);
233             doReturn(result)
234                     .when(mMockRunUtil)
235                     .runTimedCmd(
236                             Mockito.anyLong(),
237                             (OutputStream) Mockito.any(),
238                             Mockito.any(),
239                             Mockito.eq(tmpBinary.getAbsolutePath()));
240 
241             mExecutableTest.run(mTestInfo, mMockListener);
242 
243             verify(mMockListener, Mockito.times(1)).testRunStarted(eq(tmpBinary.getName()), eq(1));
244             verify(mMockListener, Mockito.times(0)).testRunFailed((String) any());
245             verify(mMockListener, Mockito.times(0)).testFailed(any(), (String) any());
246             verify(mMockListener, Mockito.times(1))
247                     .testRunEnded(Mockito.anyLong(), Mockito.<HashMap<String, Metric>>any());
248         } finally {
249             FileUtil.recursiveDelete(testsDir);
250         }
251     }
252 
253     @Test
testRunHostExecutable_notFound()254     public void testRunHostExecutable_notFound() throws Exception {
255         File testsDir = FileUtil.createTempDir("executable-tests-dir");
256         File tmpBinary = FileUtil.createTempFile("test-executable", "", testsDir);
257         try {
258             IDeviceBuildInfo info = new DeviceBuildInfo();
259             info.setTestsDir(testsDir, "testversion");
260             mTestInfo.getContext().addDeviceBuildInfo("device", info);
261             OptionSetter setter = new OptionSetter(mExecutableTest);
262             setter.setOptionValue("binary", tmpBinary.getName());
263             tmpBinary.delete();
264 
265             CommandResult result = new CommandResult(CommandStatus.SUCCESS);
266             doReturn(result)
267                     .when(mMockRunUtil)
268                     .runTimedCmd(Mockito.anyLong(), Mockito.eq(tmpBinary.getAbsolutePath()));
269 
270             mExecutableTest.run(mTestInfo, mMockListener);
271 
272             verify(mMockListener, Mockito.times(1)).testRunStarted(eq(tmpBinary.getName()), eq(0));
273             FailureDescription failure =
274                     FailureDescription.create(
275                                     String.format(
276                                             ExecutableBaseTest.NO_BINARY_ERROR,
277                                             tmpBinary.getName()),
278                                     FailureStatus.TEST_FAILURE)
279                             .setErrorIdentifier(InfraErrorIdentifier.ARTIFACT_NOT_FOUND);
280             verify(mMockListener, Mockito.times(1)).testRunFailed(eq(failure));
281             verify(mMockListener, Mockito.times(1))
282                     .testRunEnded(Mockito.anyLong(), Mockito.<HashMap<String, Metric>>any());
283         } finally {
284             FileUtil.recursiveDelete(testsDir);
285         }
286     }
287 
288     @Test
testRunHostExecutable_failure()289     public void testRunHostExecutable_failure() throws Exception {
290         File tmpBinary = FileUtil.createTempFile("test-executable", "");
291         try {
292             OptionSetter setter = new OptionSetter(mExecutableTest);
293             setter.setOptionValue("binary", tmpBinary.getAbsolutePath());
294 
295             CommandResult result = new CommandResult(CommandStatus.FAILED);
296             result.setExitCode(5);
297             result.setStdout("stdout");
298 
299             doAnswer(
300                             new Answer<CommandResult>() {
301 
302                                 @Override
303                                 public CommandResult answer(InvocationOnMock invocation)
304                                         throws Throwable {
305                                     OutputStream outputStream = invocation.getArgument(1);
306                                     outputStream.write("stdout".getBytes());
307                                     return result;
308                                 }
309                             })
310                     .when(mMockRunUtil)
311                     .runTimedCmd(
312                             Mockito.anyLong(),
313                             (OutputStream) Mockito.any(),
314                             Mockito.any(),
315                             Mockito.eq(tmpBinary.getAbsolutePath()));
316 
317             mExecutableTest.run(mTestInfo, mMockListener);
318 
319             verify(mMockListener, Mockito.times(1)).testRunStarted(eq(tmpBinary.getName()), eq(1));
320             verify(mMockListener, Mockito.times(0)).testRunFailed((String) any());
321             verify(mMockListener, Mockito.times(1))
322                     .testFailed(
323                             any(),
324                             eq(
325                                     FailureDescription.create("stdout\nExit Code: 5")
326                                             .setFailureStatus(FailureStatus.TEST_FAILURE)));
327             verify(mMockListener, Mockito.times(1))
328                     .testRunEnded(Mockito.anyLong(), Mockito.<HashMap<String, Metric>>any());
329         } finally {
330             FileUtil.recursiveDelete(tmpBinary);
331         }
332     }
333 
334     @Test
testRunHostExecutable_timeout()335     public void testRunHostExecutable_timeout() throws Exception {
336         File tmpBinary = FileUtil.createTempFile("test-executable", "");
337         try {
338             OptionSetter setter = new OptionSetter(mExecutableTest);
339             setter.setOptionValue("binary", tmpBinary.getAbsolutePath());
340 
341             CommandResult result = new CommandResult(CommandStatus.TIMED_OUT);
342             result.setExitCode(5);
343             result.setStdout("stdout");
344 
345             doAnswer(
346                             new Answer<CommandResult>() {
347 
348                                 @Override
349                                 public CommandResult answer(InvocationOnMock invocation)
350                                         throws Throwable {
351                                     OutputStream outputStream = invocation.getArgument(1);
352                                     outputStream.write("stdout".getBytes());
353                                     return result;
354                                 }
355                             })
356                     .when(mMockRunUtil)
357                     .runTimedCmd(
358                             Mockito.anyLong(),
359                             (OutputStream) Mockito.any(),
360                             Mockito.any(),
361                             Mockito.eq(tmpBinary.getAbsolutePath()));
362 
363             mExecutableTest.run(mTestInfo, mMockListener);
364 
365             verify(mMockListener, Mockito.times(1)).testRunStarted(eq(tmpBinary.getName()), eq(1));
366             verify(mMockListener, Mockito.times(0)).testRunFailed((String) any());
367             verify(mMockListener, Mockito.times(1))
368                     .testFailed(
369                             any(),
370                             eq(
371                                     FailureDescription.create("stdout\nTimeout.\nExit Code: 5")
372                                             .setFailureStatus(FailureStatus.TIMED_OUT)));
373             verify(mMockListener, Mockito.times(1))
374                     .testRunEnded(Mockito.anyLong(), Mockito.<HashMap<String, Metric>>any());
375         } finally {
376             FileUtil.recursiveDelete(tmpBinary);
377         }
378     }
379 }
380