1 /*
2  * Copyright (C) 2010 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 package com.android.tradefed.invoker;
17 
18 import static org.junit.Assert.assertEquals;
19 import static org.junit.Assert.assertTrue;
20 import static org.junit.Assert.fail;
21 
22 import com.android.ddmlib.IDevice;
23 import com.android.tradefed.build.BuildInfo;
24 import com.android.tradefed.build.BuildInfoKey.BuildInfoFileKey;
25 import com.android.tradefed.build.BuildRetrievalError;
26 import com.android.tradefed.build.IBuildInfo;
27 import com.android.tradefed.build.IBuildInfo.BuildInfoProperties;
28 import com.android.tradefed.build.IBuildProvider;
29 import com.android.tradefed.build.IDeviceBuildInfo;
30 import com.android.tradefed.build.IDeviceBuildProvider;
31 import com.android.tradefed.command.CommandOptions;
32 import com.android.tradefed.command.CommandRunner.ExitCode;
33 import com.android.tradefed.command.FatalHostError;
34 import com.android.tradefed.command.ICommandOptions;
35 import com.android.tradefed.command.remote.DeviceDescriptor;
36 import com.android.tradefed.config.Configuration;
37 import com.android.tradefed.config.ConfigurationDef;
38 import com.android.tradefed.config.ConfigurationException;
39 import com.android.tradefed.config.DeviceConfigurationHolder;
40 import com.android.tradefed.config.GlobalConfiguration;
41 import com.android.tradefed.config.IConfiguration;
42 import com.android.tradefed.config.IConfigurationFactory;
43 import com.android.tradefed.config.IDeviceConfiguration;
44 import com.android.tradefed.config.IGlobalConfiguration;
45 import com.android.tradefed.config.Option;
46 import com.android.tradefed.config.OptionSetter;
47 import com.android.tradefed.device.DeviceAllocationState;
48 import com.android.tradefed.device.DeviceNotAvailableException;
49 import com.android.tradefed.device.IDeviceRecovery;
50 import com.android.tradefed.device.ITestDevice;
51 import com.android.tradefed.device.ITestDevice.RecoveryMode;
52 import com.android.tradefed.device.StubDevice;
53 import com.android.tradefed.device.TcpDevice;
54 import com.android.tradefed.device.TestDeviceOptions;
55 import com.android.tradefed.device.TestDeviceState;
56 import com.android.tradefed.device.metric.BaseDeviceMetricCollector;
57 import com.android.tradefed.device.metric.DeviceMetricData;
58 import com.android.tradefed.device.metric.IMetricCollector;
59 import com.android.tradefed.guice.InvocationScope;
60 import com.android.tradefed.invoker.logger.InvocationMetricLogger.InvocationMetricKey;
61 import com.android.tradefed.invoker.shard.IShardHelper;
62 import com.android.tradefed.invoker.shard.ShardHelper;
63 import com.android.tradefed.log.ILeveledLogOutput;
64 import com.android.tradefed.log.ILogRegistry;
65 import com.android.tradefed.log.ITestLogger;
66 import com.android.tradefed.metrics.proto.MetricMeasurement.Measurements;
67 import com.android.tradefed.metrics.proto.MetricMeasurement.Metric;
68 import com.android.tradefed.metrics.proto.MetricMeasurement.Metric.Builder;
69 import com.android.tradefed.postprocessor.BasePostProcessor;
70 import com.android.tradefed.postprocessor.IPostProcessor;
71 import com.android.tradefed.result.ActionInProgress;
72 import com.android.tradefed.result.ByteArrayInputStreamSource;
73 import com.android.tradefed.result.FailureDescription;
74 import com.android.tradefed.result.ILogSaver;
75 import com.android.tradefed.result.ILogSaverListener;
76 import com.android.tradefed.result.ITestInvocationListener;
77 import com.android.tradefed.result.ITestSummaryListener;
78 import com.android.tradefed.result.InputStreamSource;
79 import com.android.tradefed.result.InvocationStatus;
80 import com.android.tradefed.result.LogDataType;
81 import com.android.tradefed.result.LogFile;
82 import com.android.tradefed.result.TestDescription;
83 import com.android.tradefed.result.TestSummary;
84 import com.android.tradefed.result.error.InfraErrorIdentifier;
85 import com.android.tradefed.result.proto.TestRecordProto.FailureStatus;
86 import com.android.tradefed.retry.IRetryDecision;
87 import com.android.tradefed.targetprep.BuildError;
88 import com.android.tradefed.targetprep.ITargetCleaner;
89 import com.android.tradefed.targetprep.ITargetPreparer;
90 import com.android.tradefed.targetprep.TargetSetupError;
91 import com.android.tradefed.testtype.IDeviceTest;
92 import com.android.tradefed.testtype.IInvocationContextReceiver;
93 import com.android.tradefed.testtype.IRemoteTest;
94 import com.android.tradefed.testtype.IShardableTest;
95 import com.android.tradefed.testtype.StubTest;
96 import com.android.tradefed.util.FileUtil;
97 import com.android.tradefed.util.SystemUtil.EnvVariable;
98 import com.android.tradefed.util.keystore.IKeyStoreClient;
99 import com.android.tradefed.util.keystore.StubKeyStoreFactory;
100 
101 import org.easymock.Capture;
102 import org.easymock.EasyMock;
103 import org.junit.Before;
104 import org.junit.BeforeClass;
105 import org.junit.Test;
106 import org.junit.runner.RunWith;
107 import org.junit.runners.JUnit4;
108 
109 import java.io.File;
110 import java.io.IOException;
111 import java.io.InputStream;
112 import java.util.ArrayList;
113 import java.util.Collections;
114 import java.util.HashMap;
115 import java.util.HashSet;
116 import java.util.LinkedHashMap;
117 import java.util.List;
118 import java.util.Map;
119 import java.util.Set;
120 
121 /** Unit tests for {@link TestInvocation}. */
122 @SuppressWarnings("MustBeClosedChecker")
123 @RunWith(JUnit4.class)
124 public class TestInvocationTest {
125 
126     private static final String SERIAL = "serial";
127     private static final Map<String, String> EMPTY_MAP = Collections.emptyMap();
128     private static final String PATH = "path";
129     private static final String URL = "url";
130     private static final TestSummary mSummary = new TestSummary("http://www.url.com/report.txt");
131     private static final InputStreamSource EMPTY_STREAM_SOURCE =
132             new ByteArrayInputStreamSource(new byte[0]);
133     private static final String LOGCAT_NAME_ERROR =
134             TestInvocation.getDeviceLogName(TestInvocation.Stage.ERROR);
135     private static final String LOGCAT_NAME_SETUP =
136             TestInvocation.getDeviceLogName(TestInvocation.Stage.SETUP);
137     private static final String LOGCAT_NAME_TEST =
138             TestInvocation.getDeviceLogName(TestInvocation.Stage.TEST);
139     private static final String LOGCAT_NAME_TEARDOWN =
140             TestInvocation.getDeviceLogName(TestInvocation.Stage.TEARDOWN);
141     /** The {@link TestInvocation} under test, with all dependencies mocked out */
142     private TestInvocation mTestInvocation;
143 
144     private FailureStatus mExceptedStatus = null;
145 
146     private IConfiguration mStubConfiguration;
147     private IConfiguration mStubMultiConfiguration;
148     private IGlobalConfiguration mGlobalConfiguration;
149 
150     private IInvocationContext mStubInvocationMetadata;
151 
152     // The mock objects.
153     private ITestDevice mMockDevice;
154     private ITargetPreparer mMockPreparer;
155     private IBuildProvider mMockBuildProvider;
156     private IBuildInfo mMockBuildInfo;
157     private ITestInvocationListener mMockTestListener;
158     private ITestSummaryListener mMockSummaryListener;
159     private ILeveledLogOutput mMockLogger;
160     private ILogSaver mMockLogSaver;
161     private IDeviceRecovery mMockRecovery;
162     private Capture<List<TestSummary>> mUriCapture;
163     private ILogRegistry mMockLogRegistry;
164     private IConfigurationFactory mMockConfigFactory;
165     private IRescheduler mockRescheduler;
166     private DeviceDescriptor mFakeDescriptor;
167 
168     @BeforeClass
setUpClass()169     public static void setUpClass() throws Exception {
170         try {
171             GlobalConfiguration.createGlobalConfiguration(new String[] {"empty"});
172         } catch (IllegalStateException e) {
173             // Avoid exception in case of multi-init
174         }
175     }
176 
177     @Before
setUp()178     public void setUp() throws Exception {
179         mStubConfiguration =
180                 new Configuration("foo", "bar") {
181                     @Override
182                     public IConfiguration partialDeepClone(
183                             List<String> objectToDeepClone, IKeyStoreClient client)
184                             throws ConfigurationException {
185                         return new Configuration(this.getName(), this.getDescription());
186                     }
187                 };
188         mStubMultiConfiguration = new Configuration("foo", "bar");
189 
190         mGlobalConfiguration = EasyMock.createMock(IGlobalConfiguration.class);
191 
192         mMockDevice = EasyMock.createMock(ITestDevice.class);
193         mMockRecovery = EasyMock.createMock(IDeviceRecovery.class);
194         mMockPreparer = EasyMock.createMock(ITargetPreparer.class);
195         mMockBuildProvider = EasyMock.createMock(IBuildProvider.class);
196 
197         // Use strict mocks here since the order of Listener calls is important
198         mMockTestListener = EasyMock.createStrictMock(ITestInvocationListener.class);
199         mMockSummaryListener = EasyMock.createStrictMock(ITestSummaryListener.class);
200         mMockBuildInfo = EasyMock.createMock(IBuildInfo.class);
201         mMockLogger = EasyMock.createMock(ILeveledLogOutput.class);
202         mMockLogRegistry = EasyMock.createMock(ILogRegistry.class);
203         mMockLogSaver = EasyMock.createMock(ILogSaver.class);
204         mMockConfigFactory = EasyMock.createMock(IConfigurationFactory.class);
205         mockRescheduler = EasyMock.createMock(IRescheduler.class);
206 
207         mStubConfiguration.setDeviceRecovery(mMockRecovery);
208         mStubConfiguration.setTargetPreparer(mMockPreparer);
209         mStubConfiguration.setBuildProvider(mMockBuildProvider);
210 
211         EasyMock.expect(mMockPreparer.isDisabled()).andStubReturn(false);
212         EasyMock.expect(mMockPreparer.isTearDownDisabled()).andStubReturn(false);
213 
214         List<IDeviceConfiguration> deviceConfigs = new ArrayList<IDeviceConfiguration>();
215         IDeviceConfiguration device1 =
216                 new DeviceConfigurationHolder(ConfigurationDef.DEFAULT_DEVICE_NAME);
217         device1.addSpecificConfig(mMockRecovery);
218         device1.addSpecificConfig(mMockPreparer);
219         device1.addSpecificConfig(mMockBuildProvider);
220         deviceConfigs.add(device1);
221         mStubMultiConfiguration.setDeviceConfigList(deviceConfigs);
222 
223         mStubConfiguration.setLogSaver(mMockLogSaver);
224         mStubMultiConfiguration.setLogSaver(mMockLogSaver);
225 
226         List<ITestInvocationListener> listenerList = new ArrayList<ITestInvocationListener>(1);
227         listenerList.add(mMockTestListener);
228         listenerList.add(mMockSummaryListener);
229         mStubConfiguration.setTestInvocationListeners(listenerList);
230         mStubMultiConfiguration.setTestInvocationListeners(listenerList);
231 
232         mStubConfiguration.setLogOutput(mMockLogger);
233         mStubMultiConfiguration.setLogOutput(mMockLogger);
234         EasyMock.expect(mMockDevice.getSerialNumber()).andStubReturn(SERIAL);
235         EasyMock.expect(mMockDevice.getIDevice()).andStubReturn(null);
236         EasyMock.expect(mMockDevice.getBattery()).andStubReturn(null);
237         EasyMock.expect(mMockDevice.getDeviceState()).andStubReturn(TestDeviceState.NOT_AVAILABLE);
238         mMockDevice.setRecoveryMode(RecoveryMode.AVAILABLE);
239         mMockDevice.setRecovery(mMockRecovery);
240         mMockDevice.preInvocationSetup((IBuildInfo) EasyMock.anyObject());
241         EasyMock.expectLastCall().anyTimes();
242         mFakeDescriptor =
243                 new DeviceDescriptor(
244                         SERIAL,
245                         false,
246                         DeviceAllocationState.Available,
247                         "unknown",
248                         "unknown",
249                         "unknown",
250                         "unknown",
251                         "unknown");
252         EasyMock.expect(mMockDevice.getDeviceDescriptor()).andStubReturn(mFakeDescriptor);
253 
254         EasyMock.expect(mMockBuildInfo.getBuildId()).andStubReturn("1");
255         EasyMock.expect(mMockBuildInfo.getBuildAttributes()).andStubReturn(EMPTY_MAP);
256         EasyMock.expect(mMockBuildInfo.getBuildBranch()).andStubReturn("branch");
257         EasyMock.expect(mMockBuildInfo.getBuildFlavor()).andStubReturn("flavor");
258         EasyMock.expect(mMockBuildInfo.getProperties()).andStubReturn(new HashSet<>());
259 
260         // always expect logger initialization and cleanup calls
261         mMockLogRegistry.registerLogger(mMockLogger);
262         EasyMock.expectLastCall().times(2);
263         mMockLogger.init();
264         EasyMock.expectLastCall().times(2);
265         mMockLogger.closeLog();
266         EasyMock.expectLastCall().times(2);
267         mMockLogRegistry.unregisterLogger();
268         EasyMock.expectLastCall().times(2);
269         mUriCapture = new Capture<List<TestSummary>>();
270 
271         mStubInvocationMetadata = new InvocationContext();
272         mStubInvocationMetadata.addAllocatedDevice(ConfigurationDef.DEFAULT_DEVICE_NAME,
273                 mMockDevice);
274         mStubInvocationMetadata.addDeviceBuildInfo(ConfigurationDef.DEFAULT_DEVICE_NAME,
275                 mMockBuildInfo);
276 
277         // create the BaseTestInvocation to test
278         mTestInvocation =
279                 new TestInvocation() {
280                     @Override
281                     ILogRegistry getLogRegistry() {
282                         return mMockLogRegistry;
283                     }
284 
285                     @Override
286                     public IInvocationExecution createInvocationExec(RunMode mode) {
287                         return new InvocationExecution() {
288                             @Override
289                             protected IShardHelper createShardHelper() {
290                                 return new ShardHelper() {
291                                     @Override
292                                     protected IGlobalConfiguration getGlobalConfiguration() {
293                                         return mGlobalConfiguration;
294                                     }
295                                 };
296                             }
297 
298                             @Override
299                             protected String getAdbVersion() {
300                                 return null;
301                             }
302 
303                             @Override
304                             void logHostAdb(ITestLogger logger) {
305                                 // inop for the common test case.
306                             }
307                         };
308                     }
309 
310                     @Override
311                     protected void setExitCode(ExitCode code, Throwable stack) {
312                         // Empty on purpose
313                     }
314 
315                     @Override
316                     InvocationScope getInvocationScope() {
317                         // Avoid re-entry in the current TF invocation scope for unit tests.
318                         return new InvocationScope();
319                     }
320 
321                     @Override
322                     public void registerExecutionFiles(ExecutionFiles executionFiles) {
323                         // Empty on purpose
324                     }
325 
326                     @Override
327                     protected void applyAutomatedReporters(IConfiguration config) {
328                         // Empty on purpose
329                     }
330 
331                     @Override
332                     protected void addInvocationMetric(InvocationMetricKey key, long value) {}
333 
334                     @Override
335                     protected void addInvocationMetric(InvocationMetricKey key, String value) {}
336                 };
337     }
338 
339     /**
340      * Test the normal case invoke scenario with a {@link IRemoteTest}.
341      *
342      * <p>Verifies that all external interfaces get notified as expected.
343      */
344     @Test
testInvoke_RemoteTest()345     public void testInvoke_RemoteTest() throws Throwable {
346         IRemoteTest test = EasyMock.createMock(IRemoteTest.class);
347         setupMockSuccessListeners();
348 
349         test.run(EasyMock.anyObject(), EasyMock.anyObject());
350         mMockPreparer.tearDown(EasyMock.anyObject(), EasyMock.isNull());
351         setEarlyDeviceReleaseExpectation();
352         setupNormalInvoke(test);
353         EasyMock.replay(mockRescheduler);
354         mTestInvocation.invoke(mStubInvocationMetadata, mStubConfiguration, mockRescheduler);
355         verifyMocks(test, mockRescheduler);
356         verifySummaryListener();
357     }
358 
359     /**
360      * Test the normal case for multi invoke scenario with a {@link IRemoteTest}.
361      *
362      * <p>Verifies that all external interfaces get notified as expected.
363      */
364     @Test
testInvokeMulti_RemoteTest()365     public void testInvokeMulti_RemoteTest() throws Throwable {
366         IRemoteTest test = EasyMock.createMock(IRemoteTest.class);
367         setupMockSuccessListeners();
368 
369         test.run(EasyMock.anyObject(), EasyMock.anyObject());
370         mMockPreparer.tearDown(EasyMock.anyObject(), EasyMock.isNull());
371         setEarlyDeviceReleaseExpectation();
372         setupNormalInvoke(test);
373         EasyMock.replay(mockRescheduler);
374         mTestInvocation.invoke(mStubInvocationMetadata, mStubMultiConfiguration, mockRescheduler);
375         verifyMocks(test, mockRescheduler);
376         verifySummaryListener();
377     }
378 
379     /**
380      * Test the normal case invoke scenario with an {@link ITestSummaryListener} masquerading as an
381      * {@link ITestInvocationListener}.
382      *
383      * <p>Verifies that all external interfaces get notified as expected.
384      */
385     @Test
testInvoke_twoSummary()386     public void testInvoke_twoSummary() throws Throwable {
387 
388         IRemoteTest test = EasyMock.createMock(IRemoteTest.class);
389         setupMockSuccessListeners();
390 
391         test.run(EasyMock.anyObject(), EasyMock.anyObject());
392         mMockPreparer.tearDown(EasyMock.anyObject(), EasyMock.isNull());
393         setEarlyDeviceReleaseExpectation();
394         setupNormalInvoke(test);
395         EasyMock.replay(mockRescheduler);
396         mTestInvocation.invoke(mStubInvocationMetadata, mStubConfiguration, mockRescheduler);
397         verifyMocks(test, mockRescheduler);
398         verifySummaryListener();
399     }
400 
401     /**
402      * Test the invoke scenario where build retrieve fails.
403      *
404      * <p>An invocation will be started in this scenario.
405      */
406     @Test
testInvoke_buildFailed()407     public void testInvoke_buildFailed() throws Throwable {
408         BuildRetrievalError exception =
409                 new BuildRetrievalError("testInvoke_buildFailed", null, mMockBuildInfo);
410         EasyMock.expect(mMockBuildProvider.getBuild()).andThrow(exception);
411         EasyMock.expect(mMockBuildInfo.getTestTag()).andStubReturn(null);
412 
413         setupMockFailureListeners(exception);
414         setupInvoke();
415         EasyMock.reset(mMockLogger, mMockLogRegistry);
416         mMockLogRegistry.registerLogger(mMockLogger);
417         mMockLogger.init();
418         mMockLogger.closeLog();
419         mMockLogRegistry.unregisterLogger();
420         IRemoteTest test = EasyMock.createMock(IRemoteTest.class);
421         CommandOptions cmdOptions = new CommandOptions();
422         final String expectedTestTag = "TEST_TAG";
423         mMockBuildInfo.setTestTag(expectedTestTag);
424         cmdOptions.setTestTag(expectedTestTag);
425         mStubConfiguration.setCommandOptions(cmdOptions);
426         mStubConfiguration.setTest(test);
427         EasyMock.expect(mMockLogger.getLog()).andReturn(EMPTY_STREAM_SOURCE);
428         EasyMock.expect(mMockDevice.getLogcat()).andReturn(EMPTY_STREAM_SOURCE).times(2);
429         mMockDevice.clearLogcat();
430         EasyMock.expectLastCall().times(2);
431         mMockLogRegistry.unregisterLogger();
432         mMockLogRegistry.dumpToGlobalLog(mMockLogger);
433         mMockLogger.closeLog();
434         mMockBuildProvider.cleanUp(mMockBuildInfo);
435         replayMocks(test, mockRescheduler);
436         mTestInvocation.invoke(mStubInvocationMetadata, mStubConfiguration, mockRescheduler);
437         verifyMocks(test, mockRescheduler);
438         // invocation test tag was updated.
439         assertEquals(expectedTestTag, mStubInvocationMetadata.getTestTag());
440     }
441 
442     /** Ensure we get a build info when we get a runtime exception in fetch build. */
443     @Test
testInvoke_buildFailed_runtimeException()444     public void testInvoke_buildFailed_runtimeException() throws Throwable {
445         RuntimeException runtimeException = new RuntimeException("failed to get build.");
446         EasyMock.expect(mMockBuildProvider.getBuild()).andThrow(runtimeException);
447         setupInvoke();
448         // For the mocks to be properly done, stub a BuildRetrievalError.
449         setupMockFailureListenersAny(
450                 new BuildRetrievalError("fake", InfraErrorIdentifier.ARTIFACT_DOWNLOAD_ERROR),
451                 true);
452 
453         EasyMock.reset(mMockLogger, mMockLogRegistry);
454         mMockLogRegistry.registerLogger(mMockLogger);
455         mMockLogger.init();
456         mMockLogger.closeLog();
457         mMockLogRegistry.unregisterLogger();
458 
459         EasyMock.expect(mMockLogger.getLog()).andReturn(EMPTY_STREAM_SOURCE);
460         EasyMock.expect(mMockDevice.getLogcat()).andReturn(EMPTY_STREAM_SOURCE).times(2);
461         mMockDevice.clearLogcat();
462         EasyMock.expectLastCall().times(2);
463         Capture<IBuildInfo> captured = new Capture<>();
464         mMockBuildProvider.cleanUp(EasyMock.capture(captured));
465         mMockLogRegistry.unregisterLogger();
466         mMockLogRegistry.dumpToGlobalLog(mMockLogger);
467         mMockLogger.closeLog();
468         replayMocks(mockRescheduler);
469         mTestInvocation.invoke(mStubInvocationMetadata, mStubConfiguration, mockRescheduler);
470         verifyMocks();
471 
472         IBuildInfo stubBuild = captured.getValue();
473         assertEquals(BuildInfo.UNKNOWN_BUILD_ID, stubBuild.getBuildId());
474         stubBuild.cleanUp();
475     }
476 
477     /** Test the invoke scenario where there is no build to test. */
478     @Test
testInvoke_noBuild()479     public void testInvoke_noBuild() throws Throwable {
480         EasyMock.expect(mMockBuildProvider.getBuild()).andReturn(null);
481         setupInvoke();
482         setupMockFailureListenersAny(
483                 new BuildRetrievalError(
484                         "No build found to test.", InfraErrorIdentifier.ARTIFACT_NOT_FOUND),
485                 true);
486 
487         EasyMock.reset(mMockLogger, mMockLogRegistry);
488         mMockLogRegistry.registerLogger(mMockLogger);
489         mMockLogger.init();
490         mMockLogger.closeLog();
491         mMockLogRegistry.unregisterLogger();
492 
493         EasyMock.expect(mMockLogger.getLog()).andReturn(EMPTY_STREAM_SOURCE);
494         EasyMock.expect(mMockDevice.getLogcat()).andReturn(EMPTY_STREAM_SOURCE).times(2);
495         mMockDevice.clearLogcat();
496         EasyMock.expectLastCall().times(2);
497         Capture<IBuildInfo> captured = new Capture<>();
498         mMockBuildProvider.cleanUp(EasyMock.capture(captured));
499         mMockLogRegistry.unregisterLogger();
500         mMockLogRegistry.dumpToGlobalLog(mMockLogger);
501         mMockLogger.closeLog();
502         replayMocks(mockRescheduler);
503         mTestInvocation.invoke(mStubInvocationMetadata, mStubConfiguration, mockRescheduler);
504         verifyMocks();
505 
506         IBuildInfo stubBuild = captured.getValue();
507         assertEquals(BuildInfo.UNKNOWN_BUILD_ID, stubBuild.getBuildId());
508         stubBuild.cleanUp();
509     }
510 
511     /**
512      * Test when the reporting of host_log is returning null, in this case we don't log anything.
513      */
514     @Test
testInvoke_noBuild_noHostLog()515     public void testInvoke_noBuild_noHostLog() throws Throwable {
516         EasyMock.expect(mMockBuildProvider.getBuild()).andReturn(null);
517         setupInvoke();
518         setupMockFailureListeners(
519                 new BuildRetrievalError(
520                         "No build found to test.", InfraErrorIdentifier.ARTIFACT_NOT_FOUND),
521                 true, /* don't expect host log */
522                 false);
523 
524         EasyMock.reset(mMockLogger, mMockLogRegistry);
525         mMockLogRegistry.registerLogger(mMockLogger);
526         mMockLogger.init();
527         mMockLogger.closeLog();
528         EasyMock.expectLastCall().times(2);
529 
530         IRemoteTest test = EasyMock.createMock(IRemoteTest.class);
531         mStubConfiguration.setTest(test);
532         // Host log fails to report
533         EasyMock.expect(mMockLogger.getLog()).andReturn(null);
534         EasyMock.expect(mMockDevice.getLogcat()).andReturn(EMPTY_STREAM_SOURCE).times(2);
535         mMockDevice.clearLogcat();
536         EasyMock.expectLastCall().times(2);
537         Capture<IBuildInfo> captured = new Capture<>();
538         mMockBuildProvider.cleanUp(EasyMock.capture(captured));
539         mMockLogRegistry.unregisterLogger();
540         EasyMock.expectLastCall().times(2);
541         mMockLogRegistry.dumpToGlobalLog(mMockLogger);
542         replayMocks(test, mockRescheduler);
543         mTestInvocation.invoke(mStubInvocationMetadata, mStubConfiguration, mockRescheduler);
544         verifyMocks(test);
545 
546         IBuildInfo stubBuild = captured.getValue();
547         assertEquals(BuildInfo.UNKNOWN_BUILD_ID, stubBuild.getBuildId());
548         stubBuild.cleanUp();
549     }
550 
551     /**
552      * Test the{@link TestInvocation#invoke(IInvocationContext, IConfiguration, IRescheduler,
553      * ITestInvocationListener[])} scenario where the test is a {@link IDeviceTest}
554      */
555     @Test
testInvoke_deviceTest()556     public void testInvoke_deviceTest() throws Throwable {
557         DeviceConfigTest mockDeviceTest = EasyMock.createMock(DeviceConfigTest.class);
558         mStubConfiguration.setTest(mockDeviceTest);
559         mockDeviceTest.setDevice(mMockDevice);
560         mockDeviceTest.run(EasyMock.anyObject(), EasyMock.anyObject());
561         setupMockSuccessListeners();
562         mMockPreparer.tearDown(EasyMock.anyObject(), EasyMock.isNull());
563         setEarlyDeviceReleaseExpectation();
564         setupNormalInvoke(mockDeviceTest);
565         EasyMock.replay(mockRescheduler);
566         mTestInvocation.invoke(mStubInvocationMetadata, mStubConfiguration, mockRescheduler);
567         verifyMocks(mockDeviceTest, mockRescheduler);
568         verifySummaryListener();
569     }
570 
571     /**
572      * Test the invoke scenario where test run throws {@link IllegalArgumentException}
573      *
574      * @throws Exception if unexpected error occurs
575      */
576     @Test
testInvoke_testFail()577     public void testInvoke_testFail() throws Throwable {
578         IllegalArgumentException exception = new IllegalArgumentException("testInvoke_testFail");
579         mExceptedStatus = FailureStatus.UNSET;
580         IRemoteTest test = EasyMock.createMock(IRemoteTest.class);
581         test.run(EasyMock.anyObject(), EasyMock.anyObject());
582         EasyMock.expectLastCall().andThrow(exception);
583         mMockPreparer.tearDown(EasyMock.anyObject(), EasyMock.eq(exception));
584         setupMockFailureListeners(exception);
585         setEarlyDeviceReleaseExpectation();
586         setupNormalInvoke(test);
587 
588         EasyMock.replay(mockRescheduler);
589         try {
590             mTestInvocation.invoke(mStubInvocationMetadata, mStubConfiguration, mockRescheduler);
591             fail("IllegalArgumentException was not rethrown");
592         } catch (IllegalArgumentException e) {
593             // expected
594         }
595         verifyMocks(test, mockRescheduler);
596         verifySummaryListener();
597     }
598 
599     /**
600      * Test metrics SHUTDOWN_HARD_LATENCY is collected when the invocation is stopped/interrupted.
601      */
602     @Test
testInvoke_metricsCollectedWhenStopped()603     public void testInvoke_metricsCollectedWhenStopped() throws Throwable {
604         IRemoteTest test = EasyMock.createMock(IRemoteTest.class);
605         test.run(EasyMock.anyObject(), EasyMock.anyObject());
606         mMockPreparer.tearDown(EasyMock.anyObject(), EasyMock.isNull());
607         setupMockStoppedListeners();
608         setEarlyDeviceReleaseExpectation();
609         setupNormalInvoke(test);
610         EasyMock.replay(mockRescheduler);
611         mTestInvocation.notifyInvocationStopped("Stopped");
612         mTestInvocation.invoke(mStubInvocationMetadata, mStubConfiguration, mockRescheduler);
613         assertTrue(
614                 mStubInvocationMetadata
615                         .getAttributes()
616                         .containsKey(InvocationMetricKey.SHUTDOWN_HARD_LATENCY.toString()));
617         verifyMocks(test, mockRescheduler);
618         verifySummaryListener();
619     }
620 
621     /**
622      * Test the invoke scenario where test run throws {@link FatalHostError}
623      *
624      * @throws Exception if unexpected error occurs
625      */
626     @Test
testInvoke_fatalError()627     public void testInvoke_fatalError() throws Throwable {
628         FatalHostError exception = new FatalHostError("testInvoke_fatalError");
629         mExceptedStatus = FailureStatus.UNSET;
630         IRemoteTest test = EasyMock.createMock(IRemoteTest.class);
631         test.run(EasyMock.anyObject(), EasyMock.anyObject());
632         EasyMock.expectLastCall().andThrow(exception);
633         setupMockFailureListeners(exception);
634         setEarlyDeviceReleaseExpectation();
635         setupNormalInvoke(test);
636         EasyMock.replay(mockRescheduler);
637         try {
638             mTestInvocation.invoke(mStubInvocationMetadata, mStubConfiguration, mockRescheduler);
639             fail("FatalHostError was not rethrown");
640         } catch (FatalHostError e)  {
641             // expected
642         }
643         verifyMocks(test, mockRescheduler);
644         verifySummaryListener();
645     }
646 
647     /**
648      * Test the invoke scenario where test run throws {@link DeviceNotAvailableException}
649      *
650      * @throws Exception if unexpected error occurs
651      */
652     @Test
testInvoke_deviceNotAvail()653     public void testInvoke_deviceNotAvail() throws Throwable {
654         DeviceNotAvailableException exception = new DeviceNotAvailableException("ERROR", SERIAL);
655         IRemoteTest test = EasyMock.createMock(IRemoteTest.class);
656         test.run(EasyMock.anyObject(), EasyMock.anyObject());
657         EasyMock.expectLastCall().andThrow(exception);
658         mMockDevice.setRecoveryMode(RecoveryMode.NONE);
659         EasyMock.expectLastCall();
660         setupMockFailureListeners(exception);
661         setEarlyDeviceReleaseExpectation();
662         setupNormalInvoke(test);
663         EasyMock.replay(mockRescheduler);
664         try {
665             mTestInvocation.invoke(mStubInvocationMetadata, mStubConfiguration, mockRescheduler);
666             fail("DeviceNotAvailableException not thrown");
667         } catch (DeviceNotAvailableException e) {
668             // expected
669         }
670         verifyMocks(test, mockRescheduler);
671         verifySummaryListener();
672     }
673 
674     @Test
testInvoke_setupError()675     public void testInvoke_setupError() throws Throwable {
676         // Use the deprecated constructor on purpose to simulate missing DeviceDescriptor.
677         TargetSetupError tse = new TargetSetupError("reason");
678         IRemoteTest test = EasyMock.createMock(IRemoteTest.class);
679         mMockDevice.setRecoveryMode(RecoveryMode.NONE);
680         EasyMock.expectLastCall();
681         EasyMock.expect(
682                         mMockDevice.logBugreport(
683                                 EasyMock.startsWith("target_setup_error_bugreport"),
684                                 EasyMock.anyObject()))
685                 .andReturn(true);
686         mMockDevice.postInvocationTearDown(tse);
687         setupMockFailureListeners(tse);
688         setEarlyDeviceReleaseExpectation();
689         setupInvokeWithBuild();
690         mStubConfiguration.setTest(test);
691         mStubMultiConfiguration.setTest(test);
692         EasyMock.expect(mMockBuildProvider.getBuild()).andReturn(mMockBuildInfo);
693         mMockPreparer.setUp(EasyMock.anyObject());
694         EasyMock.expectLastCall().andThrow(tse);
695         EasyMock.expect(mMockDevice.getIDevice()).andReturn(new StubDevice("stub"));
696         replayMocks(test, mockRescheduler);
697         mTestInvocation.invoke(mStubInvocationMetadata, mStubConfiguration, mockRescheduler);
698         verifyMocks(test, mockRescheduler);
699         verifySummaryListener();
700     }
701 
702     /**
703      * Test the invoke scenario where preparer throws {@link BuildError}
704      *
705      * @throws Exception if unexpected error occurs
706      */
707     @Test
testInvoke_buildError()708     public void testInvoke_buildError() throws Throwable {
709         BuildError exception =
710                 new BuildError(
711                         "error", mFakeDescriptor, InfraErrorIdentifier.ARTIFACT_DOWNLOAD_ERROR);
712         IRemoteTest test = EasyMock.createMock(IRemoteTest.class);
713         mStubConfiguration.setTest(test);
714         EasyMock.expect(mMockBuildProvider.getBuild()).andReturn(mMockBuildInfo);
715 
716         mMockPreparer.setUp(EasyMock.anyObject());
717         EasyMock.expectLastCall().andThrow(exception);
718         setupMockFailureListeners(exception);
719         EasyMock.expect(mMockDevice.getBugreport()).andReturn(EMPTY_STREAM_SOURCE);
720         setEarlyDeviceReleaseExpectation();
721         setupInvokeWithBuild();
722 
723         replayMocks(test);
724         EasyMock.replay(mockRescheduler);
725         mTestInvocation.invoke(mStubInvocationMetadata, mStubConfiguration, mockRescheduler);
726         verifyMocks(test, mockRescheduler);
727         verifySummaryListener();
728     }
729 
730     /**
731      * Test the {@link TestInvocation#invoke(IInvocationContext, IConfiguration, IRescheduler,
732      * ITestInvocationListener[])} scenario when a {@link ITargetPreparer} is part of the config.
733      */
734     @Test
testInvoke_tearDown()735     public void testInvoke_tearDown() throws Throwable {
736         IRemoteTest test = EasyMock.createNiceMock(IRemoteTest.class);
737         ITargetPreparer mockCleaner = EasyMock.createMock(ITargetPreparer.class);
738         EasyMock.expect(mockCleaner.isDisabled()).andReturn(false).times(2);
739         EasyMock.expect(mockCleaner.isTearDownDisabled()).andReturn(false);
740         mockCleaner.setUp(EasyMock.anyObject());
741         mockCleaner.tearDown(EasyMock.anyObject(), EasyMock.isNull());
742         mMockPreparer.tearDown(EasyMock.anyObject(), EasyMock.isNull());
743         mStubConfiguration.getTargetPreparers().add(mockCleaner);
744         setupMockSuccessListeners();
745         setEarlyDeviceReleaseExpectation();
746         setupNormalInvoke(test);
747         EasyMock.replay(mockCleaner, mockRescheduler);
748         mTestInvocation.invoke(mStubInvocationMetadata, mStubConfiguration, mockRescheduler);
749         verifyMocks(mockCleaner, mockRescheduler);
750         verifySummaryListener();
751     }
752 
753     /**
754      * Test the {@link TestInvocation#invoke(IInvocationContext, IConfiguration, IRescheduler,
755      * ITestInvocationListener[])} scenario when a {@link ITargetPreparer} is part of the config,
756      * and the test throws a {@link DeviceNotAvailableException}.
757      */
758     @Test
testInvoke_tearDown_deviceNotAvail()759     public void testInvoke_tearDown_deviceNotAvail() throws Throwable {
760         DeviceNotAvailableException exception = new DeviceNotAvailableException("ERROR", SERIAL);
761         IRemoteTest test = EasyMock.createMock(IRemoteTest.class);
762         test.run(EasyMock.anyObject(), EasyMock.anyObject());
763         EasyMock.expectLastCall().andThrow(exception);
764         ITargetPreparer mockCleaner = EasyMock.createMock(ITargetPreparer.class);
765         EasyMock.expect(mockCleaner.isDisabled()).andReturn(false).times(2);
766         EasyMock.expect(mockCleaner.isTearDownDisabled()).andReturn(false);
767         mockCleaner.setUp(EasyMock.anyObject());
768         EasyMock.expectLastCall();
769         mockCleaner.tearDown(EasyMock.anyObject(), EasyMock.eq(exception));
770         EasyMock.expectLastCall();
771         mMockDevice.setRecoveryMode(RecoveryMode.NONE);
772         EasyMock.expectLastCall();
773         EasyMock.replay(mockCleaner, mockRescheduler);
774         mStubConfiguration.getTargetPreparers().add(mockCleaner);
775         setupMockFailureListeners(exception);
776         setEarlyDeviceReleaseExpectation();
777         setupNormalInvoke(test);
778         try {
779             mTestInvocation.invoke(mStubInvocationMetadata, mStubConfiguration, mockRescheduler);
780             fail("DeviceNotAvailableException not thrown");
781         } catch (DeviceNotAvailableException e) {
782             // expected
783         }
784         verifyMocks(mockCleaner, mockRescheduler);
785         verifySummaryListener();
786     }
787 
788     /**
789      * Test the {@link TestInvocation#invoke(IInvocationContext, IConfiguration, IRescheduler,
790      * ITestInvocationListener[])} scenario when a {@link ITargetCleaner} is part of the config, and
791      * the test throws a {@link RuntimeException}.
792      */
793     @Test
testInvoke_tearDown_runtime()794     public void testInvoke_tearDown_runtime() throws Throwable {
795         RuntimeException exception = new RuntimeException("testInvoke_tearDown_runtime");
796         mExceptedStatus = FailureStatus.UNSET;
797         IRemoteTest test = EasyMock.createMock(IRemoteTest.class);
798         test.run(EasyMock.anyObject(), EasyMock.anyObject());
799         EasyMock.expectLastCall().andThrow(exception);
800         ITargetCleaner mockCleaner = EasyMock.createMock(ITargetCleaner.class);
801         EasyMock.expect(mockCleaner.isDisabled()).andReturn(false).times(2);
802         EasyMock.expect(mockCleaner.isTearDownDisabled()).andReturn(false);
803         mockCleaner.setUp(EasyMock.anyObject());
804         // tearDown should be called
805         mockCleaner.tearDown(EasyMock.anyObject(), EasyMock.eq(exception));
806         mStubConfiguration.getTargetPreparers().add(mockCleaner);
807         setupMockFailureListeners(exception);
808         setEarlyDeviceReleaseExpectation();
809         setupNormalInvoke(test);
810         EasyMock.replay(mockCleaner, mockRescheduler);
811         try {
812             mTestInvocation.invoke(mStubInvocationMetadata, mStubConfiguration, mockRescheduler);
813             fail("RuntimeException not thrown");
814         } catch (RuntimeException e) {
815             // expected
816         }
817         verifyMocks(mockCleaner, mockRescheduler);
818         verifySummaryListener();
819     }
820 
821     /**
822      * Test the {@link TestInvocation#invoke(IInvocationContext, IConfiguration, IRescheduler,
823      * ITestInvocationListener[])} scenario when there is {@link ITestInvocationListener} which
824      * implements the {@link ILogSaverListener} interface.
825      */
826     @Test
testInvoke_logFileSaved()827     public void testInvoke_logFileSaved() throws Throwable {
828         List<ITestInvocationListener> listenerList =
829                 mStubConfiguration.getTestInvocationListeners();
830         ILogSaverListener logSaverListener = EasyMock.createMock(ILogSaverListener.class);
831         listenerList.add(logSaverListener);
832         mStubConfiguration.setTestInvocationListeners(listenerList);
833 
834         logSaverListener.setLogSaver(mMockLogSaver);
835         logSaverListener.invocationStarted(mStubInvocationMetadata);
836         logSaverListener.testLog(
837                 EasyMock.startsWith(LOGCAT_NAME_SETUP),
838                 EasyMock.eq(LogDataType.LOGCAT),
839                 (InputStreamSource) EasyMock.anyObject());
840         logSaverListener.testLogSaved(
841                 EasyMock.startsWith(LOGCAT_NAME_SETUP),
842                 EasyMock.eq(LogDataType.LOGCAT),
843                 (InputStreamSource) EasyMock.anyObject(),
844                 (LogFile) EasyMock.anyObject());
845         logSaverListener.logAssociation(
846                 EasyMock.startsWith(LOGCAT_NAME_SETUP), EasyMock.anyObject());
847         logSaverListener.testLog(
848                 EasyMock.startsWith(LOGCAT_NAME_TEST),
849                 EasyMock.eq(LogDataType.LOGCAT),
850                 (InputStreamSource) EasyMock.anyObject());
851         logSaverListener.testLogSaved(
852                 EasyMock.startsWith(LOGCAT_NAME_TEST),
853                 EasyMock.eq(LogDataType.LOGCAT),
854                 (InputStreamSource) EasyMock.anyObject(),
855                 (LogFile) EasyMock.anyObject());
856         logSaverListener.logAssociation(
857                 EasyMock.startsWith(LOGCAT_NAME_TEST), EasyMock.anyObject());
858         logSaverListener.testLog(
859                 EasyMock.startsWith(LOGCAT_NAME_TEARDOWN),
860                 EasyMock.eq(LogDataType.LOGCAT),
861                 (InputStreamSource) EasyMock.anyObject());
862         logSaverListener.testLogSaved(
863                 EasyMock.startsWith(LOGCAT_NAME_TEARDOWN),
864                 EasyMock.eq(LogDataType.LOGCAT),
865                 (InputStreamSource) EasyMock.anyObject(),
866                 (LogFile) EasyMock.anyObject());
867         logSaverListener.logAssociation(
868                 EasyMock.startsWith(LOGCAT_NAME_TEARDOWN), EasyMock.anyObject());
869         logSaverListener.testLog(
870                 EasyMock.eq(TestInvocation.TRADEFED_LOG_NAME),
871                 EasyMock.eq(LogDataType.HOST_LOG),
872                 (InputStreamSource) EasyMock.anyObject());
873         logSaverListener.testLogSaved(
874                 EasyMock.eq(TestInvocation.TRADEFED_LOG_NAME),
875                 EasyMock.eq(LogDataType.HOST_LOG),
876                 (InputStreamSource) EasyMock.anyObject(),
877                 (LogFile) EasyMock.anyObject());
878         logSaverListener.logAssociation(
879                 EasyMock.eq(TestInvocation.TRADEFED_LOG_NAME), EasyMock.anyObject());
880         logSaverListener.invocationEnded(EasyMock.anyLong());
881         EasyMock.expect(logSaverListener.getSummary()).andReturn(mSummary).times(2);
882 
883         IRemoteTest test = EasyMock.createMock(IRemoteTest.class);
884         setupMockSuccessListeners();
885         test.run(EasyMock.anyObject(), EasyMock.anyObject());
886         mMockPreparer.tearDown(EasyMock.anyObject(), EasyMock.isNull());
887         setEarlyDeviceReleaseExpectation();
888         setupNormalInvoke(test);
889         EasyMock.replay(logSaverListener, mockRescheduler);
890         mTestInvocation.invoke(mStubInvocationMetadata, mStubConfiguration, mockRescheduler);
891         verifyMocks(test, logSaverListener, mockRescheduler);
892         assertEquals(2, mUriCapture.getValue().size());
893     }
894 
895     /** Test the test-tag is set when the IBuildInfo's test-tag is not. */
896     @Test
testInvoke_testtag()897     public void testInvoke_testtag() throws Throwable {
898         String[] commandLine = {"run", "empty"};
899         mStubConfiguration.setCommandLine(commandLine);
900         mStubConfiguration.getCommandOptions().setTestTag("not-default");
901         setEarlyDeviceReleaseExpectation();
902         setupInvoke();
903         EasyMock.expect(mMockDevice.getLogcat()).andReturn(EMPTY_STREAM_SOURCE).times(3);
904         mMockDevice.clearLogcat();
905         EasyMock.expectLastCall().times(3);
906         EasyMock.expect(mMockLogger.getLog()).andReturn(EMPTY_STREAM_SOURCE);
907         mMockBuildInfo.setDeviceSerial(SERIAL);
908         mMockBuildProvider.cleanUp(mMockBuildInfo);
909         setupMockSuccessListeners();
910         EasyMock.expect(mMockBuildProvider.getBuild()).andReturn(mMockBuildInfo);
911         mMockBuildInfo.addBuildAttribute("command_line_args", "run empty");
912         mMockPreparer.setUp(EasyMock.anyObject());
913         mMockPreparer.tearDown(EasyMock.anyObject(), EasyMock.isNull());
914         // Default build is "stub" so we set the test-tag
915         mMockBuildInfo.setTestTag(EasyMock.eq("not-default"));
916         EasyMock.expectLastCall();
917         EasyMock.expect(mMockBuildInfo.getTestTag()).andStubReturn("stub");
918         mMockLogRegistry.unregisterLogger();
919         mMockLogRegistry.dumpToGlobalLog(mMockLogger);
920         mMockLogger.closeLog();
921         replayMocks();
922         mTestInvocation.invoke(mStubInvocationMetadata, mStubConfiguration, mockRescheduler);
923         verifyMocks();
924     }
925 
926     /**
927      * Test the test-tag of the IBuildInfo is not modified when the CommandOption default test-tag
928      * is not modified.
929      */
930     @Test
testInvoke_testtag_notset()931     public void testInvoke_testtag_notset() throws Throwable {
932         String[] commandLine = {"run", "empty"};
933         mStubConfiguration.setCommandLine(commandLine);
934         setEarlyDeviceReleaseExpectation();
935         setupInvoke();
936         EasyMock.expect(mMockDevice.getLogcat()).andReturn(EMPTY_STREAM_SOURCE).times(3);
937         mMockDevice.clearLogcat();
938         EasyMock.expectLastCall().times(3);
939         EasyMock.expect(mMockLogger.getLog()).andReturn(EMPTY_STREAM_SOURCE);
940         mMockBuildInfo.setDeviceSerial(SERIAL);
941         mMockBuildProvider.cleanUp(mMockBuildInfo);
942         setupMockSuccessListeners();
943         EasyMock.expect(mMockBuildProvider.getBuild()).andReturn(mMockBuildInfo);
944         mMockBuildInfo.addBuildAttribute("command_line_args", "run empty");
945         mMockPreparer.setUp(EasyMock.anyObject());
946         mMockPreparer.tearDown(EasyMock.anyObject(), EasyMock.isNull());
947         EasyMock.expect(mMockBuildInfo.getTestTag()).andStubReturn("buildprovidertesttag");
948         mMockLogRegistry.dumpToGlobalLog(mMockLogger);
949         mMockLogRegistry.unregisterLogger();
950         mMockLogger.closeLog();
951         replayMocks();
952         mTestInvocation.invoke(mStubInvocationMetadata, mStubConfiguration, mockRescheduler);
953         verifyMocks();
954     }
955 
956     /**
957      * Test the test-tag of the IBuildInfo is not set and Command Option is not set either. A
958      * default 'stub' test-tag is set to ensure reporting is done.
959      */
960     @Test
testInvoke_notesttag()961     public void testInvoke_notesttag() throws Throwable {
962         String[] commandLine = {"run", "empty"};
963         mStubConfiguration.setCommandLine(commandLine);
964         setEarlyDeviceReleaseExpectation();
965         setupInvoke();
966         EasyMock.expect(mMockDevice.getLogcat()).andReturn(EMPTY_STREAM_SOURCE).times(3);
967         mMockDevice.clearLogcat();
968         EasyMock.expectLastCall().times(3);
969         EasyMock.expect(mMockLogger.getLog()).andReturn(EMPTY_STREAM_SOURCE);
970         mMockBuildInfo.setDeviceSerial(SERIAL);
971         mMockBuildProvider.cleanUp(mMockBuildInfo);
972         setupMockSuccessListeners();
973         EasyMock.expect(mMockBuildProvider.getBuild()).andReturn(mMockBuildInfo);
974         mMockBuildInfo.addBuildAttribute("command_line_args", "run empty");
975         mMockPreparer.setUp(EasyMock.anyObject());
976         mMockPreparer.tearDown(EasyMock.anyObject(), EasyMock.isNull());
977         EasyMock.expect(mMockBuildInfo.getTestTag()).andStubReturn(null);
978         mMockBuildInfo.setTestTag(EasyMock.eq("stub"));
979         EasyMock.expectLastCall();
980         mMockLogRegistry.dumpToGlobalLog(mMockLogger);
981         mMockLogRegistry.unregisterLogger();
982         mMockLogger.closeLog();
983         replayMocks();
984         mTestInvocation.invoke(mStubInvocationMetadata, mStubConfiguration, mockRescheduler);
985         verifyMocks();
986     }
987 
988     /**
989      * Helper tests class to expose all the interfaces needed for the tests.
990      */
991     private interface IFakeBuildProvider extends IDeviceBuildProvider, IInvocationContextReceiver {
992     }
993 
994     /**
995      * Test the injection of test-tag from TestInvocation to the build provider via the {@link
996      * IInvocationContextReceiver}.
997      */
998     @Test
testInvoke_buildProviderNeedTestTag()999     public void testInvoke_buildProviderNeedTestTag() throws Throwable {
1000         final String testTag = "THISISTHETAG";
1001         String[] commandLine = {"run", "empty"};
1002         mStubConfiguration.setCommandLine(commandLine);
1003         ICommandOptions commandOption = new CommandOptions();
1004         commandOption.setTestTag(testTag);
1005         IFakeBuildProvider mockProvider = EasyMock.createMock(IFakeBuildProvider.class);
1006         mStubConfiguration.setBuildProvider(mockProvider);
1007         mStubConfiguration.setCommandOptions(commandOption);
1008         setEarlyDeviceReleaseExpectation();
1009         setupInvoke();
1010         EasyMock.expect(mMockDevice.getLogcat()).andReturn(EMPTY_STREAM_SOURCE).times(3);
1011         mMockDevice.clearLogcat();
1012         EasyMock.expectLastCall().times(3);
1013         EasyMock.expect(mMockLogger.getLog()).andReturn(EMPTY_STREAM_SOURCE);
1014         mMockBuildInfo.setDeviceSerial(SERIAL);
1015         setupMockSuccessListeners();
1016         mMockBuildInfo.addBuildAttribute("command_line_args", "run empty");
1017         mMockPreparer.setUp(EasyMock.anyObject());
1018         mMockPreparer.tearDown(EasyMock.anyObject(), EasyMock.isNull());
1019         EasyMock.expect(mMockBuildInfo.getTestTag()).andStubReturn(null);
1020         // Validate proper tag is set on the build.
1021         mMockBuildInfo.setTestTag(EasyMock.eq(testTag));
1022         mockProvider.setInvocationContext((IInvocationContext)EasyMock.anyObject());
1023         EasyMock.expect(mockProvider.getBuild(mMockDevice)).andReturn(mMockBuildInfo);
1024         EasyMock.expectLastCall().anyTimes();
1025         mockProvider.cleanUp(mMockBuildInfo);
1026         mMockLogRegistry.dumpToGlobalLog(mMockLogger);
1027         mMockLogRegistry.unregisterLogger();
1028         mMockLogger.closeLog();
1029 
1030         replayMocks(mockProvider);
1031         mTestInvocation.invoke(mStubInvocationMetadata, mStubConfiguration, mockRescheduler);
1032         verifyMocks(mockProvider);
1033     }
1034 
1035     /**
1036      * Set up expected conditions for normal run up to the part where tests are run.
1037      *
1038      * @param test the {@link Test} to use.
1039      */
setupNormalInvoke(IRemoteTest test)1040     private void setupNormalInvoke(IRemoteTest test) throws Throwable {
1041         setupInvokeWithBuild();
1042         mStubConfiguration.setTest(test);
1043         mStubMultiConfiguration.setTest(test);
1044         EasyMock.expect(mMockBuildProvider.getBuild()).andReturn(mMockBuildInfo);
1045 
1046         mMockPreparer.setUp(EasyMock.anyObject());
1047         replayMocks(test);
1048     }
1049 
1050     /**
1051      * Set up expected calls that occur on every invoke, regardless of result
1052      */
setupInvoke()1053     private void setupInvoke() {
1054         mMockDevice.clearLastConnectedWifiNetwork();
1055         mMockDevice.setOptions((TestDeviceOptions)EasyMock.anyObject());
1056         mMockDevice.startLogcat();
1057         mMockDevice.clearLastConnectedWifiNetwork();
1058         mMockDevice.stopLogcat();
1059         EasyMock.expectLastCall().anyTimes();
1060     }
1061 
1062     /**
1063      * Set up expected calls that occur on every invoke that gets a valid build
1064      */
setupInvokeWithBuild()1065     private void setupInvokeWithBuild() {
1066         setupInvoke();
1067         EasyMock.expect(mMockDevice.getLogcat()).andReturn(EMPTY_STREAM_SOURCE).times(3);
1068         mMockDevice.clearLogcat();
1069         EasyMock.expectLastCall().times(3);
1070 
1071         EasyMock.expect(mMockLogger.getLog()).andReturn(EMPTY_STREAM_SOURCE);
1072         mMockBuildInfo.setDeviceSerial(SERIAL);
1073         mMockBuildProvider.cleanUp(mMockBuildInfo);
1074         EasyMock.expectLastCall().anyTimes();
1075         mMockBuildInfo.setTestTag(EasyMock.eq("stub"));
1076         EasyMock.expectLastCall();
1077         EasyMock.expect(mMockBuildInfo.getTestTag()).andStubReturn("");
1078 
1079         mMockLogRegistry.unregisterLogger();
1080         mMockLogRegistry.dumpToGlobalLog(mMockLogger);
1081         mMockLogger.closeLog();
1082     }
1083 
1084     /**
1085      * Set up expected conditions for the test InvocationListener and SummaryListener
1086      *
1087      * <p>The order of calls for a single listener should be:
1088      *
1089      * <ol>
1090      *   <li>invocationStarted
1091      *   <li>testLog(LOGCAT_NAME_SETUP, ...) (if no build or retrieval error)
1092      *   <li>invocationFailed (if run failed)
1093      *   <li>testLog(LOGCAT_NAME_ERROR, ...) (if build retrieval error)
1094      *   <li>testLog(LOGCAT_NAME_TEST, ...) (otherwise)
1095      *   <li>testLog(build error bugreport, ...) (otherwise and if build error)
1096      *   <li>testLog(LOGCAT_NAME_TEARDOWN, ...) (otherwise)
1097      *   <li>testLog(TRADEFED_LOG_NAME, ...)
1098      *   <li>putSummary (for an ITestSummaryListener)
1099      *   <li>invocationEnded
1100      *   <li>getSummary (for an ITestInvocationListener)
1101      * </ol>
1102      *
1103      * However note that, across all listeners, any getSummary call will precede all putSummary
1104      * calls.
1105      */
setupMockListeners( InvocationStatus status, Throwable throwable, boolean stubFailures, boolean reportHostLog, boolean stopped)1106     private void setupMockListeners(
1107             InvocationStatus status,
1108             Throwable throwable,
1109             boolean stubFailures,
1110             boolean reportHostLog,
1111             boolean stopped)
1112             throws IOException {
1113         // invocationStarted
1114         mMockLogSaver.invocationStarted(mStubInvocationMetadata);
1115         mMockTestListener.invocationStarted(mStubInvocationMetadata);
1116         EasyMock.expect(mMockTestListener.getSummary()).andReturn(null);
1117         mMockSummaryListener.putEarlySummary(EasyMock.anyObject());
1118         mMockSummaryListener.invocationStarted(mStubInvocationMetadata);
1119         EasyMock.expect(mMockSummaryListener.getSummary()).andReturn(null);
1120 
1121         if (throwable == null) {
1122             mMockDevice.postInvocationTearDown(null);
1123             EasyMock.expectLastCall().anyTimes();
1124         } else {
1125             mMockDevice.postInvocationTearDown(throwable);
1126             EasyMock.expectLastCall().anyTimes();
1127         }
1128 
1129         if (!(throwable instanceof BuildRetrievalError)) {
1130             EasyMock.expect(
1131                             mMockLogSaver.saveLogData(
1132                                     EasyMock.startsWith(LOGCAT_NAME_SETUP),
1133                                     EasyMock.eq(LogDataType.LOGCAT),
1134                                     (InputStream) EasyMock.anyObject()))
1135                     .andReturn(new LogFile(PATH, URL, LogDataType.TEXT));
1136             mMockTestListener.testLog(
1137                     EasyMock.startsWith(LOGCAT_NAME_SETUP),
1138                     EasyMock.eq(LogDataType.LOGCAT),
1139                     (InputStreamSource) EasyMock.anyObject());
1140             mMockSummaryListener.testLog(
1141                     EasyMock.startsWith(LOGCAT_NAME_SETUP),
1142                     EasyMock.eq(LogDataType.LOGCAT),
1143                     (InputStreamSource) EasyMock.anyObject());
1144         }
1145 
1146         // invocationFailed
1147         if (!status.equals(InvocationStatus.SUCCESS)) {
1148             if (stubFailures) {
1149                 mMockTestListener.invocationFailed(EasyMock.<FailureDescription>anyObject());
1150                 mMockSummaryListener.invocationFailed(EasyMock.<FailureDescription>anyObject());
1151             } else {
1152                 FailureDescription failure =
1153                         FailureDescription.create(
1154                                         throwable.getMessage(), FailureStatus.INFRA_FAILURE)
1155                                 .setCause(throwable);
1156                 if (throwable instanceof BuildRetrievalError) {
1157                     failure.setActionInProgress(ActionInProgress.FETCHING_ARTIFACTS);
1158                 } else if (throwable instanceof BuildError
1159                         || throwable instanceof TargetSetupError) {
1160                     failure.setActionInProgress(ActionInProgress.SETUP);
1161                 } else {
1162                     failure.setActionInProgress(ActionInProgress.TEST);
1163                 }
1164                 if (mExceptedStatus != null) {
1165                     failure.setFailureStatus(mExceptedStatus);
1166                 }
1167                 mMockTestListener.invocationFailed(EasyMock.eq(failure));
1168                 mMockSummaryListener.invocationFailed(EasyMock.eq(failure));
1169             }
1170         }
1171 
1172         if (throwable instanceof BuildRetrievalError) {
1173             // Handle logcat error listeners
1174             EasyMock.expect(
1175                             mMockLogSaver.saveLogData(
1176                                     EasyMock.startsWith(LOGCAT_NAME_ERROR),
1177                                     EasyMock.eq(LogDataType.LOGCAT),
1178                                     (InputStream) EasyMock.anyObject()))
1179                     .andReturn(new LogFile(PATH, URL, LogDataType.TEXT));
1180             mMockTestListener.testLog(
1181                     EasyMock.startsWith(LOGCAT_NAME_ERROR),
1182                     EasyMock.eq(LogDataType.LOGCAT),
1183                     (InputStreamSource) EasyMock.anyObject());
1184             mMockSummaryListener.testLog(
1185                     EasyMock.startsWith(LOGCAT_NAME_ERROR),
1186                     EasyMock.eq(LogDataType.LOGCAT),
1187                     (InputStreamSource) EasyMock.anyObject());
1188         } else {
1189             // Handle build error bugreport listeners
1190             if (throwable instanceof BuildError) {
1191                 EasyMock.expect(
1192                                 mMockDevice.logBugreport(
1193                                         EasyMock.eq(
1194                                                 TestInvocation.BUILD_ERROR_BUGREPORT_NAME
1195                                                         + "_"
1196                                                         + SERIAL),
1197                                         EasyMock.anyObject()))
1198                         .andReturn(true);
1199             } else if (!(throwable instanceof TargetSetupError)) {
1200                 // Handle test logcat listeners
1201                 EasyMock.expect(
1202                                 mMockLogSaver.saveLogData(
1203                                         EasyMock.startsWith(LOGCAT_NAME_TEST),
1204                                         EasyMock.eq(LogDataType.LOGCAT),
1205                                         (InputStream) EasyMock.anyObject()))
1206                         .andReturn(new LogFile(PATH, URL, LogDataType.TEXT));
1207                 mMockTestListener.testLog(
1208                         EasyMock.startsWith(LOGCAT_NAME_TEST),
1209                         EasyMock.eq(LogDataType.LOGCAT),
1210                         (InputStreamSource) EasyMock.anyObject());
1211                 mMockSummaryListener.testLog(
1212                         EasyMock.startsWith(LOGCAT_NAME_TEST),
1213                         EasyMock.eq(LogDataType.LOGCAT),
1214                         (InputStreamSource) EasyMock.anyObject());
1215             }
1216             // Handle teardown logcat listeners
1217             EasyMock.expect(
1218                             mMockLogSaver.saveLogData(
1219                                     EasyMock.startsWith(LOGCAT_NAME_TEARDOWN),
1220                                     EasyMock.eq(LogDataType.LOGCAT),
1221                                     (InputStream) EasyMock.anyObject()))
1222                     .andReturn(new LogFile(PATH, URL, LogDataType.TEXT));
1223             mMockTestListener.testLog(
1224                     EasyMock.startsWith(LOGCAT_NAME_TEARDOWN),
1225                     EasyMock.eq(LogDataType.LOGCAT),
1226                     (InputStreamSource) EasyMock.anyObject());
1227             mMockSummaryListener.testLog(
1228                     EasyMock.startsWith(LOGCAT_NAME_TEARDOWN),
1229                     EasyMock.eq(LogDataType.LOGCAT),
1230                     (InputStreamSource) EasyMock.anyObject());
1231 
1232             if (stopped) {
1233                 mMockTestListener.invocationFailed(EasyMock.<FailureDescription>anyObject());
1234                 mMockSummaryListener.invocationFailed(EasyMock.<FailureDescription>anyObject());
1235             }
1236         }
1237 
1238         EasyMock.expect(
1239                         mMockLogSaver.saveLogData(
1240                                 EasyMock.eq(TestInvocation.TRADEFED_END_HOST_LOG),
1241                                 EasyMock.eq(LogDataType.HOST_LOG),
1242                                 (InputStream) EasyMock.anyObject()))
1243                 .andReturn(new LogFile(PATH, URL, LogDataType.HOST_LOG));
1244         if (reportHostLog) {
1245             EasyMock.expect(
1246                             mMockLogSaver.saveLogData(
1247                                     EasyMock.eq(TestInvocation.TRADEFED_LOG_NAME),
1248                                     EasyMock.eq(LogDataType.HOST_LOG),
1249                                     (InputStream) EasyMock.anyObject()))
1250                     .andReturn(new LogFile(PATH, URL, LogDataType.HOST_LOG));
1251             mMockTestListener.testLog(
1252                     EasyMock.eq(TestInvocation.TRADEFED_LOG_NAME),
1253                     EasyMock.eq(LogDataType.HOST_LOG),
1254                     (InputStreamSource) EasyMock.anyObject());
1255             mMockSummaryListener.testLog(
1256                     EasyMock.eq(TestInvocation.TRADEFED_LOG_NAME),
1257                     EasyMock.eq(LogDataType.HOST_LOG),
1258                     (InputStreamSource) EasyMock.anyObject());
1259         }
1260 
1261         // invocationEnded, getSummary (mMockTestListener)
1262         mMockTestListener.invocationEnded(EasyMock.anyLong());
1263         EasyMock.expect(mMockTestListener.getSummary()).andReturn(mSummary);
1264 
1265         // putSummary, invocationEnded (mMockSummaryListener)
1266         mMockSummaryListener.putSummary(EasyMock.capture(mUriCapture));
1267         mMockSummaryListener.invocationEnded(EasyMock.anyLong());
1268         mMockLogSaver.invocationEnded(EasyMock.anyLong());
1269     }
1270 
1271     /**
1272      * Test the {@link TestInvocation#invoke(IInvocationContext, IConfiguration, IRescheduler,
1273      * ITestInvocationListener[])} scenario with {@link IShardableTest}.
1274      */
1275     @Test
testInvoke_shardableTest_legacy()1276     public void testInvoke_shardableTest_legacy() throws Throwable {
1277         TestInformation info =
1278                 TestInformation.newBuilder().setInvocationContext(mStubInvocationMetadata).build();
1279         mStubConfiguration.setConfigurationObject(ShardHelper.SHARED_TEST_INFORMATION, info);
1280         String command = "empty --test-tag t";
1281         String[] commandLine = {"empty", "--test-tag", "t"};
1282         int shardCount = 2;
1283         IShardableTest test = EasyMock.createMock(IShardableTest.class);
1284         List<IRemoteTest> shards = new ArrayList<>();
1285         IRemoteTest shard1 = EasyMock.createMock(IRemoteTest.class);
1286         IRemoteTest shard2 = EasyMock.createMock(IRemoteTest.class);
1287         shards.add(shard1);
1288         shards.add(shard2);
1289         EasyMock.expect(test.split(EasyMock.isNull(), EasyMock.anyObject())).andReturn(shards);
1290         mStubConfiguration.setTest(test);
1291         mStubConfiguration.setCommandLine(commandLine);
1292         mMockBuildProvider.cleanUp(mMockBuildInfo);
1293         // The keystore is cloned for each shard.
1294         EasyMock.expect(mGlobalConfiguration.getKeyStoreFactory())
1295                 .andReturn(new StubKeyStoreFactory())
1296                 .times(2);
1297         setupInvoke();
1298         EasyMock.reset(mMockLogger, mMockLogRegistry);
1299         mMockLogRegistry.registerLogger(mMockLogger);
1300         mMockLogger.init();
1301         mMockLogger.closeLog();
1302         mMockLogRegistry.unregisterLogger();
1303         mMockLogSaver.invocationStarted(mStubInvocationMetadata);
1304         mMockLogSaver.invocationEnded(0L);
1305         setupNShardInvocation(shardCount, command);
1306         // Ensure that the host_log gets logged after sharding.
1307         EasyMock.expect(mMockLogger.getLog()).andReturn(EMPTY_STREAM_SOURCE);
1308         String logName = "host_log_before_sharding";
1309         mMockTestListener.testLog(
1310                 EasyMock.eq(logName), EasyMock.eq(LogDataType.HOST_LOG), EasyMock.anyObject());
1311         mMockSummaryListener.testLog(
1312                 EasyMock.eq(logName), EasyMock.eq(LogDataType.HOST_LOG), EasyMock.anyObject());
1313         EasyMock.expect(
1314                         mMockLogSaver.saveLogData(
1315                                 EasyMock.eq(logName),
1316                                 EasyMock.eq(LogDataType.HOST_LOG),
1317                                 EasyMock.anyObject()))
1318                 .andReturn(new LogFile(PATH, URL, LogDataType.HOST_LOG));
1319         mMockLogRegistry.unregisterLogger();
1320         EasyMock.expectLastCall();
1321         mMockLogger.closeLog();
1322         EasyMock.expectLastCall();
1323 
1324         mMockLogRegistry.dumpToGlobalLog(mMockLogger);
1325         replayMocks(test, mockRescheduler, shard1, shard2, mGlobalConfiguration);
1326         mTestInvocation.invoke(mStubInvocationMetadata, mStubConfiguration, mockRescheduler);
1327         verifyMocks(test, mockRescheduler, shard1, shard2, mGlobalConfiguration);
1328     }
1329 
1330     /** Test that the before sharding log is properly carried even with auto-retry. */
1331     @Test
testInvoke_shardableTest_autoRetry()1332     public void testInvoke_shardableTest_autoRetry() throws Throwable {
1333         TestInformation info =
1334                 TestInformation.newBuilder().setInvocationContext(mStubInvocationMetadata).build();
1335         mStubConfiguration.setConfigurationObject(ShardHelper.SHARED_TEST_INFORMATION, info);
1336         List<ITestInvocationListener> listenerList =
1337                 mStubConfiguration.getTestInvocationListeners();
1338         ILogSaverListener logSaverListener = EasyMock.createMock(ILogSaverListener.class);
1339         listenerList.add(logSaverListener);
1340         mStubConfiguration.setTestInvocationListeners(listenerList);
1341 
1342         logSaverListener.setLogSaver(mMockLogSaver);
1343         logSaverListener.invocationStarted(mStubInvocationMetadata);
1344 
1345         String command = "empty --test-tag t";
1346         String[] commandLine = {"empty", "--test-tag", "t"};
1347         int shardCount = 2;
1348         IShardableTest test = EasyMock.createMock(IShardableTest.class);
1349         List<IRemoteTest> shards = new ArrayList<>();
1350         IRemoteTest shard1 = EasyMock.createMock(IRemoteTest.class);
1351         IRemoteTest shard2 = EasyMock.createMock(IRemoteTest.class);
1352         shards.add(shard1);
1353         shards.add(shard2);
1354         EasyMock.expect(test.split(EasyMock.isNull(), EasyMock.anyObject())).andReturn(shards);
1355         mStubConfiguration.setTest(test);
1356         mStubConfiguration.setCommandLine(commandLine);
1357 
1358         IRetryDecision decision = mStubConfiguration.getRetryDecision();
1359         OptionSetter decisionSetter = new OptionSetter(decision);
1360         decisionSetter.setOptionValue("auto-retry", "true");
1361         decisionSetter.setOptionValue("max-testcase-run-count", "2");
1362         decisionSetter.setOptionValue("retry-strategy", "RETRY_ANY_FAILURE");
1363         mMockBuildProvider.cleanUp(mMockBuildInfo);
1364         // The keystore is cloned for each shard.
1365         EasyMock.expect(mGlobalConfiguration.getKeyStoreFactory())
1366                 .andReturn(new StubKeyStoreFactory())
1367                 .times(2);
1368         setupInvoke();
1369         EasyMock.reset(mMockLogger, mMockLogRegistry);
1370         mMockLogRegistry.registerLogger(mMockLogger);
1371         mMockLogger.init();
1372         mMockLogger.closeLog();
1373         mMockLogRegistry.unregisterLogger();
1374         mMockLogSaver.invocationStarted(mStubInvocationMetadata);
1375         mMockLogSaver.invocationEnded(0L);
1376         setupNShardInvocation(shardCount, command);
1377         // Ensure that the host_log gets logged after sharding.
1378         EasyMock.expect(mMockLogger.getLog()).andReturn(EMPTY_STREAM_SOURCE);
1379         String logName = "host_log_before_sharding";
1380         LogFile loggedFile = new LogFile(PATH, URL, LogDataType.HOST_LOG);
1381         for (ITestInvocationListener listener : listenerList) {
1382             listener.testLog(
1383                     EasyMock.eq(logName), EasyMock.eq(LogDataType.HOST_LOG), EasyMock.anyObject());
1384         }
1385         EasyMock.expect(
1386                         mMockLogSaver.saveLogData(
1387                                 EasyMock.eq(logName),
1388                                 EasyMock.eq(LogDataType.HOST_LOG),
1389                                 EasyMock.anyObject()))
1390                 .andReturn(loggedFile);
1391         logSaverListener.logAssociation(logName, loggedFile);
1392         mMockLogRegistry.unregisterLogger();
1393         EasyMock.expectLastCall();
1394         mMockLogger.closeLog();
1395         EasyMock.expectLastCall();
1396 
1397         mMockLogRegistry.dumpToGlobalLog(mMockLogger);
1398         replayMocks(test, mockRescheduler, shard1, shard2, mGlobalConfiguration, logSaverListener);
1399         mTestInvocation.invoke(mStubInvocationMetadata, mStubConfiguration, mockRescheduler);
1400         verifyMocks(test, mockRescheduler, shard1, shard2, mGlobalConfiguration, logSaverListener);
1401     }
1402 
1403     /**
1404      * Test that {@link TestInvocation#logDeviceBatteryLevel(IInvocationContext, String)} is not
1405      * adding battery information for placeholder device.
1406      */
1407     @Test
testLogDeviceBatteryLevel_placeholderDevice()1408     public void testLogDeviceBatteryLevel_placeholderDevice() {
1409         final String fakeEvent = "event";
1410         IInvocationContext context = new InvocationContext();
1411         ITestDevice device1 = EasyMock.createMock(ITestDevice.class);
1412         EasyMock.expect(device1.getIDevice()).andReturn(new StubDevice("stub"));
1413         context.addAllocatedDevice("device1", device1);
1414         EasyMock.replay(device1);
1415         mTestInvocation.logDeviceBatteryLevel(context, fakeEvent);
1416         EasyMock.verify(device1);
1417         assertEquals(0, context.getAttributes().size());
1418     }
1419 
1420     /**
1421      * Test that {@link TestInvocation#logDeviceBatteryLevel(IInvocationContext, String)} is adding
1422      * battery information for physical real device.
1423      */
1424     @Test
testLogDeviceBatteryLevel_physicalDevice()1425     public void testLogDeviceBatteryLevel_physicalDevice() {
1426         final String fakeEvent = "event";
1427         IInvocationContext context = new InvocationContext();
1428         ITestDevice device1 = EasyMock.createMock(ITestDevice.class);
1429         EasyMock.expect(device1.getIDevice()).andReturn(EasyMock.createMock(IDevice.class));
1430         EasyMock.expect(device1.getSerialNumber()).andReturn("serial1");
1431         EasyMock.expect(device1.getBattery()).andReturn(50);
1432         context.addAllocatedDevice("device1", device1);
1433         context.addDeviceBuildInfo("device1", new BuildInfo());
1434         EasyMock.replay(device1);
1435         mTestInvocation.logDeviceBatteryLevel(context, fakeEvent);
1436         EasyMock.verify(device1);
1437         assertEquals(1, context.getBuildInfo("device1").getBuildAttributes().size());
1438         assertEquals(
1439                 "50",
1440                 context.getBuildInfo("device1")
1441                         .getBuildAttributes()
1442                         .get("serial1-battery-" + fakeEvent));
1443     }
1444 
1445     /**
1446      * Test that {@link TestInvocation#logDeviceBatteryLevel(IInvocationContext, String)} is adding
1447      * battery information for multiple physical real device.
1448      */
1449     @Test
testLogDeviceBatteryLevel_physicalDevice_multi()1450     public void testLogDeviceBatteryLevel_physicalDevice_multi() {
1451         final String fakeEvent = "event";
1452         IInvocationContext context = new InvocationContext();
1453         ITestDevice device1 = EasyMock.createMock(ITestDevice.class);
1454         EasyMock.expect(device1.getIDevice()).andReturn(EasyMock.createMock(IDevice.class));
1455         EasyMock.expect(device1.getSerialNumber()).andReturn("serial1");
1456         EasyMock.expect(device1.getBattery()).andReturn(50);
1457 
1458         ITestDevice device2 = EasyMock.createMock(ITestDevice.class);
1459         EasyMock.expect(device2.getIDevice()).andReturn(EasyMock.createMock(IDevice.class));
1460         EasyMock.expect(device2.getSerialNumber()).andReturn("serial2");
1461         EasyMock.expect(device2.getBattery()).andReturn(55);
1462 
1463         context.addAllocatedDevice("device1", device1);
1464         context.addDeviceBuildInfo("device1", new BuildInfo());
1465         context.addAllocatedDevice("device2", device2);
1466         context.addDeviceBuildInfo("device2", new BuildInfo());
1467         EasyMock.replay(device1, device2);
1468         mTestInvocation.logDeviceBatteryLevel(context, fakeEvent);
1469         EasyMock.verify(device1, device2);
1470         assertEquals(1, context.getBuildInfo("device1").getBuildAttributes().size());
1471         assertEquals(1, context.getBuildInfo("device2").getBuildAttributes().size());
1472         assertEquals(
1473                 "50",
1474                 context.getBuildInfo("device1")
1475                         .getBuildAttributes()
1476                         .get("serial1-battery-" + fakeEvent));
1477         assertEquals(
1478                 "55",
1479                 context.getBuildInfo("device2")
1480                         .getBuildAttributes()
1481                         .get("serial2-battery-" + fakeEvent));
1482     }
1483 
1484     /**
1485      * Test that {@link TestInvocation#logDeviceBatteryLevel(IInvocationContext, String)} is adding
1486      * battery information for multiple physical real device, and ignore stub device if any.
1487      */
1488     @Test
testLogDeviceBatteryLevel_physicalDevice_stub_multi()1489     public void testLogDeviceBatteryLevel_physicalDevice_stub_multi() {
1490         final String fakeEvent = "event";
1491         IInvocationContext context = new InvocationContext();
1492         ITestDevice device1 = EasyMock.createMock(ITestDevice.class);
1493         EasyMock.expect(device1.getIDevice()).andReturn(EasyMock.createMock(IDevice.class));
1494         EasyMock.expect(device1.getSerialNumber()).andReturn("serial1");
1495         EasyMock.expect(device1.getBattery()).andReturn(50);
1496 
1497         ITestDevice device2 = EasyMock.createMock(ITestDevice.class);
1498         EasyMock.expect(device2.getIDevice()).andReturn(EasyMock.createMock(IDevice.class));
1499         EasyMock.expect(device2.getSerialNumber()).andReturn("serial2");
1500         EasyMock.expect(device2.getBattery()).andReturn(50);
1501 
1502         ITestDevice device3 = EasyMock.createMock(ITestDevice.class);
1503         EasyMock.expect(device3.getIDevice()).andReturn(new StubDevice("stub-3"));
1504 
1505         ITestDevice device4 = EasyMock.createMock(ITestDevice.class);
1506         EasyMock.expect(device4.getIDevice()).andReturn(new TcpDevice("tcp-4"));
1507 
1508         context.addAllocatedDevice("device1", device1);
1509         context.addDeviceBuildInfo("device1", new BuildInfo());
1510         context.addAllocatedDevice("device2", device2);
1511         context.addDeviceBuildInfo("device2", new BuildInfo());
1512         context.addAllocatedDevice("device3", device3);
1513         context.addAllocatedDevice("device4", device4);
1514         EasyMock.replay(device1, device2, device3, device4);
1515         mTestInvocation.logDeviceBatteryLevel(context, fakeEvent);
1516         EasyMock.verify(device1, device2, device3, device4);
1517         assertEquals(1, context.getBuildInfo("device1").getBuildAttributes().size());
1518         assertEquals(1, context.getBuildInfo("device2").getBuildAttributes().size());
1519         assertEquals(
1520                 "50",
1521                 context.getBuildInfo("device1")
1522                         .getBuildAttributes()
1523                         .get("serial1-battery-" + fakeEvent));
1524         assertEquals(
1525                 "50",
1526                 context.getBuildInfo("device2")
1527                         .getBuildAttributes()
1528                         .get("serial2-battery-" + fakeEvent));
1529     }
1530 
1531     /** Helper to set the expectation for N number of shards. */
setupNShardInvocation(int numShard, String commandLine)1532     private void setupNShardInvocation(int numShard, String commandLine) throws Exception {
1533         mMockBuildInfo.setTestTag(EasyMock.eq("stub"));
1534         EasyMock.expectLastCall();
1535         EasyMock.expect(mMockBuildProvider.getBuild()).andReturn(mMockBuildInfo);
1536         EasyMock.expect(mMockBuildInfo.getTestTag()).andStubReturn("");
1537         mMockBuildInfo.addBuildAttribute("command_line_args", commandLine);
1538         mMockTestListener.invocationStarted((IInvocationContext)EasyMock.anyObject());
1539         EasyMock.expectLastCall();
1540         mMockSummaryListener.invocationStarted((IInvocationContext)EasyMock.anyObject());
1541         EasyMock.expectLastCall();
1542         EasyMock.expect(mMockBuildInfo.clone()).andReturn(mMockBuildInfo).times(numShard);
1543         EasyMock.expect(mMockBuildInfo.getVersionedFileKeys())
1544                 .andReturn(new HashSet<>())
1545                 .times(numShard);
1546         EasyMock.expect(mockRescheduler.scheduleConfig(EasyMock.anyObject()))
1547                 .andReturn(true).times(numShard);
1548         mMockBuildInfo.setDeviceSerial(SERIAL);
1549         EasyMock.expectLastCall();
1550         mMockBuildProvider.cleanUp(EasyMock.anyObject());
1551         EasyMock.expectLastCall();
1552     }
1553 
setupMockSuccessListeners()1554     private void setupMockSuccessListeners() throws IOException {
1555         setupMockListeners(InvocationStatus.SUCCESS, null, false, true, false);
1556     }
1557 
setupMockFailureListeners(Throwable throwable)1558     private void setupMockFailureListeners(Throwable throwable) throws IOException {
1559         setupMockListeners(InvocationStatus.FAILED, throwable, false, true, false);
1560     }
1561 
setupMockFailureListenersAny(Throwable throwable, boolean stubFailures)1562     private void setupMockFailureListenersAny(Throwable throwable, boolean stubFailures)
1563             throws IOException {
1564         setupMockListeners(InvocationStatus.FAILED, throwable, stubFailures, true, false);
1565     }
1566 
setupMockFailureListeners( Throwable throwable, boolean stubFailures, boolean reportHostLog)1567     private void setupMockFailureListeners(
1568             Throwable throwable, boolean stubFailures, boolean reportHostLog) throws IOException {
1569         setupMockListeners(InvocationStatus.FAILED, throwable, stubFailures, reportHostLog, false);
1570     }
1571 
setupMockStoppedListeners()1572     private void setupMockStoppedListeners() throws IOException {
1573         setupMockListeners(InvocationStatus.SUCCESS, null, false, true, true);
1574     }
1575 
verifySummaryListener()1576     private void verifySummaryListener() {
1577         // Check that we captured the expected uris List
1578         List<TestSummary> summaries = mUriCapture.getValue();
1579         assertEquals(1, summaries.size());
1580         assertEquals(mSummary, summaries.get(0));
1581     }
1582 
1583     /**
1584      * Verify all mock objects received expected calls
1585      */
verifyMocks(Object... mocks)1586     private void verifyMocks(Object... mocks) {
1587         // note: intentionally exclude configuration from verification - don't care
1588         // what methods are called
1589         EasyMock.verify(mMockTestListener, mMockSummaryListener, mMockPreparer,
1590                 mMockBuildProvider, mMockLogger, mMockBuildInfo, mMockLogRegistry,
1591                 mMockLogSaver);
1592         if (mocks.length > 0) {
1593             EasyMock.verify(mocks);
1594         }
1595     }
1596 
1597     /**
1598      * Switch all mock objects into replay mode.
1599      */
replayMocks(Object... mocks)1600     private void replayMocks(Object... mocks) {
1601         EasyMock.replay(mMockTestListener, mMockSummaryListener, mMockPreparer,
1602                 mMockBuildProvider, mMockLogger, mMockBuildInfo, mMockLogRegistry,
1603                 mMockLogSaver, mMockDevice, mMockConfigFactory);
1604         if (mocks.length > 0) {
1605             EasyMock.replay(mocks);
1606         }
1607     }
1608 
1609     /**
1610      * Interface for testing device config pass through.
1611      */
1612     private interface DeviceConfigTest extends IRemoteTest, IDeviceTest {
1613 
1614     }
1615 
1616     /**
1617      * Test when a {@link IDeviceBuildInfo} is passing through we do not attempt to add any external
1618      * directories when there is none coming from environment.
1619      */
1620     @Test
testInvoke_deviceInfoBuild_noEnv()1621     public void testInvoke_deviceInfoBuild_noEnv() throws Throwable {
1622         mTestInvocation =
1623                 new TestInvocation() {
1624                     @Override
1625                     ILogRegistry getLogRegistry() {
1626                         return mMockLogRegistry;
1627                     }
1628 
1629                     @Override
1630                     public IInvocationExecution createInvocationExec(RunMode mode) {
1631                         return new InvocationExecution() {
1632                             @Override
1633                             protected IShardHelper createShardHelper() {
1634                                 return new ShardHelper();
1635                             }
1636 
1637                             @Override
1638                             File getExternalTestCasesDirs(EnvVariable envVar) {
1639                                 // Return empty list to ensure we do not have any environment loaded
1640                                 return null;
1641                             }
1642 
1643                             @Override
1644                             protected String getAdbVersion() {
1645                                 return null;
1646                             }
1647 
1648                             @Override
1649                             void logHostAdb(ITestLogger logger) {
1650                                 // inop for the common test case.
1651                             }
1652                         };
1653                     }
1654 
1655                     @Override
1656                     protected void setExitCode(ExitCode code, Throwable stack) {
1657                         // empty on purpose
1658                     }
1659 
1660                     @Override
1661                     InvocationScope getInvocationScope() {
1662                         // Avoid re-entry in the current TF invocation scope for unit tests.
1663                         return new InvocationScope();
1664                     }
1665 
1666                     @Override
1667                     protected void applyAutomatedReporters(IConfiguration config) {
1668                         // Empty on purpose
1669                     }
1670 
1671                     @Override
1672                     protected void addInvocationMetric(InvocationMetricKey key, long value) {}
1673 
1674                     @Override
1675                     protected void addInvocationMetric(InvocationMetricKey key, String value) {}
1676                 };
1677         mMockBuildInfo = EasyMock.createMock(IDeviceBuildInfo.class);
1678         EasyMock.expect(mMockBuildInfo.getProperties()).andStubReturn(new HashSet<>());
1679 
1680         IRemoteTest test = EasyMock.createNiceMock(IRemoteTest.class);
1681         mMockPreparer.tearDown(EasyMock.anyObject(), EasyMock.isNull());
1682 
1683         File tmpTestsDir = FileUtil.createTempDir("invocation-tests-dir");
1684         try {
1685             EasyMock.expect(((IDeviceBuildInfo) mMockBuildInfo).getTestsDir())
1686                     .andReturn(tmpTestsDir);
1687             setupMockSuccessListeners();
1688             setEarlyDeviceReleaseExpectation();
1689             setupNormalInvoke(test);
1690             EasyMock.replay(mockRescheduler);
1691             mTestInvocation.invoke(mStubInvocationMetadata, mStubConfiguration, mockRescheduler);
1692             verifyMocks(mockRescheduler);
1693             verifySummaryListener();
1694         } finally {
1695             FileUtil.recursiveDelete(tmpTestsDir);
1696         }
1697     }
1698 
1699     /**
1700      * Test when a {@link IDeviceBuildInfo} is passing through we attempt to add the external
1701      * directories to it when they are available.
1702      */
1703     @Test
testInvoke_deviceInfoBuild_withEnv()1704     public void testInvoke_deviceInfoBuild_withEnv() throws Throwable {
1705         File tmpTestsDir = FileUtil.createTempDir("invocation-tests-dir");
1706         File tmpExternalTestsDir = FileUtil.createTempDir("external-tf-dir");
1707         File tmpTestsFile = FileUtil.createTempFile("testsfile", "txt", tmpExternalTestsDir);
1708         try {
1709             mTestInvocation =
1710                     new TestInvocation() {
1711                         @Override
1712                         ILogRegistry getLogRegistry() {
1713                             return mMockLogRegistry;
1714                         }
1715 
1716                         @Override
1717                         public IInvocationExecution createInvocationExec(RunMode mode) {
1718                             return new InvocationExecution() {
1719                                 @Override
1720                                 protected IShardHelper createShardHelper() {
1721                                     return new ShardHelper();
1722                                 }
1723 
1724                                 @Override
1725                                 File getExternalTestCasesDirs(EnvVariable envVar) {
1726                                     if (EnvVariable.ANDROID_TARGET_OUT_TESTCASES.equals(envVar)) {
1727                                         return tmpExternalTestsDir;
1728                                     }
1729                                     return null;
1730                                 }
1731 
1732                                 @Override
1733                                 protected String getAdbVersion() {
1734                                     return null;
1735                                 }
1736 
1737                                 @Override
1738                                 void logHostAdb(ITestLogger logger) {
1739                                     // inop for the common test case.
1740                                 }
1741                             };
1742                         }
1743 
1744                         @Override
1745                         protected void applyAutomatedReporters(IConfiguration config) {
1746                             // Empty on purpose
1747                         }
1748 
1749                         @Override
1750                         protected void setExitCode(ExitCode code, Throwable stack) {
1751                             // empty on purpose
1752                         }
1753 
1754                         @Override
1755                         InvocationScope getInvocationScope() {
1756                             // Avoid re-entry in the current TF invocation scope for unit tests.
1757                             return new InvocationScope();
1758                         }
1759 
1760                         @Override
1761                         protected void addInvocationMetric(InvocationMetricKey key, long value) {}
1762 
1763                         @Override
1764                         protected void addInvocationMetric(InvocationMetricKey key, String value) {}
1765                     };
1766             mMockBuildInfo = EasyMock.createMock(IDeviceBuildInfo.class);
1767             IRemoteTest test = EasyMock.createNiceMock(IRemoteTest.class);
1768             ITargetPreparer mockCleaner = EasyMock.createMock(ITargetPreparer.class);
1769             EasyMock.expect(mockCleaner.isDisabled()).andReturn(false).times(2);
1770             EasyMock.expect(mockCleaner.isTearDownDisabled()).andReturn(false);
1771             mockCleaner.setUp(EasyMock.anyObject());
1772             mockCleaner.tearDown(EasyMock.anyObject(), EasyMock.isNull());
1773             mMockPreparer.tearDown(EasyMock.anyObject(), EasyMock.isNull());
1774             mStubConfiguration.getTargetPreparers().add(mockCleaner);
1775 
1776             mMockBuildInfo.setFile(
1777                     EasyMock.contains(BuildInfoFileKey.TARGET_LINKED_DIR.getFileKey()),
1778                     EasyMock.anyObject(),
1779                     EasyMock.eq("v1"));
1780             EasyMock.expect(((IDeviceBuildInfo) mMockBuildInfo).getTestsDir())
1781                     .andReturn(tmpTestsDir);
1782             EasyMock.expect(mMockBuildInfo.getProperties()).andStubReturn(new HashSet<>());
1783 
1784             setupMockSuccessListeners();
1785             setEarlyDeviceReleaseExpectation();
1786             setupNormalInvoke(test);
1787             EasyMock.replay(mockCleaner, mockRescheduler);
1788             mTestInvocation.invoke(mStubInvocationMetadata, mStubConfiguration, mockRescheduler);
1789             verifyMocks(mockCleaner, mockRescheduler);
1790             verifySummaryListener();
1791             // Check that the external directory was copied in the testsDir.
1792             assertTrue(tmpTestsDir.listFiles().length == 1);
1793             // external-tf-dir - the symlink is the original file name + randomized sequence
1794             assertTrue(
1795                     tmpTestsDir
1796                             .listFiles()[0]
1797                             .getName()
1798                             .startsWith(BuildInfoFileKey.TARGET_LINKED_DIR.getFileKey()));
1799             // testsfile.txt
1800             assertTrue(tmpTestsDir.listFiles()[0].listFiles().length == 1);
1801             assertEquals(
1802                     tmpTestsFile.getName(), tmpTestsDir.listFiles()[0].listFiles()[0].getName());
1803         } finally {
1804             FileUtil.recursiveDelete(tmpTestsDir);
1805             FileUtil.recursiveDelete(tmpExternalTestsDir);
1806         }
1807     }
1808 
1809     /**
1810      * Test when a {@link IDeviceBuildInfo} is passing through we do not attempt to add the external
1811      * directories to it, since {@link BuildInfoProperties} is set to skip the linking.
1812      */
1813     @Test
testInvoke_deviceInfoBuild_withEnv_andSkipProperty()1814     public void testInvoke_deviceInfoBuild_withEnv_andSkipProperty() throws Throwable {
1815         File tmpTestsDir = FileUtil.createTempDir("invocation-tests-dir");
1816         File tmpExternalTestsDir = FileUtil.createTempDir("external-tf-dir");
1817         FileUtil.createTempFile("testsfile", "txt", tmpExternalTestsDir);
1818         try {
1819             mTestInvocation =
1820                     new TestInvocation() {
1821                         @Override
1822                         ILogRegistry getLogRegistry() {
1823                             return mMockLogRegistry;
1824                         }
1825 
1826                         @Override
1827                         public IInvocationExecution createInvocationExec(RunMode mode) {
1828                             return new InvocationExecution() {
1829                                 @Override
1830                                 protected IShardHelper createShardHelper() {
1831                                     return new ShardHelper();
1832                                 }
1833 
1834                                 @Override
1835                                 File getExternalTestCasesDirs(EnvVariable envVar) {
1836                                     return tmpExternalTestsDir;
1837                                 }
1838 
1839                                 @Override
1840                                 protected String getAdbVersion() {
1841                                     return null;
1842                                 }
1843 
1844                                 @Override
1845                                 void logHostAdb(ITestLogger logger) {
1846                                     // inop for the common test case.
1847                                 }
1848                             };
1849                         }
1850 
1851                         @Override
1852                         protected void setExitCode(ExitCode code, Throwable stack) {
1853                             // empty on purpose
1854                         }
1855 
1856                         @Override
1857                         protected void applyAutomatedReporters(IConfiguration config) {
1858                             // Empty on purpose
1859                         }
1860 
1861                         @Override
1862                         InvocationScope getInvocationScope() {
1863                             // Avoid re-entry in the current TF invocation scope for unit tests.
1864                             return new InvocationScope();
1865                         }
1866 
1867                         @Override
1868                         protected void addInvocationMetric(InvocationMetricKey key, long value) {}
1869 
1870                         @Override
1871                         protected void addInvocationMetric(InvocationMetricKey key, String value) {}
1872                     };
1873             mMockBuildInfo = EasyMock.createMock(IDeviceBuildInfo.class);
1874             IRemoteTest test = EasyMock.createNiceMock(IRemoteTest.class);
1875             mMockPreparer.tearDown(EasyMock.anyObject(), EasyMock.isNull());
1876 
1877             Set<BuildInfoProperties> prop = new HashSet<>();
1878             prop.add(BuildInfoProperties.DO_NOT_LINK_TESTS_DIR);
1879             EasyMock.expect(mMockBuildInfo.getProperties()).andStubReturn(prop);
1880             setEarlyDeviceReleaseExpectation();
1881             setupMockSuccessListeners();
1882             setupNormalInvoke(test);
1883             EasyMock.replay(mockRescheduler);
1884             mTestInvocation.invoke(mStubInvocationMetadata, mStubConfiguration, mockRescheduler);
1885             verifyMocks(mockRescheduler);
1886             verifySummaryListener();
1887             // Check that the external directory was NOT copied in the testsDir.
1888             assertTrue(tmpTestsDir.listFiles().length == 0);
1889         } finally {
1890             FileUtil.recursiveDelete(tmpTestsDir);
1891             FileUtil.recursiveDelete(tmpExternalTestsDir);
1892         }
1893     }
1894 
1895     public static class TestableCollector extends BaseDeviceMetricCollector {
1896 
1897         @Option(name = "name")
1898         private String mName;
1899 
TestableCollector()1900         public TestableCollector() {}
1901 
TestableCollector(String name)1902         public TestableCollector(String name) {
1903             mName = name;
1904         }
1905 
1906         @Override
onTestRunEnd( DeviceMetricData runData, final Map<String, Metric> currentRunMetrics)1907         public void onTestRunEnd(
1908                 DeviceMetricData runData, final Map<String, Metric> currentRunMetrics) {
1909             runData.addMetric(
1910                     mName,
1911                     Metric.newBuilder()
1912                             .setMeasurements(
1913                                     Measurements.newBuilder().setSingleString(mName).build()));
1914         }
1915     }
1916 
1917     /**
1918      * Test that when {@link IMetricCollector} are used, they wrap and call in sequence the listener
1919      * so all metrics end up on the final receiver.
1920      */
1921     @Test
testMetricCollectionChain()1922     public void testMetricCollectionChain() throws Throwable {
1923         TestInformation info =
1924                 TestInformation.newBuilder().setInvocationContext(mStubInvocationMetadata).build();
1925         IConfiguration configuration = new Configuration("test", "description");
1926         StubTest test = new StubTest();
1927         OptionSetter setter = new OptionSetter(test);
1928         setter.setOptionValue("run-a-test", "true");
1929         configuration.setTest(test);
1930 
1931         List<IMetricCollector> collectors = new ArrayList<>();
1932         collectors.add(new TestableCollector("collector1"));
1933         collectors.add(new TestableCollector("collector2"));
1934         collectors.add(new TestableCollector("collector3"));
1935         collectors.add(new TestableCollector("collector4"));
1936         configuration.setDeviceMetricCollectors(collectors);
1937 
1938         mMockTestListener.testRunStarted(
1939                 EasyMock.eq("TestStub"), EasyMock.eq(1), EasyMock.eq(0), EasyMock.anyLong());
1940         TestDescription testId = new TestDescription("StubTest", "StubMethod");
1941         mMockTestListener.testStarted(EasyMock.eq(testId), EasyMock.anyLong());
1942         mMockTestListener.testEnded(
1943                 EasyMock.eq(testId),
1944                 EasyMock.anyLong(),
1945                 EasyMock.eq(new HashMap<String, Metric>()));
1946         Capture<HashMap<String, Metric>> captured = new Capture<>();
1947         mMockTestListener.testRunEnded(EasyMock.anyLong(), EasyMock.capture(captured));
1948         EasyMock.replay(mMockTestListener);
1949         new InvocationExecution().runTests(info, configuration, mMockTestListener);
1950         EasyMock.verify(mMockTestListener);
1951         // The collectors are called in sequence
1952         List<String> listKeys = new ArrayList<>(captured.getValue().keySet());
1953         assertEquals(4, listKeys.size());
1954         assertEquals("collector4", listKeys.get(0));
1955         assertEquals("collector3", listKeys.get(1));
1956         assertEquals("collector2", listKeys.get(2));
1957         assertEquals("collector1", listKeys.get(3));
1958     }
1959 
1960     /**
1961      * Test that when a device collector is disabled, it will not be part of the initialization and
1962      * the collector chain.
1963      */
1964     @Test
testMetricCollectionChain_disabled()1965     public void testMetricCollectionChain_disabled() throws Throwable {
1966         TestInformation info =
1967                 TestInformation.newBuilder().setInvocationContext(mStubInvocationMetadata).build();
1968         IConfiguration configuration = new Configuration("test", "description");
1969         StubTest test = new StubTest();
1970         OptionSetter setter = new OptionSetter(test);
1971         setter.setOptionValue("run-a-test", "true");
1972         configuration.setTest(test);
1973 
1974         List<IMetricCollector> collectors = new ArrayList<>();
1975         collectors.add(new TestableCollector("collector1"));
1976         collectors.add(new TestableCollector("collector2"));
1977         // Collector 3 is disabled
1978         TestableCollector col3 = new TestableCollector("collector3");
1979         col3.setDisable(true);
1980         collectors.add(col3);
1981         collectors.add(new TestableCollector("collector4"));
1982         configuration.setDeviceMetricCollectors(collectors);
1983 
1984         mMockTestListener.testRunStarted(
1985                 EasyMock.eq("TestStub"), EasyMock.eq(1), EasyMock.eq(0), EasyMock.anyLong());
1986         TestDescription testId = new TestDescription("StubTest", "StubMethod");
1987         mMockTestListener.testStarted(EasyMock.eq(testId), EasyMock.anyLong());
1988         mMockTestListener.testEnded(
1989                 EasyMock.eq(testId),
1990                 EasyMock.anyLong(),
1991                 EasyMock.eq(new HashMap<String, Metric>()));
1992         Capture<HashMap<String, Metric>> captured = new Capture<>();
1993         mMockTestListener.testRunEnded(EasyMock.anyLong(), EasyMock.capture(captured));
1994         EasyMock.replay(mMockTestListener);
1995         new InvocationExecution().runTests(info, configuration, mMockTestListener);
1996         EasyMock.verify(mMockTestListener);
1997         // The collectors are called in sequence
1998         List<String> listKeys = new ArrayList<>(captured.getValue().keySet());
1999         assertEquals(3, listKeys.size());
2000         assertEquals("collector4", listKeys.get(0));
2001         assertEquals("collector2", listKeys.get(1));
2002         assertEquals("collector1", listKeys.get(2));
2003     }
2004 
2005     public static class TestableProcessor extends BasePostProcessor {
2006 
2007         @Option(name = "name")
2008         private String mName;
2009 
TestableProcessor()2010         public TestableProcessor() {}
2011 
TestableProcessor(String name)2012         public TestableProcessor(String name) {
2013             mName = name;
2014         }
2015 
2016         @Override
processRunMetricsAndLogs( HashMap<String, Metric> rawMetrics, Map<String, LogFile> runLogs)2017         public Map<String, Builder> processRunMetricsAndLogs(
2018                 HashMap<String, Metric> rawMetrics, Map<String, LogFile> runLogs) {
2019             Map<String, Builder> post = new LinkedHashMap<>();
2020             post.put(mName, Metric.newBuilder());
2021             return post;
2022         }
2023     }
2024 
2025     /** Ensure post processors are called in order. */
2026     @Test
testProcessorCollectionChain()2027     public void testProcessorCollectionChain() throws Throwable {
2028         mMockTestListener = EasyMock.createMock(ITestInvocationListener.class);
2029         List<ITestInvocationListener> listenerList = new ArrayList<ITestInvocationListener>(1);
2030         listenerList.add(mMockTestListener);
2031         mStubConfiguration.setTestInvocationListeners(listenerList);
2032 
2033         List<IPostProcessor> processors = new ArrayList<>();
2034         processors.add(new TestableProcessor("processor1"));
2035         processors.add(new TestableProcessor("processor2"));
2036         processors.add(new TestableProcessor("processor3"));
2037         processors.add(new TestableProcessor("processor4"));
2038         mStubConfiguration.setPostProcessors(processors);
2039 
2040         mMockTestListener.testRunStarted("TestStub", 1);
2041         TestDescription testId = new TestDescription("StubTest", "StubMethod");
2042         mMockTestListener.testStarted(EasyMock.eq(testId), EasyMock.anyLong());
2043         mMockTestListener.testEnded(
2044                 EasyMock.eq(testId),
2045                 EasyMock.anyLong(),
2046                 EasyMock.eq(new HashMap<String, Metric>()));
2047         Capture<HashMap<String, Metric>> captured = new Capture<>();
2048         mMockTestListener.testRunEnded(EasyMock.anyLong(), EasyMock.capture(captured));
2049 
2050         setupMockSuccessListeners();
2051         setEarlyDeviceReleaseExpectation();
2052         setupInvokeWithBuild();
2053         StubTest test = new StubTest();
2054         OptionSetter setter = new OptionSetter(test);
2055         setter.setOptionValue("run-a-test", "true");
2056         mStubConfiguration.setTest(test);
2057         EasyMock.expect(mMockBuildProvider.getBuild()).andReturn(mMockBuildInfo);
2058 
2059         mMockPreparer.setUp(EasyMock.anyObject());
2060         mMockPreparer.tearDown(EasyMock.anyObject(), EasyMock.isNull());
2061 
2062         EasyMock.reset(mMockSummaryListener);
2063         replayMocks();
2064         EasyMock.replay(mockRescheduler);
2065         mTestInvocation.invoke(mStubInvocationMetadata, mStubConfiguration, mockRescheduler);
2066         verifyMocks(mockRescheduler);
2067 
2068         // The post processors are called in sequence
2069         List<String> listKeys = new ArrayList<>(captured.getValue().keySet());
2070         assertEquals(4, listKeys.size());
2071         assertEquals("processor4", listKeys.get(0));
2072         assertEquals("processor3", listKeys.get(1));
2073         assertEquals("processor2", listKeys.get(2));
2074         assertEquals("processor1", listKeys.get(3));
2075     }
2076 
2077     /** Ensure post processors are called in order and that ILogSaver is set properly */
2078     @Test
testProcessorCollectionChain_logSaver()2079     public void testProcessorCollectionChain_logSaver() throws Throwable {
2080         TestLogSaverListener mockLogSaverListener = new TestLogSaverListener();
2081         mMockTestListener = EasyMock.createMock(ITestInvocationListener.class);
2082         List<ITestInvocationListener> listenerList = new ArrayList<ITestInvocationListener>(1);
2083         listenerList.add(mMockTestListener);
2084         listenerList.add(mockLogSaverListener);
2085         mStubConfiguration.setTestInvocationListeners(listenerList);
2086 
2087         List<IPostProcessor> processors = new ArrayList<>();
2088         processors.add(new TestableProcessor("processor1"));
2089         processors.add(new TestableProcessor("processor2"));
2090         processors.add(new TestableProcessor("processor3"));
2091         processors.add(new TestableProcessor("processor4"));
2092         mStubConfiguration.setPostProcessors(processors);
2093 
2094         mMockTestListener.testRunStarted("TestStub", 1);
2095         TestDescription testId = new TestDescription("StubTest", "StubMethod");
2096         mMockTestListener.testStarted(EasyMock.eq(testId), EasyMock.anyLong());
2097         mMockTestListener.testEnded(
2098                 EasyMock.eq(testId),
2099                 EasyMock.anyLong(),
2100                 EasyMock.eq(new HashMap<String, Metric>()));
2101         Capture<HashMap<String, Metric>> captured = new Capture<>();
2102         mMockTestListener.testRunEnded(EasyMock.anyLong(), EasyMock.capture(captured));
2103 
2104         setupMockSuccessListeners();
2105         setEarlyDeviceReleaseExpectation();
2106         setupInvokeWithBuild();
2107         StubTest test = new StubTest();
2108         OptionSetter setter = new OptionSetter(test);
2109         setter.setOptionValue("run-a-test", "true");
2110         mStubConfiguration.setTest(test);
2111         EasyMock.expect(mMockBuildProvider.getBuild()).andReturn(mMockBuildInfo);
2112 
2113         mMockPreparer.setUp(EasyMock.anyObject());
2114         mMockPreparer.tearDown(EasyMock.anyObject(), EasyMock.isNull());
2115 
2116         EasyMock.reset(mMockSummaryListener);
2117         replayMocks();
2118         EasyMock.replay(mockRescheduler);
2119         mTestInvocation.invoke(mStubInvocationMetadata, mStubConfiguration, mockRescheduler);
2120         verifyMocks(mockRescheduler);
2121 
2122         // The post processors are called in sequence
2123         List<String> listKeys = new ArrayList<>(captured.getValue().keySet());
2124         assertEquals(4, listKeys.size());
2125         assertEquals("processor4", listKeys.get(0));
2126         assertEquals("processor3", listKeys.get(1));
2127         assertEquals("processor2", listKeys.get(2));
2128         assertEquals("processor1", listKeys.get(3));
2129 
2130         assertTrue(mockLogSaverListener.mWasLoggerSet);
2131     }
2132 
2133     private class TestLogSaverListener implements ILogSaverListener {
2134 
2135         public boolean mWasLoggerSet = false;
2136 
2137         @Override
setLogSaver(ILogSaver logSaver)2138         public void setLogSaver(ILogSaver logSaver) {
2139             mWasLoggerSet = true;
2140         }
2141     }
2142 
setEarlyDeviceReleaseExpectation()2143     private void setEarlyDeviceReleaseExpectation() {
2144         EasyMock.expect(mMockDevice.getDeviceState()).andReturn(TestDeviceState.ONLINE);
2145         EasyMock.expect(mMockDevice.waitForDeviceShell(30000L)).andReturn(true);
2146         mMockDevice.setRecoveryMode(RecoveryMode.AVAILABLE);
2147     }
2148 }
2149