1 /*
2  * Copyright (C) 2018 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 package com.android.tradefed.result;
17 
18 import com.android.tradefed.metrics.proto.MetricMeasurement.Metric;
19 import com.android.tradefed.util.proto.TfMetricProtoUtil;
20 
21 import java.util.HashMap;
22 import java.util.Map;
23 
24 /**
25  * Receives event notifications during instrumentation test runs.
26  *
27  * <p>Patterned after org.junit.runner.notification.RunListener
28  *
29  * <p>The sequence of calls will be:
30  *
31  * <ul>
32  *   <li> testRunStarted
33  *   <li> testStarted
34  *   <li> [testFailed]
35  *   <li> [testAssumptionFailure]
36  *   <li> [testIgnored]
37  *   <li> testEnded
38  *   <li> ....
39  *   <li> [testRunFailed]
40  *   <li> testRunEnded
41  * </ul>
42  */
43 public interface ITestLifeCycleReceiver {
44 
45     /**
46      * Reports the start of a test run.
47      *
48      * @param runName the test run name
49      * @param testCount total number of tests in test run
50      */
testRunStarted(String runName, int testCount)51     public default void testRunStarted(String runName, int testCount) {}
52 
53     /**
54      * Reports the start of a test run.
55      *
56      * @param runName the test run name
57      * @param testCount total number of tests in test run
58      * @param attemptNumber order number, identifying the different attempts of the same runName
59      *     that run multiple times. The attemptNumber is 0-indexed and should increment everytime
60      *     a new run happens. e.g. A test is granular retried 3 times, it should have 4 total
61      *     runs under the same runName and the attemptNumber is from 0 to 3.
62      */
testRunStarted(String runName, int testCount, int attemptNumber)63     public default void testRunStarted(String runName, int testCount, int attemptNumber) {
64         testRunStarted(runName, testCount);
65     }
66 
67     /**
68      * Reports the start of a test run.
69      *
70      * @param runName the test run name
71      * @param testCount total number of tests in test run
72      * @param attemptNumber order number, identifying the different attempts of the same runName
73      *     that run multiple times. The attemptNumber is 0-indexed and should increment everytime a
74      *     new run happens. e.g. A test is granular retried 3 times, it should have 4 total runs
75      *     under the same runName and the attemptNumber is from 0 to 3.
76      * @param startTime the time the run started, measured via {@link System#currentTimeMillis()}
77      */
testRunStarted( String runName, int testCount, int attemptNumber, long startTime)78     public default void testRunStarted(
79             String runName, int testCount, int attemptNumber, long startTime) {
80         testRunStarted(runName, testCount);
81     }
82 
83     /**
84      * Reports test run failed to complete due to a fatal error.
85      *
86      * @param errorMessage {@link String} describing reason for run failure.
87      */
testRunFailed(String errorMessage)88     public default void testRunFailed(String errorMessage) {}
89 
90     /**
91      * Reports test run failed to complete due to a failure described by {@link FailureDescription}.
92      *
93      * @param failure {@link FailureDescription} describing the failure and its context.
94      */
testRunFailed(FailureDescription failure)95     public default void testRunFailed(FailureDescription failure) {
96         testRunFailed(failure.toString());
97     }
98 
99     /**
100      * Reports end of test run.
101      *
102      * @param elapsedTimeMillis device reported elapsed time, in milliseconds
103      * @param runMetrics key-value pairs reported at the end of a test run
104      */
testRunEnded(long elapsedTimeMillis, Map<String, String> runMetrics)105     public default void testRunEnded(long elapsedTimeMillis, Map<String, String> runMetrics) {
106         testRunEnded(elapsedTimeMillis, TfMetricProtoUtil.upgradeConvert(runMetrics));
107     }
108 
109     /**
110      * Reports end of test run. FIXME: We cannot have two Map<> interfaces with different type, so
111      * we have to use HashMap here.
112      *
113      * @param elapsedTimeMillis device reported elapsed time, in milliseconds
114      * @param runMetrics key-value pairs reported at the end of a test run with {@link Metric}.
115      */
testRunEnded(long elapsedTimeMillis, HashMap<String, Metric> runMetrics)116     public default void testRunEnded(long elapsedTimeMillis, HashMap<String, Metric> runMetrics) {}
117 
118     /**
119      * Reports test run stopped before completion due to a user request.
120      *
121      * <p>TODO: currently unused, consider removing
122      *
123      * @param elapsedTime device reported elapsed time, in milliseconds
124      */
testRunStopped(long elapsedTime)125     public default void testRunStopped(long elapsedTime) {}
126 
127     /**
128      * Reports the start of an individual test case. Older interface, should use {@link
129      * #testStarted(TestDescription)} whenever possible.
130      *
131      * @param test identifies the test
132      */
testStarted(TestDescription test)133     public default void testStarted(TestDescription test) {}
134 
135     /**
136      * Alternative to {@link #testStarted(TestDescription)} where we also specify when the test was
137      * started, combined with {@link #testEnded(TestDescription, long, Map)} for accurate measure.
138      *
139      * @param test identifies the test
140      * @param startTime the time the test started, measured via {@link System#currentTimeMillis()}
141      */
testStarted(TestDescription test, long startTime)142     default void testStarted(TestDescription test, long startTime) {
143         testStarted(test);
144     }
145 
146     /**
147      * Reports the failure of a individual test case.
148      *
149      * <p>Will be called between testStarted and testEnded.
150      *
151      * @param test identifies the test
152      * @param trace stack trace of failure
153      */
testFailed(TestDescription test, String trace)154     public default void testFailed(TestDescription test, String trace) {}
155 
156     /**
157      * Reports the failure of a individual test case.
158      *
159      * <p>Will be called between testStarted and testEnded.
160      *
161      * @param test identifies the test
162      * @param failure {@link FailureDescription} describing the failure and its context.
163      */
testFailed(TestDescription test, FailureDescription failure)164     public default void testFailed(TestDescription test, FailureDescription failure) {
165         testFailed(test, failure.toString());
166     }
167 
168     /**
169      * Called when an atomic test flags that it assumes a condition that is false
170      *
171      * @param test identifies the test
172      * @param trace stack trace of failure
173      */
testAssumptionFailure(TestDescription test, String trace)174     public default void testAssumptionFailure(TestDescription test, String trace) {}
175 
176     /**
177      * Called when an atomic test flags that it assumes a condition that is false
178      *
179      * @param test identifies the test
180      * @param failure {@link FailureDescription} describing the failure and its context.
181      */
testAssumptionFailure(TestDescription test, FailureDescription failure)182     public default void testAssumptionFailure(TestDescription test, FailureDescription failure) {
183         testAssumptionFailure(test, failure.toString());
184     }
185 
186     /**
187      * Called when a test will not be run, generally because a test method is annotated with
188      * org.junit.Ignore.
189      *
190      * @param test identifies the test
191      */
testIgnored(TestDescription test)192     public default void testIgnored(TestDescription test) {}
193 
194     /**
195      * Reports the execution end of an individual test case.
196      *
197      * <p>If {@link #testFailed} was not invoked, this test passed. Also returns any key/value
198      * metrics which may have been emitted during the test case's execution.
199      *
200      * @param test identifies the test
201      * @param testMetrics a {@link Map} of the metrics emitted
202      */
testEnded(TestDescription test, Map<String, String> testMetrics)203     public default void testEnded(TestDescription test, Map<String, String> testMetrics) {
204         testEnded(test, TfMetricProtoUtil.upgradeConvert(testMetrics));
205     }
206 
207     /**
208      * Reports the execution end of an individual test case.
209      *
210      * <p>If {@link #testFailed} was not invoked, this test passed. Also returns any key/value
211      * metrics which may have been emitted during the test case's execution.
212      *
213      * @param test identifies the test
214      * @param testMetrics a {@link Map} of the metrics emitted
215      */
testEnded(TestDescription test, HashMap<String, Metric> testMetrics)216     public default void testEnded(TestDescription test, HashMap<String, Metric> testMetrics) {}
217 
218     /**
219      * Alternative to {@link #testEnded(TestDescription, Map)} where we can specify the end time
220      * directly. Combine with {@link #testStarted(TestDescription, long)} for accurate measure.
221      *
222      * @param test identifies the test
223      * @param endTime the time the test ended, measured via {@link System#currentTimeMillis()}
224      * @param testMetrics a {@link Map} of the metrics emitted
225      */
testEnded( TestDescription test, long endTime, Map<String, String> testMetrics)226     public default void testEnded(
227             TestDescription test, long endTime, Map<String, String> testMetrics) {
228         testEnded(test, endTime, TfMetricProtoUtil.upgradeConvert(testMetrics));
229     }
230 
231     /**
232      * Alternative to {@link #testEnded(TestDescription, Map)} where we can specify the end time
233      * directly. Combine with {@link #testStarted(TestDescription, long)} for accurate measure.
234      *
235      * @param test identifies the test
236      * @param endTime the time the test ended, measured via {@link System#currentTimeMillis()}
237      * @param testMetrics a {@link Map} of the metrics emitted
238      */
testEnded( TestDescription test, long endTime, HashMap<String, Metric> testMetrics)239     public default void testEnded(
240             TestDescription test, long endTime, HashMap<String, Metric> testMetrics) {
241         testEnded(test, testMetrics);
242     }
243 }
244