1 /*
2  * Copyright (C) 2018 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 package com.android.regression.tests;
17 
18 import com.android.tradefed.log.LogUtil.CLog;
19 import com.android.tradefed.result.TestDescription;
20 import com.android.tradefed.util.MultiMap;
21 import com.android.tradefed.util.Pair;
22 
23 import com.google.common.annotations.VisibleForTesting;
24 
25 /** A metrics object to hold run metrics and test metrics parsed by {@link MetricsXmlParser} */
26 public class Metrics {
27     private int mNumRuns;
28     private int mNumTests = -1;
29     private final boolean mStrictMode;
30     private final MultiMap<String, Double> mRunMetrics = new MultiMap<>();
31     private final MultiMap<Pair<TestDescription, String>, Double> mTestMetrics = new MultiMap<>();
32 
33     /** Throw when metrics validation fails in strict mode. */
34     public static class MetricsException extends RuntimeException {
MetricsException(String cause)35         MetricsException(String cause) {
36             super(cause);
37         }
38     }
39 
40     /**
41      * Constructs an empty Metrics object.
42      *
43      * @param strictMode whether exception should be thrown when validation fails
44      */
Metrics(boolean strictMode)45     public Metrics(boolean strictMode) {
46         mStrictMode = strictMode;
47     }
48 
49     /**
50      * Sets the number of tests. This method also checks if each call sets the same number of test,
51      * since this number should be consistent across multiple runs.
52      *
53      * @param numTests the number of tests
54      * @throws MetricsException if subsequent calls set a different number.
55      */
setNumTests(int numTests)56     public void setNumTests(int numTests) {
57         if (mNumTests == -1) {
58             mNumTests = numTests;
59         } else {
60             if (mNumTests != numTests) {
61                 String msg =
62                         String.format(
63                                 "Number of test entries differ: expect #%d actual #%d",
64                                 mNumTests, numTests);
65                 throw new MetricsException(msg);
66             }
67         }
68     }
69 
70     /**
71      * Adds a run metric.
72      *
73      * @param name metric name
74      * @param value metric value
75      */
addRunMetric(String name, String value)76     public void addRunMetric(String name, String value) {
77         try {
78             mRunMetrics.put(name, Double.parseDouble(value));
79         } catch (NumberFormatException e) {
80             // This is normal. We often get some string metrics like device name. Just log it.
81             CLog.w(String.format("Run metric \"%s\" is not a number: \"%s\"", name, value));
82         }
83     }
84 
85     /**
86      * Adds a test metric.
87      *
88      * @param testId TestDescription of the metric
89      * @param name metric name
90      * @param value metric value
91      */
addTestMetric(TestDescription testId, String name, String value)92     public void addTestMetric(TestDescription testId, String name, String value) {
93         Pair<TestDescription, String> metricId = new Pair<>(testId, name);
94         try {
95             mTestMetrics.put(metricId, Double.parseDouble(value));
96         } catch (NumberFormatException e) {
97             // This is normal. We often get some string metrics like device name. Just log it.
98             CLog.w(
99                     String.format(
100                             "Test %s metric \"%s\" is not a number: \"%s\"", testId, name, value));
101         }
102     }
103 
104     /**
105      * Validates that the number of entries of each metric equals to the number of runs.
106      *
107      * @param numRuns number of runs
108      * @throws MetricsException when validation fails in strict mode
109      */
validate(int numRuns)110     public void validate(int numRuns) {
111         mNumRuns = numRuns;
112         for (String name : mRunMetrics.keySet()) {
113             if (mRunMetrics.get(name).size() < mNumRuns) {
114                 error(
115                         String.format(
116                                 "Run metric \"%s\" too few entries: expected #%d actual #%d",
117                                 name, mNumRuns, mRunMetrics.get(name).size()));
118             }
119         }
120         for (Pair<TestDescription, String> id : mTestMetrics.keySet()) {
121             if (mTestMetrics.get(id).size() < mNumRuns) {
122                 error(
123                         String.format(
124                                 "Test %s metric \"%s\" too few entries: expected #%d actual #%d",
125                                 id.first, id.second, mNumRuns, mTestMetrics.get(id).size()));
126             }
127         }
128     }
129 
130     /**
131      * Validates with after-patch Metrics object. Make sure two metrics object contain same run
132      * metric entries and test metric entries. Assume this object contains before-patch metrics.
133      *
134      * @param after a Metrics object containing after-patch metrics
135      * @throws MetricsException when cross validation fails in strict mode
136      */
crossValidate(Metrics after)137     public void crossValidate(Metrics after) {
138         if (mNumTests != after.mNumTests) {
139             error(
140                     String.format(
141                             "Number of test entries differ: before #%d after #%d",
142                             mNumTests, after.mNumTests));
143         }
144 
145         for (String name : mRunMetrics.keySet()) {
146             if (!after.mRunMetrics.containsKey(name)) {
147                 warn(String.format("Run metric \"%s\" only in before-patch run.", name));
148             }
149         }
150 
151         for (String name : after.mRunMetrics.keySet()) {
152             if (!mRunMetrics.containsKey(name)) {
153                 warn(String.format("Run metric \"%s\" only in after-patch run.", name));
154             }
155         }
156 
157         for (Pair<TestDescription, String> id : mTestMetrics.keySet()) {
158             if (!after.mTestMetrics.containsKey(id)) {
159                 warn(
160                         String.format(
161                                 "Test %s metric \"%s\" only in before-patch run.",
162                                 id.first, id.second));
163             }
164         }
165 
166         for (Pair<TestDescription, String> id : after.mTestMetrics.keySet()) {
167             if (!mTestMetrics.containsKey(id)) {
168                 warn(
169                         String.format(
170                                 "Test %s metric \"%s\" only in after-patch run.",
171                                 id.first, id.second));
172             }
173         }
174     }
175 
176     @VisibleForTesting
error(String msg)177     void error(String msg) {
178         if (mStrictMode) {
179             throw new MetricsException(msg);
180         } else {
181             CLog.e(msg);
182         }
183     }
184 
185     @VisibleForTesting
warn(String msg)186     void warn(String msg) {
187         if (mStrictMode) {
188             throw new MetricsException(msg);
189         } else {
190             CLog.w(msg);
191         }
192     }
193 
194     /**
195      * Gets the number of test runs stored in this object.
196      *
197      * @return number of test runs
198      */
getNumRuns()199     public int getNumRuns() {
200         return mNumRuns;
201     }
202 
203     /**
204      * Gets the number of tests stored in this object.
205      *
206      * @return number of tests
207      */
getNumTests()208     public int getNumTests() {
209         return mNumTests;
210     }
211 
212     /**
213      * Gets all run metrics stored in this object.
214      *
215      * @return a {@link MultiMap} from test name String to Double
216      */
getRunMetrics()217     public MultiMap<String, Double> getRunMetrics() {
218         return mRunMetrics;
219     }
220 
221     /**
222      * Gets all test metrics stored in this object.
223      *
224      * @return a {@link MultiMap} from (TestDescription, test name) pair to Double
225      */
getTestMetrics()226     public MultiMap<Pair<TestDescription, String>, Double> getTestMetrics() {
227         return mTestMetrics;
228     }
229 }
230