1 /*
2  * Copyright (C) 2018 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 package com.android.tradefed.postprocessor;
17 
18 import com.android.tradefed.config.OptionClass;
19 import com.android.tradefed.metrics.proto.MetricMeasurement.Measurements;
20 import com.android.tradefed.metrics.proto.MetricMeasurement.Metric;
21 import com.android.tradefed.result.LogFile;
22 import com.android.tradefed.result.TestDescription;
23 
24 import com.google.common.collect.ArrayListMultimap;
25 
26 import java.util.ArrayList;
27 import java.util.Arrays;
28 import java.util.Collection;
29 import java.util.Collections;
30 import java.util.HashMap;
31 import java.util.List;
32 import java.util.Map;
33 import java.util.stream.Collectors;
34 
35 /**
36  * A metric aggregator that gives the min, max, mean, variance and standard deviation for numeric
37  * metrics collected during multiple-iteration test runs, treating them as doubles. Non-numeric
38  * metrics are ignored.
39  *
40  * <p>It parses metrics from single string as currently metrics are passed this way.
41  */
42 @OptionClass(alias = "aggregate-post-processor")
43 public class AggregatePostProcessor extends BasePostProcessor {
44     private static final String STATS_KEY_MIN = "min";
45     private static final String STATS_KEY_MAX = "max";
46     private static final String STATS_KEY_MEAN = "mean";
47     private static final String STATS_KEY_VAR = "var";
48     private static final String STATS_KEY_STDEV = "stdev";
49     private static final String STATS_KEY_MEDIAN = "median";
50     private static final String STATS_KEY_TOTAL = "total";
51     // Separator for final upload
52     private static final String STATS_KEY_SEPARATOR = "-";
53 
54     // Stores the test metrics for aggregation by test description.
55     // TODO(b/118708851): Remove this workaround once AnTS is ready.
56     private HashMap<String, ArrayListMultimap<String, Metric>> mStoredTestMetrics =
57             new HashMap<String, ArrayListMultimap<String, Metric>>();
58 
59     @Override
processTestMetricsAndLogs( TestDescription testDescription, HashMap<String, Metric> testMetrics, Map<String, LogFile> testLogs)60     public Map<String, Metric.Builder> processTestMetricsAndLogs(
61             TestDescription testDescription,
62             HashMap<String, Metric> testMetrics,
63             Map<String, LogFile> testLogs) {
64         // TODO(b/118708851): Move this processing elsewhere once AnTS is ready.
65         // Use the string representation of the test description to key the tests.
66         String fullTestName = testDescription.toString();
67         // Store result from the current test.
68         if (!mStoredTestMetrics.containsKey(fullTestName)) {
69             mStoredTestMetrics.put(fullTestName, ArrayListMultimap.create());
70         }
71         ArrayListMultimap<String, Metric> storedMetricsForThisTest =
72                 mStoredTestMetrics.get(fullTestName);
73         for (Map.Entry<String, Metric> entry : testMetrics.entrySet()) {
74             storedMetricsForThisTest.put(entry.getKey(), entry.getValue());
75         }
76         // Aggregate all data in iterations of this test.
77         Map<String, Metric.Builder> aggregateMetrics = new HashMap<String, Metric.Builder>();
78         for (String metricKey : storedMetricsForThisTest.keySet()) {
79             List<Metric> metrics = storedMetricsForThisTest.get(metricKey);
80             List<Measurements> measures =
81                     metrics.stream().map(Metric::getMeasurements).collect(Collectors.toList());
82             // Parse metrics into a list of SingleString values, concating lists in the process
83             List<String> rawValues =
84                     measures.stream()
85                             .map(Measurements::getSingleString)
86                             .map(
87                                     m -> {
88                                         // Split results; also deals with the case of empty results
89                                         // in a certain run
90                                         List<String> splitVals = Arrays.asList(m.split(",", 0));
91                                         if (splitVals.size() == 1 && splitVals.get(0).isEmpty()) {
92                                             return Collections.<String>emptyList();
93                                         }
94                                         return splitVals;
95                                     })
96                             .flatMap(Collection::stream)
97                             .map(String::trim)
98                             .collect(Collectors.toList());
99             // Do not report empty metrics
100             if (rawValues.isEmpty()) {
101                 continue;
102             }
103             if (isAllDoubleValues(rawValues)) {
104                 buildStats(metricKey, rawValues, aggregateMetrics);
105             }
106         }
107         return aggregateMetrics;
108     }
109 
110     @Override
processRunMetricsAndLogs( HashMap<String, Metric> rawMetrics, Map<String, LogFile> runLogs)111     public Map<String, Metric.Builder> processRunMetricsAndLogs(
112             HashMap<String, Metric> rawMetrics, Map<String, LogFile> runLogs) {
113         // Aggregate the test run metrics which has comma separated values which can be
114         // parsed to double values.
115         Map<String, Metric.Builder> aggregateMetrics = new HashMap<String, Metric.Builder>();
116         for (Map.Entry<String, Metric> entry : rawMetrics.entrySet()) {
117             String values = entry.getValue().getMeasurements().getSingleString();
118             List<String> splitVals = Arrays.asList(values.split(",", 0));
119             // Build stats for keys with any values, even only one.
120             if (isAllDoubleValues(splitVals)) {
121                 buildStats(entry.getKey(), splitVals, aggregateMetrics);
122             }
123         }
124         return aggregateMetrics;
125     }
126 
127     /**
128      * Return true is all the values can be parsed to double value.
129      * Otherwise return false.
130      * @param rawValues list whose values are validated.
131      * @return
132      */
isAllDoubleValues(List<String> rawValues)133     private boolean isAllDoubleValues(List<String> rawValues) {
134         return rawValues
135                 .stream()
136                 .allMatch(
137                         val -> {
138                             try {
139                                 Double.parseDouble(val);
140                                 return true;
141                             } catch (NumberFormatException e) {
142                                 return false;
143                             }
144                         });
145     }
146 
147     /**
148      * Build stats for the given set of values and build the metrics using the metric key
149      * and stats name and update the results in aggregated metrics.
150      *
151      * @param metricKey key to which the values correspond to.
152      * @param values list of raw values.
153      * @param aggregateMetrics where final metrics will be stored.
154      */
155     private void buildStats(String metricKey, List<String> values,
156             Map<String, Metric.Builder> aggregateMetrics) {
157         List<Double> doubleValues =
158                 values.stream().map(Double::parseDouble).collect(Collectors.toList());
159         HashMap<String, Double> stats = getStats(doubleValues);
160         for (String statKey : stats.keySet()) {
161             Metric.Builder metricBuilder = Metric.newBuilder();
162             metricBuilder
163                     .getMeasurementsBuilder()
164                     .setSingleString(String.format("%2.2f", stats.get(statKey)));
165             aggregateMetrics.put(
166                     String.join(STATS_KEY_SEPARATOR, metricKey, statKey),
167                     metricBuilder);
168         }
169     }
170 
171     private HashMap<String, Double> getStats(Collection<Double> values) {
172         List<Double> valuesList = new ArrayList<>(values);
173         Collections.sort(valuesList);
174         HashMap<String, Double> stats = new HashMap<>();
175         double sum = values.stream().mapToDouble(Double::doubleValue).sum();
176         double count = (double) valuesList.size();
177         // The orElse situation should never happen.
178         double mean =
179                 values.stream()
180                         .mapToDouble(Double::doubleValue)
181                         .average()
182                         .orElseThrow(IllegalStateException::new);
183         double variance = values.stream().reduce(0.0, (a, b) -> a + Math.pow(b - mean, 2) / count);
184         // Calculate median.
185         double median = valuesList.get(valuesList.size() / 2);
186         if (valuesList.size() % 2 == 0) {
187             median = (median + valuesList.get(valuesList.size() / 2 - 1)) / 2.0;
188         }
189 
190         stats.put(STATS_KEY_MIN, valuesList.get(0));
191         stats.put(STATS_KEY_MAX, valuesList.get(valuesList.size() - 1));
192         stats.put(STATS_KEY_MEAN, mean);
193         stats.put(STATS_KEY_VAR, variance);
194         stats.put(STATS_KEY_STDEV, Math.sqrt(variance));
195         stats.put(STATS_KEY_MEDIAN, median);
196         stats.put(STATS_KEY_TOTAL, sum);
197         return stats;
198     }
199 }
200