1 /* 2 * Copyright (C) 2018 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package com.android.build.tests; 18 19 import com.android.ddmlib.Log; 20 import com.android.tradefed.build.IBuildInfo; 21 import com.android.tradefed.config.Option; 22 import com.android.tradefed.config.OptionClass; 23 import com.android.tradefed.device.DeviceNotAvailableException; 24 import com.android.tradefed.log.LogUtil.CLog; 25 import com.android.tradefed.metrics.proto.MetricMeasurement.Metric; 26 import com.android.tradefed.result.ITestInvocationListener; 27 import com.android.tradefed.result.TestDescription; 28 import com.android.tradefed.testtype.IBuildReceiver; 29 import com.android.tradefed.testtype.IRemoteTest; 30 import com.android.tradefed.util.proto.TfMetricProtoUtil; 31 32 import java.io.BufferedReader; 33 import java.io.File; 34 import java.io.FileInputStream; 35 import java.io.IOException; 36 import java.io.InputStream; 37 import java.io.InputStreamReader; 38 import java.util.HashMap; 39 import java.util.HashSet; 40 import java.util.Map; 41 import java.util.Set; 42 import java.util.regex.Matcher; 43 import java.util.regex.Pattern; 44 45 /** 46 * A device-less test that parses standard Android build image stats file and performs aggregation 47 */ 48 @OptionClass(alias = "image-stats") 49 public class ImageStats implements IRemoteTest, IBuildReceiver { 50 51 // built-in aggregation labels 52 private static final String LABEL_TOTAL = "total"; 53 private static final String LABEL_CATEGORIZED = "categorized"; 54 private static final String LABEL_UNCATEGORIZED = "uncategorized"; 55 56 private static final String FILE_SIZES = "fileSizes"; 57 58 @Option( 59 name = "size-stats-file", 60 description = 61 "Specify the name of the file containing image " 62 + "stats; when \"file-from-build-info\" is set to true, the name refers to a file that " 63 + "can be found in build info (note that build provider must be properly configured to " 64 + "download it), otherwise it refers to a local file, typically used for debugging " 65 + "purposes", 66 mandatory = true) 67 private String mStatsFileName = null; 68 69 @Option( 70 name = "file-from-build-info", 71 description = 72 "If the \"size-stats-file\" references a " 73 + "file from build info, or local; use local file for debugging purposes.") 74 private boolean mFileFromBuildInfo = false; 75 76 @Option( 77 name = "aggregation-pattern", 78 description = 79 "A key value pair consists of a regex as " 80 + "key and a string label as value. The regex is used to scan and group file size " 81 + "entries together for aggregation; that is, all files with names matching the " 82 + "pattern are grouped together for summing; this also means that a file could be " 83 + "counted multiple times; note that the regex must be a full match, not substring. " 84 + "The string label is used for identifying the summed group of file sizes when " 85 + "reporting; the regex may contain unnamed capturing groups, and values may contain " 86 + "numerical \"back references\" as place holders to be replaced with content of " 87 + "corresponding capturing group, example: ^.+\\.(.+)$ -> group-by-extension-\\1; back " 88 + "references are 1-indexed and there maybe up to 9 capturing groups; no strict checks " 89 + "are performed to ensure that capturing groups and place holders are 1:1 mapping. " 90 + "There are 3 built-in aggregations: total, categorized and uncategorized.") 91 private Map<String, String> mAggregationPattern = new HashMap<>(); 92 93 @Option( 94 name = "min-report-size", 95 description = 96 "Minimum size in bytes that an aggregated " 97 + "category needs to reach before being included in reported metrics. 0 for no limit. " 98 + "Note that built-in categories are always reported.") 99 private long mMinReportSize = 0; 100 101 private IBuildInfo mBuildInfo; 102 103 @Override setBuild(IBuildInfo buildInfo)104 public void setBuild(IBuildInfo buildInfo) { 105 mBuildInfo = buildInfo; 106 } 107 108 /** {@inheritDoc} */ 109 @Override run(ITestInvocationListener listener)110 public void run(ITestInvocationListener listener) throws DeviceNotAvailableException { 111 File statsFile; 112 if (mFileFromBuildInfo) { 113 statsFile = mBuildInfo.getFile(mStatsFileName); 114 } else { 115 statsFile = new File(mStatsFileName); 116 } 117 long start = System.currentTimeMillis(); 118 Map<String, String> fileSizes = null; 119 // fixed run name, 1 test to run 120 listener.testRunStarted("image-stats", 1); 121 if (statsFile == null || !statsFile.exists()) { 122 throw new RuntimeException( 123 "Invalid image stats file (<null>) specified or it does not exist."); 124 } else { 125 TestDescription td = new TestDescription(ImageStats.class.getName(), FILE_SIZES); 126 listener.testStarted(td); 127 try (InputStream in = new FileInputStream(statsFile)) { 128 fileSizes = 129 performAggregation( 130 parseFileSizes(in), 131 processAggregationPatterns(mAggregationPattern)); 132 } catch (IOException ioe) { 133 String message = 134 String.format( 135 "Failed to parse image stats file: %s", 136 statsFile.getAbsolutePath()); 137 CLog.e(message); 138 CLog.e(ioe); 139 listener.testFailed(td, ioe.toString()); 140 listener.testEnded(td, new HashMap<String, Metric>()); 141 listener.testRunFailed(message); 142 listener.testRunEnded( 143 System.currentTimeMillis() - start, new HashMap<String, Metric>()); 144 throw new RuntimeException(message, ioe); 145 } 146 String logOutput = String.format("File sizes: %s", fileSizes.toString()); 147 if (mFileFromBuildInfo) { 148 CLog.v(logOutput); 149 } else { 150 // assume local debug, print outloud 151 CLog.logAndDisplay(Log.LogLevel.VERBOSE, logOutput); 152 } 153 listener.testEnded(td, TfMetricProtoUtil.upgradeConvert(fileSizes)); 154 } 155 listener.testRunEnded(System.currentTimeMillis() - start, new HashMap<String, Metric>()); 156 } 157 158 /** 159 * Processes text files like 'installed-files.txt' (as built by standard Android build rules for 160 * device targets) into a map of file path to file sizes 161 * 162 * @param in an unread {@link InputStream} for the content of the file sizes; the stream will be 163 * fully read after executing the method 164 * @return 165 */ parseFileSizes(InputStream in)166 protected Map<String, Long> parseFileSizes(InputStream in) throws IOException { 167 Map<String, Long> ret = new HashMap<>(); 168 try (BufferedReader br = new BufferedReader(new InputStreamReader(in))) { 169 String line; 170 while ((line = br.readLine()) != null) { 171 // trim both ends of the raw line and make a split by whitespaces 172 // e.g. trying to match a line like this: 173 // 96992106 /system/app/Chrome/Chrome.apk 174 String[] fields = line.trim().split("\\s+"); 175 if (fields.length != 2) { 176 CLog.w("Unable to split line to file size and name: %s", line); 177 continue; 178 } 179 long size = 0; 180 try { 181 size = Long.parseLong(fields[0]); 182 } catch (NumberFormatException nfe) { 183 CLog.w("Failed to parse file size from field '%s', ignored", fields[0]); 184 continue; 185 } 186 ret.put(fields[1], size); 187 } 188 } 189 return ret; 190 } 191 192 /** Compiles the supplied aggregation regex's */ processAggregationPatterns(Map<String, String> rawPatterns)193 protected Map<Pattern, String> processAggregationPatterns(Map<String, String> rawPatterns) { 194 Map<Pattern, String> ret = new HashMap<>(); 195 for (Map.Entry<String, String> e : rawPatterns.entrySet()) { 196 Pattern p = Pattern.compile(e.getKey()); 197 ret.put(p, e.getValue()); 198 } 199 return ret; 200 } 201 202 /** 203 * Converts a matched file entry to the final aggregation label name. 204 * 205 * <p>The main thing being converted here is that capturing groups in the regex (used to match 206 * the filenames) are extracted, and used to replace the corresponding placeholders in raw 207 * label. For each 1-indexed capturing group, the captured content is used to replace the "\x" 208 * placeholders in raw label, with x being a number between 1-9, corresponding to the index of 209 * the capturing group. 210 * 211 * @param matcher the {@link Matcher} representing the matched result from the regex and input 212 * @param rawLabel the corresponding aggregation label 213 * @return 214 */ getAggregationLabel(Matcher matcher, String rawLabel)215 protected String getAggregationLabel(Matcher matcher, String rawLabel) { 216 if (matcher.groupCount() == 0) { 217 // no capturing groups, return label as is 218 return rawLabel; 219 } 220 if (matcher.groupCount() > 9) { 221 // since we are doing replacement of back references to capturing groups manually, 222 // artificially limiting this to avoid overly complex code to handle \1 vs \10 223 // in other words, "9 capturing groups ought to be enough for anybody" 224 throw new RuntimeException("too many capturing groups"); 225 } 226 String label = rawLabel; 227 for (int i = 1; i <= matcher.groupCount(); i++) { 228 String marker = String.format("\\%d", i); // e.g. "\1" 229 if (label.indexOf(marker) == -1) { 230 CLog.w( 231 "Capturing groups were defined in regex '%s', but corresponding " 232 + "back-reference placeholder '%s' not found in label '%s'", 233 matcher.pattern(), marker, rawLabel); 234 continue; 235 } 236 label = label.replace(marker, matcher.group(i)); 237 } 238 // ensure that the resulting label is not the same as the fixed "uncategorized" label 239 if (LABEL_UNCATEGORIZED.equals(label)) { 240 throw new IllegalArgumentException( 241 String.format( 242 "Use of aggregation label '%s' " + "conflicts with built-in default.", 243 LABEL_UNCATEGORIZED)); 244 } 245 return label; 246 } 247 248 /** 249 * Performs aggregation by adding raw file size entries together based on the regex's the full 250 * path names are matched. Note that this means a file entry could get aggregated multiple 251 * times. The returned map will also include a fixed entry called "uncategorized" that adds the 252 * sizes of all file entries that were never matched together. 253 * 254 * @param stats the map of raw stats: full path name -> file size 255 * @param patterns the map of aggregation patterns: a regex that could match file names -> the 256 * name of the aggregated result category (e.g. all apks) 257 * @return 258 */ performAggregation( Map<String, Long> stats, Map<Pattern, String> patterns)259 protected Map<String, String> performAggregation( 260 Map<String, Long> stats, Map<Pattern, String> patterns) { 261 Set<String> uncategorizedFiles = new HashSet<>(stats.keySet()); 262 Map<String, Long> result = new HashMap<>(); 263 long total = 0; 264 265 for (Map.Entry<String, Long> stat : stats.entrySet()) { 266 // aggregate for total first 267 total += stat.getValue(); 268 for (Map.Entry<Pattern, String> pattern : patterns.entrySet()) { 269 Matcher m = pattern.getKey().matcher(stat.getKey()); 270 if (m.matches()) { 271 // the file entry being looked at matches one of the preconfigured rules 272 String label = getAggregationLabel(m, pattern.getValue()); 273 Long size = result.get(label); 274 if (size == null) { 275 size = 0L; 276 } 277 size += stat.getValue(); 278 result.put(label, size); 279 // keep track of files that we've already aggregated at least once 280 if (uncategorizedFiles.contains(stat.getKey())) { 281 uncategorizedFiles.remove(stat.getKey()); 282 } 283 } 284 } 285 } 286 // final pass for uncategorized files 287 long uncategorized = 0; 288 for (String file : uncategorizedFiles) { 289 uncategorized += stats.get(file); 290 } 291 Map<String, String> ret = new HashMap<>(); 292 for (Map.Entry<String, Long> e : result.entrySet()) { 293 if (mMinReportSize > 0 && e.getValue() < mMinReportSize) { 294 // has a min report size requirement and current category does not meet it 295 CLog.v( 296 "Skipped reporting for %s (value %d): it's below threshold %d", 297 e.getKey(), e.getValue(), mMinReportSize); 298 continue; 299 } 300 ret.put(e.getKey(), Long.toString(e.getValue())); 301 } 302 ret.put(LABEL_UNCATEGORIZED, Long.toString(uncategorized)); 303 ret.put(LABEL_TOTAL, Long.toString(total)); 304 ret.put(LABEL_CATEGORIZED, Long.toString(total - uncategorized)); 305 return ret; 306 } 307 } 308