1 /* 2 * Copyright (C) 2018 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package com.android.tradefed.device.metric; 17 18 import com.android.annotations.Nullable; 19 import com.android.annotations.VisibleForTesting; 20 import com.android.tradefed.build.IBuildInfo; 21 import com.android.tradefed.config.Option; 22 import com.android.tradefed.config.OptionClass; 23 import com.android.tradefed.device.DeviceNotAvailableException; 24 import com.android.tradefed.device.ITestDevice; 25 import com.android.tradefed.device.LargeOutputReceiver; 26 import com.android.tradefed.log.LogUtil.CLog; 27 import com.android.tradefed.metrics.proto.MetricMeasurement.DataType; 28 import com.android.tradefed.metrics.proto.MetricMeasurement.Metric; 29 import com.android.tradefed.result.FileInputStreamSource; 30 import com.android.tradefed.result.InputStreamSource; 31 import com.android.tradefed.result.LogDataType; 32 import com.android.tradefed.util.CommandResult; 33 import com.android.tradefed.util.CommandStatus; 34 import com.android.tradefed.util.FileUtil; 35 import com.android.tradefed.util.Pair; 36 import com.android.tradefed.util.RunUtil; 37 import com.android.tradefed.util.StreamUtil; 38 import com.android.tradefed.util.ZipUtil; 39 40 import java.io.ByteArrayOutputStream; 41 import java.io.File; 42 import java.io.FileNotFoundException; 43 import java.io.FileOutputStream; 44 import java.io.IOException; 45 import java.io.InputStream; 46 import java.io.OutputStream; 47 import java.util.ArrayList; 48 import java.util.Collection; 49 import java.util.List; 50 import java.util.concurrent.TimeUnit; 51 52 /** 53 * Base implementation of {@link FilePullerDeviceMetricCollector} that allows 54 * pulling the perfetto files from the device and collect the metrics from it. 55 * Also used for converting the raw trace file into perfetto metric file. 56 */ 57 @OptionClass(alias = "perfetto-metric-collector") 58 public class PerfettoPullerMetricCollector extends FilePullerDeviceMetricCollector { 59 60 private static final String LINE_SEPARATOR = "\\r?\\n"; 61 private static final char KEY_VALUE_SEPARATOR = ':'; 62 private static final String EXTRACTOR_STATUS = "trace_extractor_status"; 63 private static final String EXTRACTOR_SUCCESS = "1"; 64 private static final String EXTRACTOR_FAILURE = "0"; 65 private static final String EXTRACTOR_RUNTIME = "trace_extractor_runtime"; 66 67 public enum METRIC_FILE_FORMAT { 68 text, 69 binary, 70 json, 71 } 72 73 @Option(name = "compress-perfetto", 74 description = "If enabled retrieves the perfetto compressed content," 75 + "decompress for processing and upload the compressed file. If" 76 + "this flag is not enabled uncompressed version of perfetto file is" 77 + "pulled, processed and uploaded.") 78 private boolean mCompressPerfetto = false; 79 80 @Option(name = "max-compressed-file-size", description = "Max size of the compressed" 81 + " perfetto file. If the compressed file size exceeds the max then" 82 + " post processing and uploading the compressed file will not happen.") 83 private long mMaxCompressedFileSize = 10000L * 1024 * 1024; 84 85 @Option( 86 name = "compressed-trace-shell-timeout", 87 description = "Timeout for retrieving compressed trace content through shell", 88 isTimeVal = true) 89 private long mCompressedTimeoutMs = TimeUnit.MINUTES.toMillis(20); 90 91 @Option( 92 name = "compress-response-timeout", 93 description = "Timeout to receive the shell response when running the gzip command.", 94 isTimeVal = true) 95 private long mCompressResponseTimeoutMs = TimeUnit.SECONDS.toMillis(30); 96 97 @Option( 98 name = "decompress-perfetto-timeout", 99 description = "Timeout to decompress perfetto compressed file.", 100 isTimeVal = true) 101 private long mDecompressTimeoutMs = TimeUnit.MINUTES.toMillis(20); 102 103 @Option( 104 name = "perfetto-binary-path", 105 description = "Path to the script files used to analyze the trace files." 106 + "Used for collecting the key value metrics from the perfetto" 107 + "trace file.") 108 @Deprecated 109 private List<File> mScriptFiles = new ArrayList<>(); 110 111 @Option( 112 name = "perfetto-binary-args", 113 description = "Extra arguments to be passed to the binaries.") 114 @Deprecated 115 private List<String> mPerfettoBinaryArgs = new ArrayList<>(); 116 117 @Option( 118 name = "perfetto-metric-prefix", 119 description = "Prefix to be used with the metrics collected from perfetto.") 120 @Deprecated 121 private String mMetricPrefix = "perfetto"; 122 123 // List of process names passed to perfetto binary. 124 @Option( 125 name = "process-name", 126 description = 127 "Process names to be passed in perfetto script.") 128 @Deprecated 129 private Collection<String> mProcessNames = new ArrayList<String>(); 130 131 // Timeout for the script to process the trace files. 132 // The default is arbitarily chosen to be 5 mins to prevent the test spending more time in 133 // processing the files. 134 @Option( 135 name = "perfetto-script-timeout", 136 description = "Timeout for the perfetto script.", 137 isTimeVal = true) 138 @Deprecated 139 private long mScriptTimeoutMs = TimeUnit.MINUTES.toMillis(5); 140 141 @Option(name = "convert-metric-file", 142 description = "Convert the raw trace file to perfetto metric file.") 143 private boolean mConvertToMetricFile = true; 144 145 @Option( 146 name = "trace-processor-binary", 147 description = "Path to the trace processor shell. This will" 148 + " override the trace-processor-name which is used to " 149 + " lookup in build artifacts. Used for converting the raw" 150 + " trace into perfetto metric file.") 151 private File mTraceProcessorBinary = null; 152 153 @Option( 154 name = "trace-processor-name", 155 description = "Trace processor name to look up from the test artifacts" 156 + " or module artifacts.") 157 private String mTraceProcessorName = "trace_processor_shell"; 158 159 @Option( 160 name = "trace-processor-run-metrics", 161 description = "Comma separated list of metrics to extract from raw trace file." 162 + " For example android_mem.") 163 private String mTraceProcessorMetrics = "android_mem"; 164 165 @Option( 166 name = "trace-processor-output-format", 167 description = "Trace processor output format. [binary|text|json]") 168 private METRIC_FILE_FORMAT mTraceProcessorOutputFormat = METRIC_FILE_FORMAT.text; 169 170 @Option( 171 name = "trace-processor-timeout", 172 description = "Timeout to convert the raw trace file to metric proto file.", 173 isTimeVal = true) 174 private long mTraceConversionTimeout = TimeUnit.MINUTES.toMillis(20); 175 176 177 /** 178 * Process the perfetto trace file for the additional metrics and add it to final metrics. 179 * Decompress the perfetto file for processing if the compression was enabled. 180 * 181 * @param key the option key associated to the file that was pulled from the device. 182 * @param metricFile the {@link File} pulled from the device matching the option key. 183 * @param data where metrics will be stored. 184 */ 185 @Override processMetricFile(String key, File metricFile, DeviceMetricData data)186 public void processMetricFile(String key, File metricFile, 187 DeviceMetricData data) { 188 File processSrcFile = metricFile; 189 if (mCompressPerfetto) { 190 processSrcFile = decompressFile(metricFile); 191 } 192 193 // Convert to perfetto metric format. 194 if (mConvertToMetricFile) { 195 File convertedMetricFile = convertToMetricProto(processSrcFile); 196 if (convertedMetricFile != null) { 197 try (InputStreamSource source = new FileInputStreamSource(convertedMetricFile, 198 true)) { 199 testLog(convertedMetricFile.getName(), getLogDataType(), source); 200 } 201 } 202 } 203 204 if (processSrcFile != null) { 205 // Extract the metrics from the trace file. 206 for (File scriptFile : mScriptFiles) { 207 // Apply necessary execute permissions to the script. 208 FileUtil.chmodGroupRWX(scriptFile); 209 210 List<String> commandArgsList = new ArrayList<String>(); 211 commandArgsList.add(scriptFile.getAbsolutePath()); 212 commandArgsList.addAll(mPerfettoBinaryArgs); 213 commandArgsList.add("-trace_file"); 214 commandArgsList.add(processSrcFile.getAbsolutePath()); 215 216 if (!mProcessNames.isEmpty()) { 217 commandArgsList.add("-process_names"); 218 commandArgsList.add(String.join(",", mProcessNames)); 219 } 220 221 String traceExtractorStatus = EXTRACTOR_SUCCESS; 222 223 double scriptDuration = 0; 224 double scriptStartTime = System.currentTimeMillis(); 225 CommandResult cr = runHostCommand(mScriptTimeoutMs, 226 commandArgsList.toArray(new String[commandArgsList.size()]), null, 227 null); 228 scriptDuration = System.currentTimeMillis() - scriptStartTime; 229 230 // Update the script duration metrics. 231 Metric.Builder metricDurationBuilder = Metric.newBuilder(); 232 metricDurationBuilder.getMeasurementsBuilder().setSingleDouble(scriptDuration); 233 data.addMetric( 234 String.format("%s_%s", mMetricPrefix, EXTRACTOR_RUNTIME), 235 metricDurationBuilder.setType(DataType.RAW)); 236 237 if (CommandStatus.SUCCESS.equals(cr.getStatus())) { 238 String[] metrics = cr.getStdout().split(LINE_SEPARATOR); 239 for (String metric : metrics) { 240 Pair<String, String> kv = splitKeyValue(metric); 241 242 if (kv != null) { 243 Metric.Builder metricBuilder = Metric.newBuilder(); 244 metricBuilder.getMeasurementsBuilder().setSingleString(kv.second); 245 data.addMetric( 246 String.format("%s_%s", mMetricPrefix, kv.first), 247 metricBuilder.setType(DataType.RAW)); 248 } else { 249 CLog.e("Output %s not in the expected format.", metric); 250 } 251 } 252 CLog.i(cr.getStdout()); 253 } else { 254 traceExtractorStatus = EXTRACTOR_FAILURE; 255 CLog.e("Unable to parse the trace file %s due to %s - Status - %s ", 256 processSrcFile.getName(), cr.getStderr(), cr.getStatus()); 257 } 258 259 if (mCompressPerfetto) { 260 processSrcFile.delete(); 261 } 262 Metric.Builder metricStatusBuilder = Metric.newBuilder(); 263 metricStatusBuilder.getMeasurementsBuilder() 264 .setSingleString(traceExtractorStatus); 265 data.addMetric( 266 String.format("%s_%s", mMetricPrefix, EXTRACTOR_STATUS), 267 metricStatusBuilder.setType(DataType.RAW)); 268 } 269 } 270 271 // Upload and delete the host trace file. 272 try (InputStreamSource source = new FileInputStreamSource(metricFile, true)) { 273 if (mCompressPerfetto) { 274 if (processSrcFile != null) { 275 testLog(metricFile.getName(), LogDataType.GZIP, source); 276 } else { 277 metricFile.delete(); 278 } 279 280 } else { 281 testLog(metricFile.getName(), LogDataType.PERFETTO, source); 282 } 283 } 284 285 } 286 287 /** 288 * Converts the raw trace file into perfetto metric file. 289 * 290 * @param perfettoRawTraceFile Raw perfetto trace file that needs to be 291 * converted. 292 * @return File converted perfetto metric file. 293 */ convertToMetricProto(File perfettoRawTraceFile)294 private File convertToMetricProto(File perfettoRawTraceFile) { 295 296 // Use absolute path to the trace file if it is available otherwise 297 // resolve the trace processor name from the test or module artifacts. 298 if (mTraceProcessorBinary == null) { 299 mTraceProcessorBinary = getFileFromTestArtifacts(mTraceProcessorName); 300 } 301 302 File metricOutputFile = null; 303 if (mTraceProcessorBinary == null) { 304 CLog.e("Failed to locate the trace processor shell binary file."); 305 return metricOutputFile; 306 } 307 308 FileUtil.chmodGroupRWX(mTraceProcessorBinary); 309 List<String> commandArgsList = new ArrayList<String>(); 310 commandArgsList.add(mTraceProcessorBinary.getAbsolutePath()); 311 312 // Comma separated list of metrics to extract. 313 if (!mTraceProcessorMetrics.isEmpty()) { 314 commandArgsList.add("--run-metrics"); 315 commandArgsList.add(mTraceProcessorMetrics); 316 } 317 // Metric file output format. 318 commandArgsList.add("--metrics-output=" + mTraceProcessorOutputFormat); 319 commandArgsList.add(perfettoRawTraceFile.getAbsolutePath()); 320 321 try { 322 metricOutputFile = FileUtil.createTempFile( 323 "metric_" + getRawTraceFileName(perfettoRawTraceFile.getName()), ""); 324 } catch (IOException e) { 325 CLog.e("Not able to create metric perfetto output file."); 326 CLog.e(e); 327 return null; 328 } 329 330 // Running the trace conversion. 331 CLog.i("Run the trace convertor."); 332 boolean isConversionSuccess = true; 333 try (FileOutputStream outStream = new FileOutputStream(metricOutputFile); 334 ByteArrayOutputStream errStream = new ByteArrayOutputStream()) { 335 CommandResult conversionResult = runHostCommand(mTraceConversionTimeout, 336 commandArgsList.toArray(new String[commandArgsList 337 .size()]), 338 outStream, errStream); 339 if (!CommandStatus.SUCCESS.equals(conversionResult.getStatus())) { 340 CLog.e("Unable to convert the raw trace - %s to metric file due to" 341 + " %s - Status - %s ", perfettoRawTraceFile.getName(), 342 errStream.toString(), conversionResult.getStatus()); 343 isConversionSuccess = false; 344 } else if (mTraceProcessorOutputFormat.equals(METRIC_FILE_FORMAT.text)) { 345 CLog.i("Compressing the perfetto metric text proto."); 346 File compressedFile = getCompressedFile(metricOutputFile); 347 metricOutputFile.delete(); 348 return compressedFile; 349 } 350 } catch (FileNotFoundException e) { 351 CLog.e("Not able to find the result metric file to write the " 352 + "metric output."); 353 CLog.e(e); 354 isConversionSuccess = false; 355 } catch (IOException e1) { 356 CLog.e("Unable to close the streams."); 357 CLog.e(e1); 358 isConversionSuccess = false; 359 } finally { 360 if (!isConversionSuccess) { 361 metricOutputFile.delete(); 362 return null; 363 } 364 } 365 return metricOutputFile; 366 } 367 368 369 /** 370 * Pull the file from the specified path in the device. Pull the compressed content of the 371 * perfetto file if the compress perfetto option is enabled. 372 * 373 * @param device which has the file. 374 * @param remoteFilePath location in the device. 375 * @return compressed or decompressed version of perfetto file based on mCompressPerfetto 376 * option is set or not. 377 * @throws DeviceNotAvailableException 378 */ 379 @Override retrieveFile(ITestDevice device, String remoteFilePath)380 protected File retrieveFile(ITestDevice device, String remoteFilePath) 381 throws DeviceNotAvailableException { 382 if (!mCompressPerfetto) { 383 return super.retrieveFile(device, remoteFilePath); 384 } 385 File perfettoCompressedFile = null; 386 try { 387 String filePathInDevice = remoteFilePath; 388 CLog.i("Retrieving the compressed perfetto trace content from device."); 389 LargeOutputReceiver compressedOutputReceiver = new LargeOutputReceiver( 390 "perfetto_compressed_temp", 391 device.getSerialNumber(), mMaxCompressedFileSize); 392 device.executeShellCommand( 393 String.format("gzip -c %s", filePathInDevice), 394 compressedOutputReceiver, 395 mCompressedTimeoutMs, mCompressResponseTimeoutMs, TimeUnit.MILLISECONDS, 1); 396 compressedOutputReceiver.flush(); 397 compressedOutputReceiver.cancel(); 398 399 // Copy to temp file which will be used for decompression, perfetto 400 // metrics extraction and uploading the file later. 401 try (InputStreamSource largeStreamSrc = compressedOutputReceiver.getData(); 402 InputStream inputStream = largeStreamSrc.createInputStream()) { 403 perfettoCompressedFile = FileUtil.createTempFile( 404 "perfetto_compressed", ".gz"); 405 FileOutputStream outStream = new FileOutputStream( 406 perfettoCompressedFile); 407 byte[] buffer = new byte[4096]; 408 int bytesRead = -1; 409 while ((bytesRead = inputStream.read(buffer)) > -1) { 410 outStream.write(buffer, 0, bytesRead); 411 } 412 StreamUtil.close(outStream); 413 CLog.i("Successfully copied the compressed content from device to" 414 + " host."); 415 } catch (IOException e) { 416 if (perfettoCompressedFile != null) { 417 perfettoCompressedFile.delete(); 418 } 419 CLog.e("Failed to copy compressed perfetto to temporary file."); 420 CLog.e(e); 421 } finally { 422 compressedOutputReceiver.delete(); 423 } 424 } catch (DeviceNotAvailableException e) { 425 CLog.e( 426 "Exception when retrieveing compressed perfetto trace file '%s' " 427 + "from %s", remoteFilePath, device.getSerialNumber()); 428 CLog.e(e); 429 } 430 return perfettoCompressedFile; 431 } 432 433 /** 434 * Decompress the file to a temporary file in the host. 435 * 436 * @param compressedFile file to be decompressed. 437 * @return decompressed file used for postprocessing. 438 */ decompressFile(File compressedFile)439 private File decompressFile(File compressedFile) { 440 File decompressedFile = null; 441 try { 442 decompressedFile = FileUtil.createTempFile("perfetto_decompressed", ".pb"); 443 } catch (IOException e) { 444 CLog.e("Not able to create decompressed perfetto file."); 445 CLog.e(e); 446 return null; 447 } 448 // Keep the original file for uploading. 449 List<String> decompressArgsList = new ArrayList<String>(); 450 decompressArgsList.add("gzip"); 451 decompressArgsList.add("-k"); 452 decompressArgsList.add("-c"); 453 decompressArgsList.add("-d"); 454 decompressArgsList.add(compressedFile.getAbsolutePath()); 455 456 // Decompress perfetto trace file. 457 CLog.i("Start decompressing the perfetto trace file."); 458 try (FileOutputStream outStream = new FileOutputStream(decompressedFile); 459 ByteArrayOutputStream errStream = new ByteArrayOutputStream()) { 460 CommandResult decompressResult = runHostCommand(mDecompressTimeoutMs, 461 decompressArgsList.toArray(new String[decompressArgsList 462 .size()]), outStream, errStream); 463 464 if (!CommandStatus.SUCCESS.equals(decompressResult.getStatus())) { 465 CLog.e("Unable decompress the metric file %s due to %s - Status - %s ", 466 compressedFile.getName(), errStream.toString(), 467 decompressResult.getStatus()); 468 decompressedFile.delete(); 469 return null; 470 } 471 } catch (FileNotFoundException e) { 472 CLog.e("Not able to find the decompressed file to copy the" 473 + " decompressed contents."); 474 CLog.e(e); 475 return null; 476 } catch (IOException e1) { 477 CLog.e("Unable to close the streams."); 478 CLog.e(e1); 479 } 480 CLog.i("Successfully decompressed the perfetto trace file."); 481 return decompressedFile; 482 } 483 484 @Override processMetricDirectory(String key, File metricDirectory, DeviceMetricData runData)485 public void processMetricDirectory(String key, File metricDirectory, DeviceMetricData runData) { 486 // Implement if all the files under specific directory have to be post processed. 487 } 488 489 /** 490 * Run a host command with the given array of command args. 491 * 492 * @param commandArgs args to be used to construct the host command. 493 * @param stdout output of the command. 494 * @param stderr error message if any from the command. 495 * @return return the command results. 496 */ 497 @VisibleForTesting runHostCommand(long timeOut, String[] commandArgs, OutputStream stdout, OutputStream stderr)498 CommandResult runHostCommand(long timeOut, String[] commandArgs, OutputStream stdout, 499 OutputStream stderr) { 500 if (stdout != null && stderr != null) { 501 return RunUtil.getDefault().runTimedCmd(timeOut, stdout, stderr, commandArgs); 502 } 503 return RunUtil.getDefault().runTimedCmd(timeOut, commandArgs); 504 } 505 506 @VisibleForTesting 507 @Nullable splitKeyValue(String s)508 static Pair<String, String> splitKeyValue(String s) { 509 // Expected script test output format. 510 // Key1:Value1 511 // Key2:Value2 512 int separatorIdx = s.lastIndexOf(KEY_VALUE_SEPARATOR); 513 if (separatorIdx > 0 && separatorIdx + 1 < s.length()) { 514 return new Pair<>(s.substring(0, separatorIdx), s.substring(separatorIdx + 1)); 515 } 516 return null; 517 } 518 519 /** 520 * Get the log data type based on the output metric perfetto file. 521 * 522 * @return LogDataType type of the file used for uploading the artifacts. 523 */ getLogDataType()524 private LogDataType getLogDataType() { 525 // text option in perfetto trace processor means text proto. 526 if(mTraceProcessorOutputFormat.equals(METRIC_FILE_FORMAT.text)) { 527 return LogDataType.ZIP; 528 } else if(mTraceProcessorOutputFormat.equals(METRIC_FILE_FORMAT.binary)) { 529 return LogDataType.PB; 530 } else { 531 return LogDataType.TEXT; 532 } 533 } 534 535 /** 536 * Extract the raw trace file name used for constructing the output 537 * perfetto metric file name 538 * 539 * @param rawTraceFileName 540 * @return String name of the raw trace file name excluding the UUID. 541 */ getRawTraceFileName(String rawTraceFileName)542 private String getRawTraceFileName(String rawTraceFileName) { 543 // For example return perfetto_<test_name>-1_ from 544 // perfetto_<test_name>-1_13388308985625987330.pb excluding the UID. 545 int lastIndex = rawTraceFileName.lastIndexOf("_"); 546 if (lastIndex != -1) { 547 return rawTraceFileName.substring(0, lastIndex + 1); 548 } 549 return rawTraceFileName; 550 } 551 552 /** 553 * Retrieve the current build info. 554 * 555 * @return BuildInfo which has access to test artifacts directory. 556 */ 557 @VisibleForTesting getCurrentBuildInfo()558 IBuildInfo getCurrentBuildInfo() { 559 return getBuildInfos().get(0); 560 } 561 562 /** 563 * Compress the given file. 564 * 565 * @return File compressed version of the file. 566 */ 567 @VisibleForTesting getCompressedFile(File metricOutputFile)568 File getCompressedFile(File metricOutputFile) throws IOException { 569 return ZipUtil.createZip(metricOutputFile, 570 metricOutputFile.getName()); 571 } 572 } 573