1 /* 2 * Copyright (C) 2016 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package com.android.tradefed.testtype; 17 18 import com.android.tradefed.build.IFolderBuildInfo; 19 import com.android.tradefed.config.Option; 20 import com.android.tradefed.log.LogUtil.CLog; 21 import com.android.tradefed.metrics.proto.MetricMeasurement.Metric; 22 import com.android.tradefed.result.FileInputStreamSource; 23 import com.android.tradefed.result.ITestInvocationListener; 24 import com.android.tradefed.result.InputStreamSource; 25 import com.android.tradefed.result.LogDataType; 26 import com.android.tradefed.result.TestDescription; 27 import com.android.tradefed.util.CommandResult; 28 import com.android.tradefed.util.CommandStatus; 29 import com.android.tradefed.util.FileUtil; 30 import com.android.tradefed.util.HprofAllocSiteParser; 31 import com.android.tradefed.util.RunUtil; 32 import com.android.tradefed.util.StreamUtil; 33 import com.android.tradefed.util.SystemUtil.EnvVariable; 34 import com.android.tradefed.util.TarUtil; 35 import com.android.tradefed.util.proto.TfMetricProtoUtil; 36 37 import com.google.common.annotations.VisibleForTesting; 38 39 import java.io.File; 40 import java.io.IOException; 41 import java.io.InputStream; 42 import java.util.ArrayList; 43 import java.util.Arrays; 44 import java.util.HashMap; 45 import java.util.LinkedHashSet; 46 import java.util.List; 47 import java.util.Map; 48 import java.util.Set; 49 import java.util.regex.Pattern; 50 51 /** 52 * A {@link IRemoteTest} for running unit or functional tests against a separate TF installation. 53 * <p/> 54 * Launches an external java process to run the tests. Used for running the TF unit or 55 * functional tests continuously. 56 */ 57 public class TfTestLauncher extends SubprocessTfLauncher { 58 59 private static final long COVERAGE_REPORT_TIMEOUT_MS = 5 * 60 * 1000; 60 61 @Option(name = "jacoco-code-coverage", description = "Enable jacoco code coverage on the java " 62 + "sub process. Run will be slightly slower because of the overhead.") 63 private boolean mEnableCoverage = false; 64 65 @Option(name = "include-coverage", description = "Patterns to include in the code coverage.") 66 private Set<String> mIncludeCoverage = new LinkedHashSet<>(); 67 68 @Option(name = "exclude-coverage", description = "Patterns to exclude in the code coverage.") 69 private Set<String> mExcludeCoverage = new LinkedHashSet<>(); 70 71 @Option( 72 name = "hprof-heap-memory", 73 description = 74 "Enable hprof agent while running the java" 75 + "sub process. Run will be slightly slower because of the overhead." 76 ) 77 private boolean mEnableHprof = false; 78 79 @Option(name = "ant-config-res", description = "The name of the ant resource configuration to " 80 + "transform the results in readable format.") 81 private String mAntConfigResource = "/jacoco/ant-tf-coverage.xml"; 82 83 @Option(name = "sub-branch", description = "The branch to be provided to the sub invocation, " 84 + "if null, the branch in build info will be used.") 85 private String mSubBranch = null; 86 87 @Option(name = "sub-build-flavor", description = "The build flavor to be provided to the " 88 + "sub invocation, if null, the build flavor in build info will be used.") 89 private String mSubBuildFlavor = null; 90 91 @Option(name = "sub-build-id", description = "The build id that the sub invocation will try " 92 + "to use in case where it needs its own device.") 93 private String mSubBuildId = null; 94 95 @Option(name = "use-virtual-device", description = 96 "Flag if the subprocess is going to need to instantiate a virtual device to run.") 97 private boolean mUseVirtualDevice = false; 98 99 @Option(name = "sub-apk-path", description = "The name of all the Apks that needs to be " 100 + "installed by the subprocess invocation. Apk need to be inside the downloaded zip. " 101 + "Can be repeated.") 102 private List<String> mSubApkPath = new ArrayList<String>(); 103 104 // The regex pattern of temp files to be found in the temporary dir of the subprocess. 105 // Any file not matching the patterns, or multiple files in the temporary dir match the same 106 // pattern, is considered as test failure. 107 private static final String[] EXPECTED_TMP_FILE_PATTERNS = { 108 "inv_.*", "tradefed_global_log_.*", "lc_cache", "stage-android-build-api", 109 }; 110 111 // A destination file where the report will be put. 112 private File mDestCoverageFile = null; 113 // A destination file where the hprof report will be put. 114 private File mHprofFile = null; 115 // A {@link File} pointing to the jacoco args jar file extracted from the resources 116 private File mAgent = null; 117 118 /** {@inheritDoc} */ 119 @Override addJavaArguments(List<String> args)120 protected void addJavaArguments(List<String> args) { 121 super.addJavaArguments(args); 122 try { 123 if (mEnableCoverage) { 124 mDestCoverageFile = FileUtil.createTempFile("coverage", ".exec"); 125 mAgent = extractJacocoAgent(); 126 addCoverageArgs(mAgent, args, mDestCoverageFile); 127 } 128 if (mEnableHprof) { 129 mHprofFile = FileUtil.createTempFile("java.hprof", ".txt"); 130 // verbose=n to avoid dump in stderr 131 // cutoff the min value we look at. 132 String hprofAgent = 133 String.format( 134 "-agentlib:hprof=heap=sites,cutoff=0.01,depth=16,verbose=n,file=%s", 135 mHprofFile.getAbsolutePath()); 136 args.add(hprofAgent); 137 } 138 } catch (IOException e) { 139 throw new RuntimeException(e); 140 } 141 } 142 143 /** {@inheritDoc} */ 144 @Override preRun()145 protected void preRun() { 146 super.preRun(); 147 148 if (!mUseVirtualDevice) { 149 mCmdArgs.add("-n"); 150 } else { 151 // if it needs a device we also enable more logs 152 mCmdArgs.add("--log-level"); 153 mCmdArgs.add("VERBOSE"); 154 mCmdArgs.add("--log-level-display"); 155 mCmdArgs.add("VERBOSE"); 156 } 157 mCmdArgs.add("--test-tag"); 158 mCmdArgs.add(mBuildInfo.getTestTag()); 159 mCmdArgs.add("--build-id"); 160 if (mSubBuildId != null) { 161 mCmdArgs.add(mSubBuildId); 162 } else { 163 mCmdArgs.add(mBuildInfo.getBuildId()); 164 } 165 mCmdArgs.add("--branch"); 166 if (mSubBranch != null) { 167 mCmdArgs.add(mSubBranch); 168 } else if (mBuildInfo.getBuildBranch() != null) { 169 mCmdArgs.add(mBuildInfo.getBuildBranch()); 170 } else { 171 throw new RuntimeException("Branch option is required for the sub invocation."); 172 } 173 mCmdArgs.add("--build-flavor"); 174 if (mSubBuildFlavor != null) { 175 mCmdArgs.add(mSubBuildFlavor); 176 } else if (mBuildInfo.getBuildFlavor() != null) { 177 mCmdArgs.add(mBuildInfo.getBuildFlavor()); 178 } else { 179 throw new RuntimeException("Build flavor option is required for the sub invocation."); 180 } 181 182 for (String apk : mSubApkPath) { 183 mCmdArgs.add("--apk-path"); 184 String apkPath = 185 String.format( 186 "%s%s%s", 187 ((IFolderBuildInfo) mBuildInfo).getRootDir().getAbsolutePath(), 188 File.separator, 189 apk); 190 mCmdArgs.add(apkPath); 191 } 192 // Unset potential build environment to ensure they do not affect the unit tests 193 getRunUtil().unsetEnvVariable(EnvVariable.ANDROID_HOST_OUT_TESTCASES.name()); 194 getRunUtil().unsetEnvVariable(EnvVariable.ANDROID_TARGET_OUT_TESTCASES.name()); 195 } 196 197 /** {@inheritDoc} */ 198 @Override postRun(ITestInvocationListener listener, boolean exception, long elapsedTime)199 protected void postRun(ITestInvocationListener listener, boolean exception, long elapsedTime) { 200 super.postRun(listener, exception, elapsedTime); 201 reportMetrics(elapsedTime, listener); 202 FileUtil.deleteFile(mAgent); 203 204 // Evaluate coverage from the subprocess 205 if (mEnableCoverage) { 206 InputStreamSource coverage = null; 207 File xmlResult = null; 208 try { 209 xmlResult = processExecData(mDestCoverageFile, mRootDir); 210 coverage = new FileInputStreamSource(xmlResult); 211 listener.testLog("coverage_xml", LogDataType.JACOCO_XML, coverage); 212 } catch (IOException e) { 213 if (exception) { 214 // If exception was thrown above, we only log this one since it's most 215 // likely related to it. 216 CLog.e(e); 217 } else { 218 throw new RuntimeException(e); 219 } 220 } finally { 221 FileUtil.deleteFile(mDestCoverageFile); 222 StreamUtil.cancel(coverage); 223 FileUtil.deleteFile(xmlResult); 224 } 225 } 226 if (mEnableHprof) { 227 logHprofResults(mHprofFile, listener); 228 } 229 230 if (mTmpDir != null) { 231 testTmpDirClean(mTmpDir, listener); 232 } 233 cleanTmpFile(); 234 } 235 236 @VisibleForTesting cleanTmpFile()237 void cleanTmpFile() { 238 FileUtil.deleteFile(mHprofFile); 239 FileUtil.deleteFile(mDestCoverageFile); 240 FileUtil.deleteFile(mAgent); 241 } 242 243 /** 244 * Helper to add arguments required for code coverage collection. 245 * 246 * @param jacocoAgent the jacoco args file to run the coverage. 247 * @param args list of arguments that will be run in the subprocess. 248 * @param destfile destination file where the report will be put. 249 */ addCoverageArgs(File jacocoAgent, List<String> args, File destfile)250 private void addCoverageArgs(File jacocoAgent, List<String> args, File destfile) { 251 if (mIncludeCoverage.isEmpty() && mExcludeCoverage.isEmpty()) { 252 mIncludeCoverage.add("com.android.tradefed*"); 253 mIncludeCoverage.add("com.google.android.tradefed*"); 254 } 255 String includeFilter = String.join(":", mIncludeCoverage); 256 String javaagent = 257 String.format( 258 "-javaagent:%s=destfile=%s," + "includes=%s", 259 jacocoAgent.getAbsolutePath(), destfile.getAbsolutePath(), includeFilter); 260 if (!mExcludeCoverage.isEmpty()) { 261 String excludeFilter = String.join(":", mExcludeCoverage); 262 javaagent += ",excludes=" + excludeFilter; 263 } 264 args.add(javaagent); 265 } 266 267 /** 268 * Returns a {@link File} pointing to the jacoco args jar file extracted from the resources. 269 */ extractJacocoAgent()270 private File extractJacocoAgent() throws IOException { 271 String jacocoAgentRes = "/jacoco/jacocoagent.jar"; 272 InputStream jacocoAgentStream = getClass().getResourceAsStream(jacocoAgentRes); 273 if (jacocoAgentStream == null) { 274 throw new IOException("Could not find " + jacocoAgentRes); 275 } 276 File jacocoAgent = FileUtil.createTempFile("jacocoagent", ".jar"); 277 FileUtil.writeToFile(jacocoAgentStream, jacocoAgent); 278 return jacocoAgent; 279 } 280 281 /** 282 * Helper to process the execution data into user readable format (xml) that can easily be 283 * parsed. 284 * 285 * @param executionData output files of the java args jacoco. 286 * @param rootDir base directory of downloaded TF 287 * @return a {@link File} pointing to the human readable xml result file. 288 */ processExecData(File executionData, String rootDir)289 private File processExecData(File executionData, String rootDir) throws IOException { 290 File xmlReport = FileUtil.createTempFile("coverage_xml", ".xml"); 291 InputStream template = getClass().getResourceAsStream(mAntConfigResource); 292 if (template == null) { 293 throw new IOException("Could not find " + mAntConfigResource); 294 } 295 String jacocoAntRes = "/jacoco/jacocoant.jar"; 296 InputStream jacocoAntStream = getClass().getResourceAsStream(jacocoAntRes); 297 if (jacocoAntStream == null) { 298 throw new IOException("Could not find " + jacocoAntRes); 299 } 300 File antConfig = FileUtil.createTempFile("ant-merge_", ".xml"); 301 File jacocoAnt = FileUtil.createTempFile("jacocoant", ".jar"); 302 try { 303 FileUtil.writeToFile(template, antConfig); 304 FileUtil.writeToFile(jacocoAntStream, jacocoAnt); 305 String[] cmd = {"ant", "-f", antConfig.getPath(), 306 "-Djacocoant.path=" + jacocoAnt.getAbsolutePath(), 307 "-Dexecution.files=" + executionData.getAbsolutePath(), 308 "-Droot.dir=" + rootDir, 309 "-Ddest.file=" + xmlReport.getAbsolutePath()}; 310 CommandResult result = RunUtil.getDefault().runTimedCmd(COVERAGE_REPORT_TIMEOUT_MS, 311 cmd); 312 CLog.d(result.getStdout()); 313 if (!CommandStatus.SUCCESS.equals(result.getStatus())) { 314 throw new IOException(result.getStderr()); 315 } 316 return xmlReport; 317 } finally { 318 FileUtil.deleteFile(antConfig); 319 FileUtil.deleteFile(jacocoAnt); 320 } 321 } 322 323 /** 324 * Report an elapsed-time metric to keep track of it. 325 * 326 * @param elapsedTime time it took the subprocess to run. 327 * @param listener the {@link ITestInvocationListener} where to report the metric. 328 */ reportMetrics(long elapsedTime, ITestInvocationListener listener)329 private void reportMetrics(long elapsedTime, ITestInvocationListener listener) { 330 if (elapsedTime == -1l) { 331 return; 332 } 333 listener.testRunStarted("elapsed-time", 1); 334 TestDescription tid = new TestDescription("elapsed-time", "run-elapsed-time"); 335 listener.testStarted(tid); 336 HashMap<String, Metric> runMetrics = new HashMap<>(); 337 runMetrics.put( 338 "elapsed-time", TfMetricProtoUtil.stringToMetric(Long.toString(elapsedTime))); 339 listener.testEnded(tid, runMetrics); 340 listener.testRunEnded(0L, runMetrics); 341 } 342 343 /** 344 * Extra test to ensure no files are created by the unit tests in the subprocess and not 345 * cleaned. 346 * 347 * @param tmpDir the temporary dir of the subprocess. 348 * @param listener the {@link ITestInvocationListener} where to report the test. 349 */ 350 @VisibleForTesting testTmpDirClean(File tmpDir, ITestInvocationListener listener)351 protected void testTmpDirClean(File tmpDir, ITestInvocationListener listener) { 352 listener.testRunStarted("temporaryFiles", 1); 353 TestDescription tid = new TestDescription("temporary-files", "testIfClean"); 354 listener.testStarted(tid); 355 String[] listFiles = tmpDir.list(); 356 List<String> unmatchedFiles = new ArrayList<String>(); 357 List<String> patterns = new ArrayList<String>(Arrays.asList(EXPECTED_TMP_FILE_PATTERNS)); 358 patterns.add(mBuildInfo.getBuildBranch()); 359 for (String file : Arrays.asList(listFiles)) { 360 Boolean matchFound = false; 361 for (String pattern : patterns) { 362 if (Pattern.matches(pattern, file)) { 363 patterns.remove(pattern); 364 matchFound = true; 365 break; 366 } 367 } 368 if (!matchFound) { 369 unmatchedFiles.add(file); 370 } 371 } 372 if (unmatchedFiles.size() > 0) { 373 String trace = 374 String.format( 375 "Found '%d' unexpected temporary files: %s.\nOnly " 376 + "expected files are: %s. And each should appears only once.", 377 unmatchedFiles.size(), unmatchedFiles, patterns); 378 listener.testFailed(tid, trace); 379 } 380 listener.testEnded(tid, new HashMap<String, Metric>()); 381 listener.testRunEnded(0, new HashMap<String, Metric>()); 382 } 383 384 /** 385 * Helper to log and report as metric the hprof data. 386 * 387 * @param hprofFile file containing the Hprof report 388 * @param listener the {@link ITestInvocationListener} where to report the test. 389 */ logHprofResults(File hprofFile, ITestInvocationListener listener)390 private void logHprofResults(File hprofFile, ITestInvocationListener listener) { 391 if (hprofFile == null) { 392 CLog.w("Hprof file was null. Skipping parsing."); 393 return; 394 } 395 if (!hprofFile.exists()) { 396 CLog.w("Hprof file %s was not found. Skipping parsing.", hprofFile.getAbsolutePath()); 397 return; 398 } 399 InputStreamSource memory = null; 400 File tmpGzip = null; 401 try { 402 tmpGzip = TarUtil.gzip(hprofFile); 403 memory = new FileInputStreamSource(tmpGzip); 404 listener.testLog("hprof", LogDataType.GZIP, memory); 405 } catch (IOException e) { 406 CLog.e(e); 407 return; 408 } finally { 409 StreamUtil.cancel(memory); 410 FileUtil.deleteFile(tmpGzip); 411 } 412 HprofAllocSiteParser parser = new HprofAllocSiteParser(); 413 try { 414 Map<String, String> results = parser.parse(hprofFile); 415 if (results.isEmpty()) { 416 CLog.d("No allocation site found from hprof file"); 417 return; 418 } 419 listener.testRunStarted("hprofAllocSites", 1); 420 TestDescription tid = new TestDescription("hprof", "allocationSites"); 421 listener.testStarted(tid); 422 listener.testEnded(tid, TfMetricProtoUtil.upgradeConvert(results)); 423 listener.testRunEnded(0, new HashMap<String, Metric>()); 424 } catch (IOException e) { 425 throw new RuntimeException(e); 426 } 427 } 428 } 429