1 /* 2 * Copyright (C) 2017 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package com.android.tradefed.testtype; 18 19 import com.android.annotations.VisibleForTesting; 20 import com.android.compatibility.common.tradefed.build.CompatibilityBuildHelper; 21 import com.android.tradefed.build.IBuildInfo; 22 import com.android.tradefed.config.Option; 23 import com.android.tradefed.config.OptionClass; 24 import com.android.tradefed.device.DeviceNotAvailableException; 25 import com.android.tradefed.device.ITestDevice; 26 import com.android.tradefed.invoker.IInvocationContext; 27 import com.android.tradefed.log.LogUtil.CLog; 28 import com.android.tradefed.metrics.proto.MetricMeasurement.Metric; 29 import com.android.tradefed.result.FileInputStreamSource; 30 import com.android.tradefed.result.ITestInvocationListener; 31 import com.android.tradefed.result.LogDataType; 32 import com.android.tradefed.result.TestDescription; 33 import com.android.tradefed.targetprep.VtsCoveragePreparer; 34 import com.android.tradefed.targetprep.VtsPythonVirtualenvPreparer; 35 import com.android.tradefed.util.CommandResult; 36 import com.android.tradefed.util.CommandStatus; 37 import com.android.tradefed.util.FileUtil; 38 import com.android.tradefed.util.JsonUtil; 39 import com.android.tradefed.util.OutputUtil; 40 import com.android.tradefed.util.StreamUtil; 41 import com.android.tradefed.util.VtsDashboardUtil; 42 import com.android.tradefed.util.VtsPythonRunnerHelper; 43 import com.android.tradefed.util.VtsVendorConfigFileUtil; 44 import java.io.BufferedWriter; 45 import java.io.File; 46 import java.io.FileNotFoundException; 47 import java.io.FileOutputStream; 48 import java.io.FileWriter; 49 import java.io.IOException; 50 import java.io.OutputStream; 51 import java.io.PrintWriter; 52 import java.nio.file.Paths; 53 import java.util.ArrayList; 54 import java.util.Arrays; 55 import java.util.Collection; 56 import java.util.HashMap; 57 import java.util.List; 58 import java.util.Set; 59 import java.util.TreeMap; 60 import java.util.TreeSet; 61 import org.json.JSONArray; 62 import org.json.JSONException; 63 import org.json.JSONObject; 64 65 /** 66 * A Test that runs a vts multi device test package (part of Vendor Test Suite, VTS10) on given 67 * device.<p> 68 * TODO: Complete unit tests 69 */ 70 @OptionClass(alias = "vtsmultidevicetest") 71 public class VtsMultiDeviceTest 72 implements IDeviceTest, IRemoteTest, ITestFilterReceiver, IRuntimeHintProvider, 73 ITestCollector, IBuildReceiver, IAbiReceiver, IInvocationContextReceiver { 74 static final String ACTS_TEST_MODULE = "ACTS_TEST_MODULE"; 75 static final String ADAPTER_ACTS_PATH = "vts/runners/adapters/acts/acts_adapter"; 76 static final String ANDROIDDEVICE = "AndroidDevice"; 77 static final String BUILD = "build"; 78 static final String BUILD_ID = "build_id"; 79 static final String BUILD_TARGET = "build_target"; 80 static final String COVERAGE_PROPERTY = "ro.vts.coverage"; 81 static final String DATA_FILE_PATH = "data_file_path"; 82 static final String LOG_PATH = "log_path"; 83 static final String LOG_SEVERITY = "log_severity"; 84 static final String NAME = "name"; 85 static final String SERIAL = "serial"; 86 static final String TESTMODULE = "TestModule"; 87 static final String TEST_BED = "test_bed"; 88 static final String TEST_PLAN_REPORT_FILE = "TEST_PLAN_REPORT_FILE"; 89 static final String TEST_SUITE = "test_suite"; 90 static final String TEST_TIMEOUT = "test_timeout"; 91 static final String ABI_NAME = "abi_name"; 92 static final String ABI_BITNESS = "abi_bitness"; 93 static final String SKIP_ON_32BIT_ABI = "skip_on_32bit_abi"; 94 static final String SKIP_ON_64BIT_ABI = "skip_on_64bit_abi"; 95 static final String SHELL_DEFAULT_NOHUP = "shell_default_nohup"; 96 static final String SKIP_IF_THERMAL_THROTTLING = "skip_if_thermal_throttling"; 97 static final String DISABLE_CPU_FREQUENCY_SCALING = "disable_cpu_frequency_scaling"; 98 static final String DISABLE_FRAMEWORK = "DISABLE_FRAMEWORK"; 99 static final String STOP_NATIVE_SERVERS = "STOP_NATIVE_SERVERS"; 100 static final String RUN_32BIT_ON_64BIT_ABI = "run_32bit_on_64bit_abi"; 101 static final String CONFIG_FILE_EXTENSION = ".config"; 102 static final String INCLUDE_FILTER = "include_filter"; 103 static final String EXCLUDE_FILTER = "exclude_filter"; 104 static final String EXCLUDE_OVER_INCLUDE = "exclude_over_include"; 105 static final String BINARY_TEST_SOURCE = "binary_test_source"; 106 static final String BINARY_TEST_WORKING_DIRECTORY = "binary_test_working_directory"; 107 static final String BINARY_TEST_ENVP = "binary_test_envp"; 108 static final String BINARY_TEST_ARGS = "binary_test_args"; 109 static final String BINARY_TEST_LD_LIBRARY_PATH = "binary_test_ld_library_path"; 110 static final String BINARY_TEST_PROFILING_LIBRARY_PATH = "binary_test_profiling_library_path"; 111 @Deprecated static final String BINARY_TEST_DISABLE_FRAMEWORK = "binary_test_disable_framework"; 112 @Deprecated 113 static final String BINARY_TEST_STOP_NATIVE_SERVERS = "binary_test_stop_native_servers"; 114 static final String BINARY_TEST_TYPE_GTEST = "gtest"; 115 static final String BINARY_TEST_TYPE_LLVMFUZZER = "llvmfuzzer"; 116 static final String BINARY_TEST_TYPE_HAL_HIDL_GTEST = "hal_hidl_gtest"; 117 static final String BINARY_TEST_TYPE_HAL_HIDL_REPLAY_TEST = "hal_hidl_replay_test"; 118 static final String BINARY_TEST_TYPE_HOST_BINARY_TEST = "host_binary_test"; 119 static final String BUG_REPORT_ON_FAILURE = "BUG_REPORT_ON_FAILURE"; 120 static final String COLLECT_TESTS_ONLY = "collect_tests_only"; 121 static final String CONFIG_STR = "CONFIG_STR"; 122 static final String CONFIG_INT = "CONFIG_INT"; 123 static final String CONFIG_BOOL = "CONFIG_BOOL"; 124 static final String LOGCAT_ON_FAILURE = "LOGCAT_ON_FAILURE"; 125 static final String ENABLE_COVERAGE = "enable_coverage"; 126 static final String EXCLUDE_COVERAGE_PATH = "exclude_coverage_path"; 127 static final String ENABLE_LOG_UPLOADING = "enable_log_uploading"; 128 static final String ENABLE_PROFILING = "enable_profiling"; 129 static final String PROFILING_ARG_VALUE = "profiling_arg_value"; 130 static final String ENABLE_SANCOV = "enable_sancov"; 131 static final String GTEST_BATCH_MODE = "gtest_batch_mode"; 132 static final String SAVE_TRACE_FIEL_REMOTE = "save_trace_file_remote"; 133 static final String OUTPUT_COVERAGE_REPORT = "output_coverage_report"; 134 static final String COVERAGE_REPORT_PATH = "coverage_report_path"; 135 static final String GLOBAL_COVERAGE = "global_coverage"; 136 static final String LTP_NUMBER_OF_THREADS = "ltp_number_of_threads"; 137 static final String MAX_RETRY_COUNT = "max_retry_count"; 138 static final String MOBLY_TEST_MODULE = "MOBLY_TEST_MODULE"; 139 static final String NATIVE_SERVER_PROCESS_NAME = "native_server_process_name"; 140 static final String PASSTHROUGH_MODE = "passthrough_mode"; 141 static final String PRECONDITION_HWBINDER_SERVICE = "precondition_hwbinder_service"; 142 static final String PRECONDITION_FEATURE = "precondition_feature"; 143 static final String PRECONDITION_FILE_PATH_PREFIX = "precondition_file_path_prefix"; 144 static final String PRECONDITION_FIRST_API_LEVEL = "precondition_first_api_level"; 145 static final String PRECONDITION_LSHAL = "precondition_lshal"; 146 static final String PRECONDITION_SYSPROP = "precondition_sysprop"; 147 static final String PRECONDITION_VINTF = "precondition_vintf"; 148 static final String ENABLE_SYSTRACE = "enable_systrace"; 149 static final String HAL_HIDL_REPLAY_TEST_TRACE_PATHS = "hal_hidl_replay_test_trace_paths"; 150 static final String HAL_HIDL_PACKAGE_NAME = "hal_hidl_package_name"; 151 static final String REPORT_MESSAGE_FILE_NAME = "report_proto.msg"; 152 static final String RUN_AS_VTS_SELF_TEST = "run_as_vts_self_test"; 153 static final String RUN_AS_COMPLIANCE_TEST = "run_as_compliance_test"; 154 static final String SYSTRACE_PROCESS_NAME = "systrace_process_name"; 155 static final String TEMPLATE_BINARY_TEST_PATH = "vts/testcases/template/binary_test/binary_test"; 156 static final String TEMPLATE_GTEST_BINARY_TEST_PATH = "vts/testcases/template/gtest_binary_test/gtest_binary_test"; 157 static final String TEMPLATE_LLVMFUZZER_TEST_PATH = "vts/testcases/template/llvmfuzzer_test/llvmfuzzer_test"; 158 static final String TEMPLATE_MOBLY_TEST_PATH = "vts/testcases/template/mobly/mobly_test"; 159 static final String TEMPLATE_HAL_HIDL_GTEST_PATH = "vts/testcases/template/hal_hidl_gtest/hal_hidl_gtest"; 160 static final String TEMPLATE_HAL_HIDL_REPLAY_TEST_PATH = "vts/testcases/template/hal_hidl_replay_test/hal_hidl_replay_test"; 161 static final String TEMPLATE_HOST_BINARY_TEST_PATH = "vts/testcases/template/host_binary_test/host_binary_test"; 162 static final String TEST_RUN_SUMMARY_FILE_NAME = "test_run_summary.json"; 163 static final float DEFAULT_TARGET_VERSION = -1; 164 static final String DEFAULT_TESTCASE_CONFIG_PATH = 165 "vts/tools/vts-tradefed/res/default/DefaultTestCase.runner_conf"; 166 167 private ITestDevice mDevice = null; 168 private IAbi mAbi = null; 169 170 @Option(name = "test-timeout", 171 description = "The amount of time (in milliseconds) for a test invocation. " 172 + "If the test cannot finish before timeout, it is interrupted. As some " 173 + "classes generate test cases during setup, they can use the given timeout " 174 + "value for each generated test set.", 175 isTimeVal = true) 176 private long mTestTimeout = 1000 * 60 * 3; 177 178 @Option(name = "max-test-timeout", 179 description = "The maximum amount of time (in milliseconds) for a test invocation. " 180 + "This timeout value doesn't change with number of generated test sets.", 181 isTimeVal = true) 182 private long mMaxTestTimeout = 1000 * 60 * 60 * 3; 183 184 @Option(name = "test-module-name", 185 description = "The name for a test module.") 186 private String mTestModuleName = null; 187 188 @Option(name = "test-case-path", 189 description = "The path for test case.") 190 private String mTestCasePath = null; 191 192 @Option(name = "test-case-path-type", 193 description = "The type of test case path ('module' by default or 'file').") 194 private String mTestCasePathType = null; 195 196 @Option(name = "test-config-path", 197 description = "The path for test case config file.") 198 private String mTestConfigPath = null; 199 200 @Option(name = "precondition-hwbinder-service", 201 description = "The name of a HW binder service needed to run the test.") 202 private String mPreconditionHwBinderServiceName = null; 203 204 @Option(name = "precondition-feature", 205 description = "The name of a `pm`-listable feature needed to run the test.") 206 private String mPreconditionFeature = null; 207 208 @Option(name = "precondition-file-path-prefix", 209 description = "The path prefix of a target-side file needed to run the test." 210 + "Format of tags:" 211 + " <source>: source without tag." 212 + " <tag>::<source>: <tag> specifies bitness of testcase: _32bit or _64bit" 213 + " Note: multiple sources are ANDed" 214 + "Format of each source string:" 215 + " <source>: absolute path of file prefix on device") 216 private Collection<String> mPreconditionFilePathPrefix = new ArrayList<>(); 217 218 @Option(name = "precondition-first-api-level", 219 description = "The lowest first API level required to run the test.") 220 private int mPreconditionFirstApiLevel = 0; 221 222 @Option(name = "precondition-lshal", 223 description = "The name of a `lshal`-listable feature needed to run the test.") 224 private String mPreconditionLshal = null; 225 226 @Option(name = "precondition-sysprop", 227 description = "The name=value for a system property configuration that needs " 228 + "to be met to run the test.") 229 private String mPreconditionSysProp = null; 230 231 @Option(name = "precondition-vintf", 232 description = "The full name of a HAL specified in vendor/manifest.xml and " 233 + "needed to run the test (e.g., android.hardware.graphics.mapper@2.0). " 234 + "this can override precondition-lshal option.") 235 private String mPreconditionVintf = null; 236 237 @Option(name = "enable-dashboard-uploading", 238 description = "Enables the runner's dashboard result uploading feature.") 239 private boolean mEnableDashboardUploading = true; 240 241 @Option(name = "enable-log-uploading", 242 description = "Enables the runner's log uploading feature.") 243 private boolean mEnableLogUploading = false; 244 245 @Option(name = "include-filter", 246 description = "The positive filter of the test names to run.") 247 private Set<String> mIncludeFilters = new TreeSet<>(); 248 249 @Option(name = "exclude-filter", 250 description = "The negative filter of the test names to run.") 251 private Set<String> mExcludeFilters = new TreeSet<>(); 252 253 @Option(name = "exclude-over-include", 254 description = "The negative filter of the test names to run.") 255 private boolean mExcludeOverInclude = false; 256 257 @Option(name = "runtime-hint", description = "The hint about the test's runtime.", 258 isTimeVal = true) 259 private long mRuntimeHint = 60000; // 1 minute 260 261 @Option(name = "enable-profiling", description = "Enable profiling for the tests.") 262 private boolean mEnableProfiling = false; 263 264 @Option(name = "profiling-arg-value", description = "Whether to profile for arg value.") 265 private boolean mProfilingArgValue = false; 266 267 @Option(name = "save-trace-file-remote", 268 description = "Whether to save the trace file in remote storage.") 269 private boolean mSaveTraceFileRemote = false; 270 271 @Option(name = "enable-systrace", description = "Enable systrace for the tests.") 272 private boolean mEnableSystrace = false; 273 274 @Option(name = "enable-coverage", 275 description = "Enable coverage for the tests. In order for coverage to be measured, " + 276 "ro.vts.coverage system must have value \"1\" to indicate the target " + 277 "build is coverage instrumented.") 278 private boolean mEnableCoverage = true; 279 280 @Option(name = "global-coverage", description = "True to measure coverage for entire test, " 281 + "measure coverage for each test case otherwise. Currently, only global " 282 + "coverage is supported for binary tests") 283 private boolean mGlobalCoverage = true; 284 285 @Option(name = "enable-sancov", 286 description = "Enable sanitizer coverage for the tests. In order for coverage to be " 287 + "measured, the device must be a sancov build with its build info and " 288 + "unstripped binaries available to the sancov preparer class.") 289 private boolean mEnableSancov = true; 290 291 @Option(name = "output-coverage-report", description = "Whether to store raw coverage report.") 292 private boolean mOutputCoverageReport = false; 293 294 // Another design option is to parse a string or use enum for host preference on BINDER, 295 // PASSTHROUGH and DEFAULT(which is BINDER). Also in the future, we might want to deal with 296 // the case of target preference on PASSTHROUGH (if host does not specify to use BINDER mode). 297 @Option(name = "passthrough-mode", description = "Set getStub to use passthrough mode. " 298 + "Value true means use passthrough mode if available; false for binderized mode if " 299 + "available. Default is false") 300 private boolean mPassthroughMode = false; 301 302 @Option(name = "ltp-number-of-threads", 303 description = "Number of threads to run the LTP test cases. " 304 + "0 means using number of avaiable CPU threads.") 305 private int mLtpNumberOfThreads = -1; 306 307 @Option(name = "shell-default-nohup", 308 description = "Whether to by default use nohup for shell commands.") 309 private boolean mShellDefaultNohup = false; 310 311 @Option(name = "skip-on-32bit-abi", description = "Whether to skip tests on 32bit ABI.") 312 private boolean mSkipOn32BitAbi = false; 313 314 @Option(name = "skip-on-64bit-abi", description = "Whether to skip tests on 64bit ABI.") 315 private boolean mSkipOn64BitAbi = false; 316 317 @Option(name = "skip-if-thermal-throttling", 318 description = "Whether to skip tests if target device suffers from thermal throttling.") 319 private boolean mSkipIfThermalThrottling = false; 320 321 @Option(name = "disable-cpu-frequency-scaling", 322 description = "Whether to disable cpu frequency scaling for test.") 323 private boolean mDisableCpuFrequencyScaling = true; 324 325 @Option(name = "run-32bit-on-64bit-abi", 326 description = "Whether to run 32bit tests on 64bit ABI.") 327 private boolean mRun32bBitOn64BitAbi = false; 328 329 @Option(name = "binary-test-source", 330 description = "Binary test source paths relative to vts testcase directory on host." 331 + "Format of tags:" 332 + " <source>: source without tag." 333 + " <tag>::<source>: source with tag. Can be used to separate 32bit and 64" 334 + " bit tests with same file name." 335 + " <tag1>::<source1>, <tag1>::<source2>, <tag2>::<source3>: multiple" 336 + " sources connected by comma. White spaces in-between" 337 + " will be ignored." 338 + "Format of each source string:" 339 + " <source file>: push file and create test case." 340 + " Source is relative path of source file under vts's testcases" 341 + " folder. Source file will be pushed to working directory" 342 + " discarding original directory structure, and test cases will" 343 + " be created using the pushed file." 344 + " <source file>-><destination file>: push file and create test case." 345 + " Destination path is absolute path on device. Test cases will" 346 + " be created using the pushed file." 347 + " <source file>->: push file only." 348 + " Push the source file to its' tag's corresponding" 349 + " working directory. Test case will not be created on" 350 + " this file. This is equivalent to but simpler than specifying a" 351 + " working directory for the tag and use VtsFilePusher to push the" 352 + " file to the directory." 353 + " -><destination file>: create test case only." 354 + " Destination is absolute device side path." 355 + " Note: each path in source string can be a directory. However, the" 356 + " default binary test runner and gtest binary test runner does not" 357 + " support creating test cases from a directory. You will need to" 358 + " override the binary test runner's CreateTestCase method in python." 359 + " If you wish to push a source file to a specific destination and not" 360 + " create a test case from it, please use VtsFilePusher.") 361 private Collection<String> mBinaryTestSource = new ArrayList<>(); 362 363 @Option(name = "binary-test-working-directory", description = "Working directories for binary " 364 + "tests. Tags can be added to the front of each directory using '::' as delimiter. " 365 + "However, each tag should only has one working directory. This option is optional for " 366 + "binary tests. If not specified, different directories will be used for files with " 367 + "different tags.") 368 private Collection<String> mBinaryTestWorkingDirectory = new ArrayList<>(); 369 370 @Option(name = "binary-test-envp", description = "Additional environment path for binary " 371 + "tests. Tags can be added to the front of each directory using '::' as delimiter. " 372 + "There can be multiple instances of binary-test-envp for a same tag, which will " 373 + "later automatically be combined.") 374 private Collection<String> mBinaryTestEnvp = new ArrayList<>(); 375 376 @Option(name = "binary-test-args", description = "Additional args or flags for binary " 377 + "tests. Tags can be added to the front of each directory using '::' as delimiter. " 378 + "There can be multiple instances of binary-test-args for a same tag, which will " 379 + "later automatically be combined.") 380 private Collection<String> mBinaryTestArgs = new ArrayList<>(); 381 382 @Option(name = "binary-test-ld-library-path", description = "LD_LIBRARY_PATH for binary " 383 + "tests. Tags can be added to the front of each instance using '::' as delimiter. " 384 + "Multiple directories can be added under a same tag using ':' as delimiter. " 385 + "There can be multiple instances of ld-library-path for a same tag, which will " 386 + "later automatically be combined using ':' as delimiter. Paths without a tag " 387 + "will only used for binaries without tag. This option is optional for binary tests.") 388 private Collection<String> mBinaryTestLdLibraryPath = new ArrayList<>(); 389 390 @Option(name = "binary-test-profiling-library-path", description = "Path to lookup and load " 391 + "profiling libraries for tests with profiling enabled. Tags can be added to the " 392 + "front of each directory using '::' as delimiter. Only one directory could be " 393 + "specified for the same tag. This option is optional for binary tests. If not " 394 + "specified, default directories will be used for files with different tags.") 395 private Collection<String> mBinaryTestProfilingLibraryPath = new ArrayList<>(); 396 397 @Deprecated 398 @Option(name = "binary-test-disable-framework", 399 description = "Adb stop/start before/after test.") 400 private boolean mBinaryTestDisableFramework = false; 401 402 @Deprecated 403 @Option(name = "binary-test-stop-native-servers", 404 description = "Set to stop all properly configured native servers during the testing.") 405 private boolean mBinaryTestStopNativeServers = false; 406 407 @Option(name = "disable-framework", description = "Adb stop/start before/after test.") 408 private boolean mDisableFramework = false; 409 410 @Option(name = "stop-native-servers", 411 description = "Set to stop all properly configured native servers during the testing.") 412 private boolean mStopNativeServers = false; 413 414 @Option(name = "bug-report-on-failure", 415 description = "To catch bugreport zip file at the end of failed test cases. " 416 + "If set to true, a report will be caught through adh shell command at the " 417 + "end of each failed test cases.") 418 private boolean mBugReportOnFailure = false; 419 420 @Option(name = "logcat-on-failure", 421 description = "To catch logcat from each buffers at the end of failed test cases. " 422 + "If set to true, a report will be caught through adh shell command at the " 423 + "end of each failed test cases.") 424 private boolean mLogcatOnFailure = true; 425 426 @Option(name = "native-server-process-name", 427 description = "Name of a native server process. The runner checks to make sure " 428 + "each specified native server process is not running after the framework stop.") 429 private Collection<String> mNativeServerProcessName = new ArrayList<>(); 430 431 @Option(name = "binary-test-type", description = "Binary test type. Only specify this when " 432 + "running an extended binary test without a python test file. Available options: gtest") 433 private String mBinaryTestType = ""; 434 435 @Option(name = "hal-hidl-replay-test-trace-path", description = "The path of a trace file to replay.") 436 private Collection<String> mHalHidlReplayTestTracePaths = new ArrayList<>(); 437 438 @Option(name = "hal-hidl-package-name", description = "The name of a target HIDL HAL package " 439 + "e.g., 'android.hardware.light@2.0'.") 440 private String mHalHidlPackageName = null; 441 442 @Option(name = "systrace-process-name", description = "Process name for systrace.") 443 private String mSystraceProcessName = null; 444 445 @Option(name = "collect-tests-only", 446 description = "Only invoke setUpClass, generate*, and tearDownClass to collect list " 447 + "of applicable test cases. All collected tests pass without being executed.") 448 private boolean mCollectTestsOnly = false; 449 450 @Option(name = "gtest-batch-mode", description = "Run Gtest binaries in batch mode.") 451 private boolean mGtestBatchMode = false; 452 453 @Option(name = "log-severity", 454 description = "Set the log severity level." 455 + "Note, this is a legacy option and does not affect how log files are saved." 456 + "By setting it to INFO, it will only make python DEBUG log not showing on " 457 + "console even if TradeFed log display level is set to DEBUG." 458 + "Therefore, it is not recommemded to set or modify this value in the current" 459 + "implementation.") 460 private String mLogSeverity = "DEBUG"; 461 462 @Option(name = "run-as-vts-self-test", 463 description = "Run the module as vts-selftest. " 464 + "When the value is set to true, only setUpClass and tearDownClass function " 465 + "of the module will be called to ensure the framework is free of bug. " 466 + "Note that exception in tearDownClass will not be reported as failure.") 467 private boolean mRunAsVtsSelfTest = false; 468 469 @Option(name = "exclude-coverage-path", 470 description = "The coverage path that should be excluded in results. " 471 + "Used only when enable-coverage is true.") 472 private Collection<String> mExcludeCoveragePath = new ArrayList<>(); 473 474 @Option(name = "mobly-test-module", 475 description = "Mobly test module name. " 476 + "If this value is specified, VTS will use mobly test template " 477 + "with the configurations." 478 + "Multiple values can be added by repeatly using this option.") 479 private Collection<String> mMoblyTestModule = new ArrayList<>(); 480 481 @Option(name = "acts-test-module", 482 description = "Acts test module name. " 483 + "If this value is specified, VTS will use acts test adapter " 484 + "with the configurations." 485 + "Multiple values can be added by repeatly using this option.") 486 private String mActsTestModule = null; 487 488 @Option(name = "config-str", 489 description = "Key-value map of custom config string. " 490 + "The map will be passed directly to python runner and test module. " 491 + "Only one value per key is stored." 492 + "If the value for the same key is set multiple times, only the last value is " 493 + "used.") 494 private TreeMap<String, String> mConfigStr = new TreeMap<>(); 495 496 @Option(name = "config-int", 497 description = "Key-value map of custom config integer. " 498 + "The map will be passed directly to python runner and test module. " 499 + "Only one value per key is stored." 500 + "If the value for the same key is set multiple times, only the last value is " 501 + "used.") 502 private TreeMap<String, Integer> mConfigInt = new TreeMap<>(); 503 504 @Option(name = "config-bool", 505 description = "Key-value map of custom config boolean. " 506 + "The map will be passed directly to python runner and test module. " 507 + "Only one value per key is stored." 508 + "If the value for the same key is set multiple times, only the last value is " 509 + "used.") 510 private TreeMap<String, Boolean> mConfigBool = new TreeMap<>(); 511 512 @Option(name = "max-retry-count", 513 description = "The max number of retries. Currerntly done by VTS Python runner in " 514 + "a test case granularity.") 515 private int mMaxRetryCount = 0; 516 517 private IBuildInfo mBuildInfo = null; 518 private String mRunName = null; 519 // the path to android-vts10/testcases 520 private String mTestCaseDir = "./"; 521 522 private VtsVendorConfigFileUtil configReader = null; 523 private IInvocationContext mInvocationContext = null; 524 private OutputUtil mOutputUtil = null; 525 protected CompatibilityBuildHelper mBuildHelper = null; 526 527 /** 528 * {@inheritDoc} 529 */ 530 @Override setInvocationContext(IInvocationContext context)531 public void setInvocationContext(IInvocationContext context) { 532 mInvocationContext = context; 533 setDevice(context.getDevices().get(0)); 534 setBuild(context.getBuildInfos().get(0)); 535 } 536 537 /** 538 * @return the mInvocationContext 539 */ getInvocationContext()540 public IInvocationContext getInvocationContext() { 541 return mInvocationContext; 542 } 543 544 /** 545 * {@inheritDoc} 546 */ 547 @Override setDevice(ITestDevice device)548 public void setDevice(ITestDevice device) { 549 mDevice = device; 550 } 551 552 /** 553 * {@inheritDoc} 554 */ 555 @Override getDevice()556 public ITestDevice getDevice() { 557 return mDevice; 558 } 559 setTestCasePath(String path)560 public void setTestCasePath(String path){ 561 mTestCasePath = path; 562 } 563 setTestConfigPath(String path)564 public void setTestConfigPath(String path){ 565 mTestConfigPath = path; 566 } 567 568 /** 569 * {@inheritDoc} 570 */ 571 @Override addIncludeFilter(String filter)572 public void addIncludeFilter(String filter) { 573 mIncludeFilters.add(cleanFilter(filter)); 574 } 575 576 /** 577 * {@inheritDoc} 578 */ 579 @Override addAllIncludeFilters(Set<String> filters)580 public void addAllIncludeFilters(Set<String> filters) { 581 for (String filter : filters) { 582 mIncludeFilters.add(cleanFilter(filter)); 583 } 584 } 585 586 /** 587 * {@inheritDoc} 588 */ 589 @Override clearIncludeFilters()590 public void clearIncludeFilters() { 591 mIncludeFilters.clear(); 592 } 593 594 /** {@inheritDoc} */ 595 @Override getIncludeFilters()596 public Set<String> getIncludeFilters() { 597 return mIncludeFilters; 598 } 599 600 /** 601 * {@inheritDoc} 602 */ 603 @Override addExcludeFilter(String filter)604 public void addExcludeFilter(String filter) { 605 mExcludeFilters.add(cleanFilter(filter)); 606 } 607 608 /** 609 * {@inheritDoc} 610 */ 611 @Override addAllExcludeFilters(Set<String> filters)612 public void addAllExcludeFilters(Set<String> filters) { 613 for (String filter : filters) { 614 mExcludeFilters.add(cleanFilter(filter)); 615 } 616 } 617 618 /** {@inheritDoc} */ 619 @Override clearExcludeFilters()620 public void clearExcludeFilters() { 621 mExcludeFilters.clear(); 622 } 623 624 /** {@inheritDoc} */ 625 @Override getExcludeFilters()626 public Set<String> getExcludeFilters() { 627 return mExcludeFilters; 628 } 629 630 /** 631 * Conforms filters using a {@link TestDescription} format 632 * to be recognized by the GTest executable. 633 */ cleanFilter(String filter)634 private String cleanFilter(String filter) { 635 return filter.replace('#', '.'); 636 } 637 638 /** 639 * {@inheritDoc} 640 */ 641 @Override getRuntimeHint()642 public long getRuntimeHint() { 643 return mRuntimeHint; 644 } 645 646 /** 647 * {@inheritDoc} 648 */ 649 @Override setCollectTestsOnly(boolean shouldCollectTest)650 public void setCollectTestsOnly(boolean shouldCollectTest) { 651 mCollectTestsOnly = shouldCollectTest; 652 } 653 654 /** 655 * Generate a device json object from ITestDevice object. 656 * 657 * @param device device object 658 * @throws RuntimeException 659 * @throws JSONException 660 */ generateJsonDeviceItem(ITestDevice device)661 private JSONObject generateJsonDeviceItem(ITestDevice device) throws JSONException { 662 JSONObject deviceItemObject = new JSONObject(); 663 deviceItemObject.put(SERIAL, device.getSerialNumber()); 664 try { 665 deviceItemObject.put("product_type", device.getProductType()); 666 deviceItemObject.put("product_variant", device.getProductVariant()); 667 deviceItemObject.put("build_alias", device.getBuildAlias()); 668 deviceItemObject.put("build_id", device.getBuildId()); 669 deviceItemObject.put("build_flavor", device.getBuildFlavor()); 670 } catch (DeviceNotAvailableException e) { 671 CLog.e("Device %s not available.", device.getSerialNumber()); 672 throw new RuntimeException("Failed to get device information"); 673 } 674 return deviceItemObject; 675 } 676 677 /** 678 * {@inheritDoc} 679 */ 680 @SuppressWarnings("deprecation") 681 @Override run(ITestInvocationListener listener)682 public void run(ITestInvocationListener listener) 683 throws IllegalArgumentException, DeviceNotAvailableException { 684 mOutputUtil = new OutputUtil(listener); 685 mOutputUtil.setTestModuleName(mTestModuleName); 686 if (mAbi != null) { 687 mOutputUtil.setAbiName(mAbi.getName()); 688 } 689 690 if (mTestCasePath == null) { 691 if (!mBinaryTestSource.isEmpty()) { 692 String template; 693 switch (mBinaryTestType) { 694 case BINARY_TEST_TYPE_GTEST: 695 template = TEMPLATE_GTEST_BINARY_TEST_PATH; 696 break; 697 case BINARY_TEST_TYPE_HAL_HIDL_GTEST: 698 template = TEMPLATE_HAL_HIDL_GTEST_PATH; 699 break; 700 case BINARY_TEST_TYPE_HOST_BINARY_TEST: 701 template = TEMPLATE_HOST_BINARY_TEST_PATH; 702 break; 703 default: 704 template = TEMPLATE_BINARY_TEST_PATH; 705 } 706 CLog.d("Using default test case template at %s.", template); 707 setTestCasePath(template); 708 if (mEnableCoverage && !mGlobalCoverage) { 709 CLog.e("Only global coverage is supported for test type %s.", mBinaryTestType); 710 throw new RuntimeException("Failed to produce VTS runner test config"); 711 } 712 } else if (mBinaryTestType.equals(BINARY_TEST_TYPE_HAL_HIDL_REPLAY_TEST)) { 713 setTestCasePath(TEMPLATE_HAL_HIDL_REPLAY_TEST_PATH); 714 } else if (mBinaryTestType.equals(BINARY_TEST_TYPE_LLVMFUZZER)) { 715 // Fuzz test don't need test-case-path. 716 setTestCasePath(TEMPLATE_LLVMFUZZER_TEST_PATH); 717 } else if (!mMoblyTestModule.isEmpty()) { 718 setTestCasePath(TEMPLATE_MOBLY_TEST_PATH); 719 } else if (mActsTestModule != null) { 720 setTestCasePath(ADAPTER_ACTS_PATH); 721 } else { 722 throw new IllegalArgumentException("test-case-path is not set."); 723 } 724 } 725 726 doRunTest(listener); 727 } 728 729 /** 730 * {@inheritDoc} 731 */ 732 @Override setBuild(IBuildInfo buildInfo)733 public void setBuild(IBuildInfo buildInfo) { 734 mBuildInfo = buildInfo; 735 mBuildHelper = new CompatibilityBuildHelper(mBuildInfo); 736 } 737 738 /** 739 * Populate a jsonObject with default fields. 740 * This method uses deepMergeJsonObjects method from JsonUtil to merge the default config file with the target 741 * config file. Field already defined in target config file will not be overwritten. Also, JSONArray will not be 742 * deep merged. 743 * 744 * @param jsonObject the target json object to populate 745 * @param testCaseDataDir data file path 746 * @throws IOException 747 * @throws JSONException 748 */ populateDefaultJsonFields(JSONObject jsonObject, String testCaseDataDir)749 private void populateDefaultJsonFields(JSONObject jsonObject, String testCaseDataDir) 750 throws IOException, JSONException { 751 CLog.d("Populating default fields to json object from %s", DEFAULT_TESTCASE_CONFIG_PATH); 752 String content = 753 FileUtil.readStringFromFile(new File(mTestCaseDir, DEFAULT_TESTCASE_CONFIG_PATH)); 754 JSONObject defaultJsonObject = new JSONObject(content); 755 756 JsonUtil.deepMergeJsonObjects(jsonObject, defaultJsonObject); 757 } 758 759 /** 760 * Derive mRunName from module name or test paths. 761 * 762 * @return the derived mRunName. 763 * @throws RuntimeException if mTestModuleName, mTestConfigPath, and mTestCasePath are null. 764 */ deriveRunName()765 private String deriveRunName() throws RuntimeException { 766 if (mRunName != null) { 767 return mRunName; 768 } 769 770 if (mTestModuleName != null) { 771 mRunName = mTestModuleName; 772 } else { 773 CLog.w("--test-module-name not set (not recommended); deriving automatically"); 774 if (mTestConfigPath != null) { 775 mRunName = new File(mTestConfigPath).getName(); 776 mRunName = mRunName.replace(CONFIG_FILE_EXTENSION, ""); 777 } else if (mTestCasePath != null) { 778 mRunName = new File(mTestCasePath).getName(); 779 } else { 780 throw new RuntimeException( 781 "Failed to derive test module name; use --test-module-name option"); 782 } 783 } 784 return mRunName; 785 } 786 787 /** 788 * This method reads the provided VTS runner test json config, adds or updates some of its 789 * fields (e.g., to add build info and device serial IDs), and returns the updated json object. 790 * This method calls populateDefaultJsonFields to populate the config JSONObject if the config file is missing 791 * or some required fields is missing from the JSONObject. If test name is not specified, this method 792 * will use the config file's file name file without extension as test name. If config file is missing, this method 793 * will use the test case's class name as test name. 794 * 795 * @param log_path the path of a directory to store the VTS runner logs. 796 * @return the updated JSONObject as the new test config. 797 */ updateVtsRunnerTestConfig(JSONObject jsonObject)798 protected void updateVtsRunnerTestConfig(JSONObject jsonObject) 799 throws IOException, JSONException { 800 configReader = new VtsVendorConfigFileUtil(); 801 if (configReader.LoadVendorConfig(mBuildInfo)) { 802 JSONObject vendorConfigJson = configReader.GetVendorConfigJson(); 803 if (vendorConfigJson != null) { 804 JsonUtil.deepMergeJsonObjects(jsonObject, vendorConfigJson); 805 } 806 } 807 808 CLog.d("Load original test config %s %s", mTestCaseDir, mTestConfigPath); 809 String content = null; 810 811 if (mTestConfigPath != null) { 812 content = FileUtil.readStringFromFile( 813 new File(Paths.get(mTestCaseDir, mTestConfigPath).toString())); 814 CLog.d("Loaded original test config %s", content); 815 if (content != null) { 816 JsonUtil.deepMergeJsonObjects(jsonObject, new JSONObject(content)); 817 } 818 } 819 820 populateDefaultJsonFields(jsonObject, mTestCaseDir); 821 CLog.d("Built a Json object using the loaded original test config"); 822 823 JSONArray deviceArray = new JSONArray(); 824 825 boolean coverageBuild = false; 826 boolean sancovBuild = false; 827 828 boolean first_device = true; 829 for (ITestDevice device : mInvocationContext.getDevices()) { 830 JSONObject deviceJson = generateJsonDeviceItem(device); 831 try { 832 String coverageProperty = device.getProperty(COVERAGE_PROPERTY); 833 boolean enable_coverage_for_device = 834 coverageProperty != null && coverageProperty.equals("1"); 835 if (first_device) { 836 coverageBuild = enable_coverage_for_device; 837 first_device = false; 838 } else { 839 if (coverageBuild && (!enable_coverage_for_device)) { 840 CLog.e("Device %s is not coverage build while others are.", 841 device.getSerialNumber()); 842 throw new RuntimeException("Device build not the same."); 843 } 844 } 845 } catch (DeviceNotAvailableException e) { 846 CLog.e("Device %s not available.", device.getSerialNumber()); 847 throw new RuntimeException("Failed to get device information"); 848 } 849 850 File sancovDir = 851 mBuildInfo.getFile(VtsCoveragePreparer.getSancovResourceDirKey(device)); 852 if (sancovDir != null) { 853 deviceJson.put("sancov_resources_path", sancovDir.getAbsolutePath()); 854 sancovBuild = true; 855 } 856 File gcovDir = mBuildInfo.getFile(VtsCoveragePreparer.getGcovResourceDirKey(device)); 857 if (gcovDir != null) { 858 deviceJson.put("gcov_resources_path", gcovDir.getAbsolutePath()); 859 coverageBuild = true; 860 } 861 deviceArray.put(deviceJson); 862 } 863 864 JSONArray testBedArray = (JSONArray) jsonObject.get(TEST_BED); 865 if (testBedArray.length() == 0) { 866 JSONObject testBedItemObject = new JSONObject(); 867 String testName = deriveRunName(); 868 CLog.d("Setting test module name as %s", testName); 869 testBedItemObject.put(NAME, testName); 870 testBedItemObject.put(ANDROIDDEVICE, deviceArray); 871 testBedArray.put(testBedItemObject); 872 } else if (testBedArray.length() == 1) { 873 JSONObject testBedItemObject = (JSONObject) testBedArray.get(0); 874 JSONArray androidDeviceArray = (JSONArray) testBedItemObject.get(ANDROIDDEVICE); 875 int length; 876 length = (androidDeviceArray.length() > deviceArray.length()) 877 ? androidDeviceArray.length() 878 : deviceArray.length(); 879 for (int index = 0; index < length; index++) { 880 if (index < androidDeviceArray.length()) { 881 if (index < deviceArray.length()) { 882 JsonUtil.deepMergeJsonObjects((JSONObject) androidDeviceArray.get(index), 883 (JSONObject) deviceArray.get(index)); 884 } 885 } else if (index < deviceArray.length()) { 886 androidDeviceArray.put(index, deviceArray.get(index)); 887 } 888 } 889 } else { 890 CLog.e("Multi-device not yet supported: %d devices requested", 891 testBedArray.length()); 892 throw new RuntimeException("Failed to produce VTS runner test config"); 893 } 894 jsonObject.put(DATA_FILE_PATH, mTestCaseDir); 895 CLog.d("Added %s = %s to the Json object", DATA_FILE_PATH, mTestCaseDir); 896 897 JSONObject build = new JSONObject(); 898 build.put(BUILD_ID, mBuildInfo.getBuildId()); 899 build.put(BUILD_TARGET, mBuildInfo.getBuildTargetName()); 900 jsonObject.put(BUILD, build); 901 CLog.d("Added %s to the Json object", BUILD); 902 903 JSONObject suite = new JSONObject(); 904 suite.put(NAME, mBuildInfo.getTestTag()); 905 suite.put(INCLUDE_FILTER, new JSONArray(mIncludeFilters)); 906 CLog.d("Added include filter to test suite: %s", mIncludeFilters); 907 suite.put(EXCLUDE_FILTER, new JSONArray(mExcludeFilters)); 908 CLog.d("Added exclude filter to test suite: %s", mExcludeFilters); 909 910 String coverageReportPath = mBuildInfo.getBuildAttributes().get("coverage_report_path"); 911 if (coverageReportPath != null) { 912 jsonObject.put(OUTPUT_COVERAGE_REPORT, true); 913 CLog.d("Added %s to the Json object", OUTPUT_COVERAGE_REPORT); 914 jsonObject.put(COVERAGE_REPORT_PATH, coverageReportPath); 915 CLog.d("Added %s to the Json object", COVERAGE_REPORT_PATH); 916 } 917 918 if (mExcludeOverInclude) { 919 jsonObject.put(EXCLUDE_OVER_INCLUDE, mExcludeOverInclude); 920 CLog.d("Added %s to the Json object", EXCLUDE_OVER_INCLUDE); 921 } 922 923 jsonObject.put(TEST_SUITE, suite); 924 CLog.d("Added %s to the Json object", TEST_SUITE); 925 926 jsonObject.put(TEST_TIMEOUT, mTestTimeout); 927 CLog.i("Added %s to the Json object: %d", TEST_TIMEOUT, mTestTimeout); 928 929 if (!mLogSeverity.isEmpty()) { 930 String logSeverity = mLogSeverity.toUpperCase(); 931 ArrayList<String> severityList = 932 new ArrayList<String>(Arrays.asList("ERROR", "WARNING", "INFO", "DEBUG")); 933 if (!severityList.contains(logSeverity)) { 934 CLog.w("Unsupported log severity %s, use default log_severity:INFO instead.", 935 logSeverity); 936 logSeverity = "INFO"; 937 } 938 jsonObject.put(LOG_SEVERITY, logSeverity); 939 CLog.d("Added %s to the Json object: %s", LOG_SEVERITY, logSeverity); 940 } 941 942 if (mShellDefaultNohup) { 943 jsonObject.put(SHELL_DEFAULT_NOHUP, mShellDefaultNohup); 944 CLog.d("Added %s to the Json object", SHELL_DEFAULT_NOHUP); 945 } 946 947 if (mAbi != null) { 948 jsonObject.put(ABI_NAME, mAbi.getName()); 949 CLog.d("Added %s to the Json object", ABI_NAME); 950 jsonObject.put(ABI_BITNESS, mAbi.getBitness()); 951 CLog.d("Added %s to the Json object", ABI_BITNESS); 952 } 953 954 if (mSkipOn32BitAbi) { 955 jsonObject.put(SKIP_ON_32BIT_ABI, mSkipOn32BitAbi); 956 CLog.d("Added %s to the Json object", SKIP_ON_32BIT_ABI); 957 } 958 959 if (mSkipOn64BitAbi) { 960 jsonObject.put(SKIP_ON_64BIT_ABI, mSkipOn64BitAbi); 961 CLog.d("Added %s to the Json object", SKIP_ON_64BIT_ABI); 962 } else if (mRun32bBitOn64BitAbi) { 963 jsonObject.put(RUN_32BIT_ON_64BIT_ABI, mRun32bBitOn64BitAbi); 964 CLog.d("Added %s to the Json object", RUN_32BIT_ON_64BIT_ABI); 965 } 966 967 if (mSkipIfThermalThrottling) { 968 jsonObject.put(SKIP_IF_THERMAL_THROTTLING, mSkipIfThermalThrottling); 969 CLog.d("Added %s to the Json object", SKIP_IF_THERMAL_THROTTLING); 970 } 971 972 jsonObject.put(DISABLE_CPU_FREQUENCY_SCALING, mDisableCpuFrequencyScaling); 973 CLog.d("Added %s to the Json object, value: %s", DISABLE_CPU_FREQUENCY_SCALING, 974 mDisableCpuFrequencyScaling); 975 976 if (!mBinaryTestSource.isEmpty()) { 977 jsonObject.put(BINARY_TEST_SOURCE, new JSONArray(mBinaryTestSource)); 978 CLog.d("Added %s to the Json object", BINARY_TEST_SOURCE); 979 } 980 981 if (!mBinaryTestWorkingDirectory.isEmpty()) { 982 jsonObject.put(BINARY_TEST_WORKING_DIRECTORY, 983 new JSONArray(mBinaryTestWorkingDirectory)); 984 CLog.d("Added %s to the Json object", BINARY_TEST_WORKING_DIRECTORY); 985 } 986 987 if (!mBinaryTestEnvp.isEmpty()) { 988 jsonObject.put(BINARY_TEST_ENVP, new JSONArray(mBinaryTestEnvp)); 989 CLog.d("Added %s to the Json object", BINARY_TEST_ENVP); 990 } 991 992 if (!mBinaryTestArgs.isEmpty()) { 993 jsonObject.put(BINARY_TEST_ARGS, new JSONArray(mBinaryTestArgs)); 994 CLog.d("Added %s to the Json object", BINARY_TEST_ARGS); 995 } 996 997 if (!mBinaryTestLdLibraryPath.isEmpty()) { 998 jsonObject.put(BINARY_TEST_LD_LIBRARY_PATH, 999 new JSONArray(mBinaryTestLdLibraryPath)); 1000 CLog.d("Added %s to the Json object", BINARY_TEST_LD_LIBRARY_PATH); 1001 } 1002 1003 if (mBugReportOnFailure) { 1004 jsonObject.put(BUG_REPORT_ON_FAILURE, mBugReportOnFailure); 1005 CLog.d("Added %s to the Json object", BUG_REPORT_ON_FAILURE); 1006 } 1007 1008 if (!mLogcatOnFailure) { 1009 jsonObject.put(LOGCAT_ON_FAILURE, mLogcatOnFailure); 1010 CLog.d("Added %s to the Json object", LOGCAT_ON_FAILURE); 1011 } 1012 1013 if (mEnableProfiling) { 1014 jsonObject.put(ENABLE_PROFILING, mEnableProfiling); 1015 CLog.d("Added %s to the Json object", ENABLE_PROFILING); 1016 } 1017 1018 if (mProfilingArgValue) { 1019 jsonObject.put(PROFILING_ARG_VALUE, mProfilingArgValue); 1020 CLog.d("Added %s to the Json object", PROFILING_ARG_VALUE); 1021 } 1022 1023 if (mSaveTraceFileRemote) { 1024 jsonObject.put(SAVE_TRACE_FIEL_REMOTE, mSaveTraceFileRemote); 1025 CLog.d("Added %s to the Json object", SAVE_TRACE_FIEL_REMOTE); 1026 } 1027 1028 if (mEnableSystrace) { 1029 jsonObject.put(ENABLE_SYSTRACE, mEnableSystrace); 1030 CLog.d("Added %s to the Json object", ENABLE_SYSTRACE); 1031 } 1032 1033 if (mEnableCoverage) { 1034 jsonObject.put(GLOBAL_COVERAGE, mGlobalCoverage); 1035 if (!mExcludeCoveragePath.isEmpty()) { 1036 jsonObject.put(EXCLUDE_COVERAGE_PATH, new JSONArray(mExcludeCoveragePath)); 1037 CLog.d("Added %s to the Json object", EXCLUDE_COVERAGE_PATH); 1038 } 1039 if (coverageBuild) { 1040 jsonObject.put(ENABLE_COVERAGE, mEnableCoverage); 1041 CLog.d("Added %s to the Json object", ENABLE_COVERAGE); 1042 } else { 1043 CLog.d("Device build has coverage disabled"); 1044 } 1045 } 1046 1047 if (mEnableSancov) { 1048 if (sancovBuild) { 1049 jsonObject.put(ENABLE_SANCOV, mEnableSancov); 1050 CLog.d("Added %s to the Json object", ENABLE_SANCOV); 1051 } else { 1052 CLog.d("Device build has sancov disabled"); 1053 } 1054 } 1055 1056 if (mPreconditionHwBinderServiceName != null) { 1057 jsonObject.put(PRECONDITION_HWBINDER_SERVICE, mPreconditionHwBinderServiceName); 1058 CLog.d("Added %s to the Json object", PRECONDITION_HWBINDER_SERVICE); 1059 } 1060 1061 if (mPreconditionFeature != null) { 1062 jsonObject.put(PRECONDITION_FEATURE, mPreconditionFeature); 1063 CLog.d("Added %s to the Json object", PRECONDITION_FEATURE); 1064 } 1065 1066 if (!mPreconditionFilePathPrefix.isEmpty()) { 1067 jsonObject.put( 1068 PRECONDITION_FILE_PATH_PREFIX, new JSONArray(mPreconditionFilePathPrefix)); 1069 CLog.d("Added %s to the Json object", PRECONDITION_FILE_PATH_PREFIX); 1070 } 1071 1072 if (mPreconditionFirstApiLevel != 0) { 1073 jsonObject.put(PRECONDITION_FIRST_API_LEVEL, mPreconditionFirstApiLevel); 1074 CLog.d("Added %s to the Json object", PRECONDITION_FIRST_API_LEVEL); 1075 } 1076 1077 if (mPreconditionLshal != null) { 1078 jsonObject.put(PRECONDITION_LSHAL, mPreconditionLshal); 1079 CLog.d("Added %s to the Json object", PRECONDITION_LSHAL); 1080 } 1081 1082 if (mPreconditionVintf != null) { 1083 jsonObject.put(PRECONDITION_VINTF, mPreconditionVintf); 1084 CLog.d("Added %s to the Json object", PRECONDITION_VINTF); 1085 } 1086 1087 if (mPreconditionSysProp != null) { 1088 jsonObject.put(PRECONDITION_SYSPROP, mPreconditionSysProp); 1089 CLog.d("Added %s to the Json object", PRECONDITION_SYSPROP); 1090 } 1091 1092 if (!mBinaryTestProfilingLibraryPath.isEmpty()) { 1093 jsonObject.put(BINARY_TEST_PROFILING_LIBRARY_PATH, 1094 new JSONArray(mBinaryTestProfilingLibraryPath)); 1095 CLog.d("Added %s to the Json object", BINARY_TEST_PROFILING_LIBRARY_PATH); 1096 } 1097 1098 if (mDisableFramework) { 1099 jsonObject.put(DISABLE_FRAMEWORK, mDisableFramework); 1100 CLog.d("Added %s to the Json object", DISABLE_FRAMEWORK); 1101 } 1102 1103 if (mStopNativeServers) { 1104 jsonObject.put(STOP_NATIVE_SERVERS, mStopNativeServers); 1105 CLog.d("Added %s to the Json object", STOP_NATIVE_SERVERS); 1106 } 1107 1108 if (mBinaryTestDisableFramework) { 1109 jsonObject.put(BINARY_TEST_DISABLE_FRAMEWORK, mBinaryTestDisableFramework); 1110 CLog.d("Added %s to the Json object", BINARY_TEST_DISABLE_FRAMEWORK); 1111 } 1112 1113 if (mBinaryTestStopNativeServers) { 1114 jsonObject.put(BINARY_TEST_STOP_NATIVE_SERVERS, mBinaryTestStopNativeServers); 1115 CLog.d("Added %s to the Json object", BINARY_TEST_STOP_NATIVE_SERVERS); 1116 } 1117 1118 if (!mNativeServerProcessName.isEmpty()) { 1119 jsonObject.put(NATIVE_SERVER_PROCESS_NAME, new JSONArray(mNativeServerProcessName)); 1120 CLog.d("Added %s to the Json object", NATIVE_SERVER_PROCESS_NAME); 1121 } 1122 1123 if (!mHalHidlReplayTestTracePaths.isEmpty()) { 1124 jsonObject.put(HAL_HIDL_REPLAY_TEST_TRACE_PATHS, 1125 new JSONArray(mHalHidlReplayTestTracePaths)); 1126 CLog.d("Added %s to the Json object", HAL_HIDL_REPLAY_TEST_TRACE_PATHS); 1127 } 1128 1129 if (mHalHidlPackageName != null) { 1130 jsonObject.put(HAL_HIDL_PACKAGE_NAME, mHalHidlPackageName); 1131 CLog.d("Added %s to the Json object", SYSTRACE_PROCESS_NAME); 1132 } 1133 1134 if (mSystraceProcessName != null) { 1135 jsonObject.put(SYSTRACE_PROCESS_NAME, mSystraceProcessName); 1136 CLog.d("Added %s to the Json object", SYSTRACE_PROCESS_NAME); 1137 } 1138 1139 if (mPassthroughMode) { 1140 jsonObject.put(PASSTHROUGH_MODE, mPassthroughMode); 1141 CLog.d("Added %s to the Json object", PASSTHROUGH_MODE); 1142 } 1143 1144 if (mCollectTestsOnly) { 1145 jsonObject.put(COLLECT_TESTS_ONLY, mCollectTestsOnly); 1146 CLog.d("Added %s to the Json object", COLLECT_TESTS_ONLY); 1147 } 1148 1149 if (mGtestBatchMode) { 1150 jsonObject.put(GTEST_BATCH_MODE, mGtestBatchMode); 1151 CLog.d("Added %s to the Json object", GTEST_BATCH_MODE); 1152 } 1153 1154 if (mLtpNumberOfThreads >= 0) { 1155 jsonObject.put(LTP_NUMBER_OF_THREADS, mLtpNumberOfThreads); 1156 CLog.d("Added %s to the Json object", LTP_NUMBER_OF_THREADS); 1157 } 1158 1159 if (mRunAsVtsSelfTest) { 1160 jsonObject.put(RUN_AS_VTS_SELF_TEST, mRunAsVtsSelfTest); 1161 CLog.d("Added %s to the Json object", RUN_AS_VTS_SELF_TEST); 1162 } 1163 1164 if ("vts".equals(mBuildInfo.getTestTag())) { 1165 jsonObject.put(RUN_AS_COMPLIANCE_TEST, true); 1166 CLog.d("Added %s to the Json object", RUN_AS_COMPLIANCE_TEST); 1167 } 1168 1169 if (!mMoblyTestModule.isEmpty()) { 1170 jsonObject.put(MOBLY_TEST_MODULE, new JSONArray(mMoblyTestModule)); 1171 CLog.d("Added %s to the Json object", MOBLY_TEST_MODULE); 1172 } 1173 1174 if (mActsTestModule != null) { 1175 jsonObject.put(ACTS_TEST_MODULE, mActsTestModule); 1176 CLog.d("Added %s to the Json object", ACTS_TEST_MODULE); 1177 } 1178 1179 if (mBuildInfo.getFile(VtsPythonVirtualenvPreparer.VIRTUAL_ENV) != null) { 1180 jsonObject.put(VtsPythonVirtualenvPreparer.VIRTUAL_ENV, 1181 mBuildInfo.getFile(VtsPythonVirtualenvPreparer.VIRTUAL_ENV).getAbsolutePath()); 1182 } 1183 1184 if (mBuildInfo.getFile(VtsPythonVirtualenvPreparer.VIRTUAL_ENV_V3) != null) { 1185 jsonObject.put(VtsPythonVirtualenvPreparer.VIRTUAL_ENV_V3, 1186 mBuildInfo.getFile(VtsPythonVirtualenvPreparer.VIRTUAL_ENV_V3) 1187 .getAbsolutePath()); 1188 } 1189 1190 if (!mConfigStr.isEmpty()) { 1191 jsonObject.put(CONFIG_STR, new JSONObject(mConfigStr)); 1192 CLog.d("Added %s to the Json object", CONFIG_STR); 1193 } 1194 1195 if (!mConfigInt.isEmpty()) { 1196 jsonObject.put(CONFIG_INT, new JSONObject(mConfigInt)); 1197 CLog.d("Added %s to the Json object", CONFIG_INT); 1198 } 1199 1200 if (!mConfigBool.isEmpty()) { 1201 jsonObject.put(CONFIG_BOOL, new JSONObject(mConfigBool)); 1202 CLog.d("Added %s to the Json object", CONFIG_BOOL); 1203 } 1204 1205 if (mEnableLogUploading) { 1206 jsonObject.put(ENABLE_LOG_UPLOADING, "true"); 1207 CLog.d("Added %s to the Json object with value: true)", ENABLE_LOG_UPLOADING); 1208 } 1209 1210 if (mMaxRetryCount > 0) { 1211 jsonObject.put(MAX_RETRY_COUNT, mMaxRetryCount); 1212 CLog.d("Added %s to the Json object", MAX_RETRY_COUNT); 1213 } 1214 } 1215 1216 /** 1217 * Log a test module execution status to device logcat. 1218 * 1219 * @param status 1220 * @return true if succesful, false otherwise 1221 */ 1222 @VisibleForTesting printToDeviceLogcatAboutTestModuleStatus(String status)1223 protected void printToDeviceLogcatAboutTestModuleStatus(String status) 1224 throws DeviceNotAvailableException { 1225 mDevice.executeShellCommand(String.format( 1226 "log -p i -t \"VTS\" \"[Test Module] %s %s\"", mTestModuleName, status)); 1227 } 1228 1229 /** 1230 * Create vts python test runner test config json file. 1231 * 1232 * @param status 1233 * @throws RuntimeException 1234 * @return test config json file absolute path string 1235 */ 1236 @VisibleForTesting createVtsRunnerTestConfigJsonFile(File vtsRunnerLogDir)1237 protected String createVtsRunnerTestConfigJsonFile(File vtsRunnerLogDir) { 1238 JSONObject jsonObject = new JSONObject(); 1239 try { 1240 updateVtsRunnerTestConfig(jsonObject); 1241 1242 jsonObject.put(LOG_PATH, vtsRunnerLogDir.getAbsolutePath()); 1243 CLog.d("Added %s to the Json object", LOG_PATH); 1244 } catch (IOException e) { 1245 throw new RuntimeException("Failed to read test config json file"); 1246 } catch (JSONException e) { 1247 throw new RuntimeException("Failed to build updated test config json data"); 1248 } 1249 1250 CLog.d("VTS python test config json: %s", jsonObject.toString()); 1251 1252 String jsonFilePath = null; 1253 try { 1254 File tmpFile = FileUtil.createTempFile( 1255 mBuildInfo.getTestTag() + "-config-" + mBuildInfo.getDeviceSerial(), ".json", 1256 vtsRunnerLogDir); 1257 jsonFilePath = tmpFile.getAbsolutePath(); 1258 CLog.d("VTS test config json file path: %s", jsonFilePath); 1259 FileUtil.writeToFile(jsonObject.toString(), tmpFile); 1260 } catch (IOException e) { 1261 throw new RuntimeException("Failed to create vts test config json file"); 1262 } 1263 return jsonFilePath; 1264 } 1265 AddTestModuleKeys(String test_module_name, long test_module_timestamp)1266 private boolean AddTestModuleKeys(String test_module_name, long test_module_timestamp) { 1267 if (test_module_name.length() == 0 || test_module_timestamp == -1) { 1268 CLog.e(String.format("Test module keys (%s,%d) are invalid.", test_module_name, 1269 test_module_timestamp)); 1270 return false; 1271 } 1272 File reportFile = mBuildInfo.getFile(TEST_PLAN_REPORT_FILE); 1273 1274 if (reportFile != null) { 1275 try (FileWriter fw = new FileWriter(reportFile.getAbsoluteFile(), true); 1276 BufferedWriter bw = new BufferedWriter(fw); 1277 PrintWriter out = new PrintWriter(bw)) { 1278 out.println(String.format("%s %s", test_module_name, test_module_timestamp)); 1279 } catch (IOException e) { 1280 CLog.e(String.format( 1281 "Can't write to the test plan result file, %s", TEST_PLAN_REPORT_FILE)); 1282 return false; 1283 } 1284 } else { 1285 CLog.w("No test plan report file configured."); 1286 } 1287 return true; 1288 } 1289 1290 /** 1291 * This method prepares a command for the test and runs the python file as 1292 * given in the arguments. 1293 * 1294 * @param listener 1295 * @throws RuntimeException 1296 * @throws IllegalArgumentException 1297 */ doRunTest(ITestInvocationListener listener)1298 private void doRunTest(ITestInvocationListener listener) 1299 throws IllegalArgumentException, DeviceNotAvailableException { 1300 long methodStartTime = System.currentTimeMillis(); 1301 CLog.d("Device serial number: " + mDevice.getSerialNumber()); 1302 1303 setTestCaseDir(); 1304 1305 VtsMultiDeviceTestResultParser parser = 1306 new VtsMultiDeviceTestResultParser(listener, deriveRunName()); 1307 1308 File vtsRunnerLogDir = null; 1309 try { 1310 vtsRunnerLogDir = FileUtil.createTempDir("vts-runner-log"); 1311 } catch(IOException e) { 1312 throw new RuntimeException("Failed to creat temp vts-runner-log directory"); 1313 } 1314 1315 long timeout = mMaxTestTimeout; 1316 if (mMaxTestTimeout < mTestTimeout) { 1317 // The Python runner will receive 2 interrupts. 1318 // Delay the 2nd one to avoid interrupting the runner's teardown procedure. 1319 timeout = mTestTimeout + VtsPythonRunnerHelper.TEST_ABORT_TIMEOUT_MSECS; 1320 CLog.w("max-test-timeout is less than test-timeout. Set max timeout to %dms.", timeout); 1321 } 1322 1323 try { 1324 String jsonFilePath = createVtsRunnerTestConfigJsonFile(vtsRunnerLogDir); 1325 1326 VtsPythonRunnerHelper vtsPythonRunnerHelper = 1327 createVtsPythonRunnerHelper(new File(mTestCaseDir)); 1328 1329 List<String> cmd = new ArrayList<>(); 1330 cmd.add("python"); 1331 if (mTestCasePathType != null && mTestCasePathType.toLowerCase().equals("file")) { 1332 String testScript = mTestCasePath; 1333 if (!testScript.endsWith(".py")) { 1334 testScript += ".py"; 1335 } 1336 cmd.add(testScript); 1337 } else { 1338 cmd.add("-m"); 1339 cmd.add(mTestCasePath.replace("/", ".")); 1340 } 1341 cmd.add(jsonFilePath); 1342 1343 printToDeviceLogcatAboutTestModuleStatus("BEGIN"); 1344 1345 CommandResult commandResult = new CommandResult(); 1346 String interruptMessage; 1347 1348 File stdOutFile = FileUtil.createTempFile("vts_python_runner_stdout", ".log"); 1349 File stdErrFile = FileUtil.createTempFile("vts_python_runner_stderr", ".log"); 1350 1351 OutputStream stdOut = new FileOutputStream(stdOutFile); 1352 OutputStream stdErr = new FileOutputStream(stdErrFile); 1353 List<String> errorMsgs = new ArrayList<>(); 1354 1355 try { 1356 interruptMessage = vtsPythonRunnerHelper.runPythonRunner( 1357 cmd.toArray(new String[0]), commandResult, timeout, stdOut, stdErr); 1358 1359 if (commandResult != null) { 1360 CommandStatus commandStatus = commandResult.getStatus(); 1361 if (commandStatus != CommandStatus.SUCCESS 1362 && commandStatus != CommandStatus.TIMED_OUT) { 1363 errorMsgs.add("Python process failed"); 1364 // Log the last 2k bytes of stdOutFile and stdErrFile 1365 String skippedBytesMsg = ""; 1366 long startOffset = 0; 1367 if (stdOutFile.length() > 2048) { 1368 startOffset = stdOutFile.length() - 2048; 1369 skippedBytesMsg = String.format( 1370 "...... <%d bytes skipped> ......\n", startOffset); 1371 } 1372 errorMsgs.add(String.format("Command stdout: %s%s", skippedBytesMsg, 1373 FileUtil.readStringFromFile(stdOutFile, startOffset, 2048))); 1374 skippedBytesMsg = ""; 1375 startOffset = 0; 1376 if (stdErrFile.length() > 2048) { 1377 startOffset = stdErrFile.length() - 2048; 1378 skippedBytesMsg = String.format( 1379 "...... <%d bytes skipped> ......\n", startOffset); 1380 } 1381 errorMsgs.add(String.format("Command stderr: %s%s", skippedBytesMsg, 1382 FileUtil.readStringFromFile(stdErrFile, startOffset, 2048))); 1383 errorMsgs.add("Command status: " + commandStatus); 1384 } 1385 } 1386 } finally { 1387 StreamUtil.close(stdOut); 1388 StreamUtil.close(stdErr); 1389 listener.testLog(stdOutFile.getName(), LogDataType.TEXT, 1390 new FileInputStreamSource(stdOutFile)); 1391 listener.testLog(stdErrFile.getName(), LogDataType.TEXT, 1392 new FileInputStreamSource(stdErrFile)); 1393 FileUtil.deleteFile(stdOutFile); 1394 FileUtil.deleteFile(stdErrFile); 1395 } 1396 1397 // parse from test_run_summary.json instead of stdout 1398 File testRunSummary = getFileTestRunSummary(vtsRunnerLogDir); 1399 if (testRunSummary == null) { 1400 errorMsgs.add("Couldn't locate the file : " + TEST_RUN_SUMMARY_FILE_NAME); 1401 } else { 1402 JSONObject object = null; 1403 try { 1404 String jsonData = FileUtil.readStringFromFile(testRunSummary); 1405 CLog.d("Test Result Summary: %s", jsonData); 1406 object = new JSONObject(jsonData); 1407 parser.processJsonFile(object); 1408 } catch (IOException | JSONException e) { 1409 errorMsgs.add("Error occurred in parsing Json file " + testRunSummary.toPath()); 1410 CLog.e(e); 1411 } 1412 try { 1413 JSONObject planObject = object.getJSONObject(TESTMODULE); 1414 String test_module_name = planObject.getString("Name"); 1415 long test_module_timestamp = planObject.getLong("Timestamp"); 1416 AddTestModuleKeys(test_module_name, test_module_timestamp); 1417 } catch (JSONException e) { 1418 // Do not report this as part of errorMsgs. These are optional metadata 1419 CLog.e(e); 1420 } 1421 } 1422 1423 if (errorMsgs.size() > 0) { 1424 CLog.e(String.join(".\n", errorMsgs)); 1425 listener.testRunFailed(String.join(".\n", errorMsgs)); 1426 listener.testRunEnded(System.currentTimeMillis() - methodStartTime, 1427 new HashMap<String, Metric>()); 1428 } 1429 1430 printToDeviceLogcatAboutTestModuleStatus("END"); 1431 if (interruptMessage != null) { 1432 throw new RuntimeException(interruptMessage); 1433 } 1434 } catch (IOException e) { 1435 throw new RuntimeException(e); 1436 } finally { 1437 try { 1438 mOutputUtil.ZipVtsRunnerOutputDir(vtsRunnerLogDir); 1439 1440 if (mEnableDashboardUploading) { 1441 File reportMsg = FileUtil.findFile(vtsRunnerLogDir, REPORT_MESSAGE_FILE_NAME); 1442 CLog.d("Report message path: %s", reportMsg); 1443 if (reportMsg == null) { 1444 CLog.e("Cannot find report message proto file."); 1445 } else if (reportMsg.length() > 0) { 1446 CLog.i("Uploading report message. File size: %s", reportMsg.length()); 1447 VtsDashboardUtil dashboardUtil = new VtsDashboardUtil(configReader); 1448 dashboardUtil.Upload(reportMsg.getAbsolutePath()); 1449 } 1450 } 1451 } finally { 1452 CLog.d("Deleted the runner log dir, %s.", vtsRunnerLogDir); 1453 FileUtil.recursiveDelete(vtsRunnerLogDir); 1454 } 1455 // If the framework was disabled in python, make sure we re-enable it no matter what. 1456 // The python side never re-enable the framework. 1457 if (mBinaryTestDisableFramework || mDisableFramework) { 1458 for (ITestDevice device : mInvocationContext.getDevices()) { 1459 device.executeShellCommand("start"); 1460 } 1461 } 1462 } 1463 for (ITestDevice device : mInvocationContext.getDevices()) { 1464 device.waitForDeviceAvailable(); 1465 } 1466 } 1467 1468 /** 1469 * This method return the file test_run_summary.json which is then used to parse logs. 1470 * 1471 * @param logDir : The file that needs to be converted 1472 * @return the file named test_run_summary.json 1473 * @throws IllegalArgumentException 1474 */ getFileTestRunSummary(File logDir)1475 private File getFileTestRunSummary(File logDir) throws IllegalArgumentException { 1476 File[] children; 1477 if (logDir == null) { 1478 throw new IllegalArgumentException("Argument logDir is null."); 1479 } 1480 children = logDir.listFiles(); 1481 if (children != null) { 1482 for (File child : children) { 1483 if (!child.isDirectory()) { 1484 if (child.getName().equals(TEST_RUN_SUMMARY_FILE_NAME)) { 1485 return child; 1486 } 1487 } else { 1488 File file = getFileTestRunSummary(child); 1489 if (file != null) { 1490 return file; 1491 } 1492 } 1493 } 1494 } 1495 return null; 1496 } 1497 1498 /** 1499 * Set the path for android-vts10/testcases/ which keeps the VTS python code under vts. 1500 */ setTestCaseDir()1501 private void setTestCaseDir() { 1502 try { 1503 mTestCaseDir = mBuildHelper.getTestsDir().getAbsolutePath(); 1504 } catch (FileNotFoundException e) { 1505 CLog.e("Cannot get testcase dir. Tests may not run correctly."); 1506 } 1507 } 1508 1509 /** 1510 * {@inheritDoc} 1511 */ 1512 @Override setAbi(IAbi abi)1513 public void setAbi(IAbi abi){ 1514 mAbi = abi; 1515 } 1516 1517 /** 1518 * Creates a {@link VtsPythonRunnerHelper}. 1519 */ 1520 @VisibleForTesting createVtsPythonRunnerHelper(File workingDir)1521 protected VtsPythonRunnerHelper createVtsPythonRunnerHelper(File workingDir) { 1522 return new VtsPythonRunnerHelper(mBuildInfo, workingDir); 1523 } 1524 } 1525