1 /* 2 * Copyright (C) 2010 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package com.android.tradefed.config; 18 19 import com.android.ddmlib.Log; 20 import com.android.tradefed.command.CommandOptions; 21 import com.android.tradefed.config.yaml.ConfigurationYamlParser; 22 import com.android.tradefed.log.LogUtil.CLog; 23 import com.android.tradefed.util.ClassPathScanner; 24 import com.android.tradefed.util.ClassPathScanner.IClassPathFilter; 25 import com.android.tradefed.util.DirectedGraph; 26 import com.android.tradefed.util.FileUtil; 27 import com.android.tradefed.util.StreamUtil; 28 import com.android.tradefed.util.SystemUtil; 29 import com.android.tradefed.util.keystore.DryRunKeyStore; 30 import com.android.tradefed.util.keystore.IKeyStoreClient; 31 32 import com.google.common.annotations.VisibleForTesting; 33 import com.google.common.base.Strings; 34 import com.google.common.collect.ImmutableSortedSet; 35 36 import java.io.BufferedInputStream; 37 import java.io.ByteArrayOutputStream; 38 import java.io.File; 39 import java.io.FileInputStream; 40 import java.io.FileNotFoundException; 41 import java.io.IOException; 42 import java.io.InputStream; 43 import java.io.PrintStream; 44 import java.util.ArrayList; 45 import java.util.Arrays; 46 import java.util.Comparator; 47 import java.util.HashMap; 48 import java.util.Hashtable; 49 import java.util.List; 50 import java.util.Map; 51 import java.util.Set; 52 import java.util.SortedSet; 53 import java.util.TreeSet; 54 import java.util.regex.Pattern; 55 56 /** 57 * Factory for creating {@link IConfiguration}. 58 */ 59 public class ConfigurationFactory implements IConfigurationFactory { 60 61 /** Currently supported extensions for Tradefed configurations */ 62 private static final Set<String> SUPPORTED_EXTENSIONS = 63 ImmutableSortedSet.of(".xml", ".config"); 64 65 private static final String LOG_TAG = "ConfigurationFactory"; 66 private static IConfigurationFactory sInstance = null; 67 private static final String CONFIG_PREFIX = "config/"; 68 private static final String DRY_RUN_TEMPLATE_CONFIG = "empty"; 69 private static final String CONFIG_ERROR_PATTERN = "(Could not find option with name )(.*)"; 70 71 private Map<ConfigId, ConfigurationDef> mConfigDefMap; 72 73 /** 74 * A simple struct-like class that stores a configuration's name alongside 75 * the arguments for any {@code <template-include>} tags it may contain. 76 * Because the actual bits stored by the configuration may vary with 77 * template arguments, they must be considered as essential a part of the 78 * configuration's identity as the filename. 79 */ 80 static class ConfigId { 81 public String name = null; 82 public Map<String, String> templateMap = new HashMap<>(); 83 84 /** 85 * No-op constructor 86 */ ConfigId()87 public ConfigId() { 88 } 89 90 /** 91 * Convenience constructor. Equivalent to calling two-arg constructor 92 * with {@code null} {@code templateMap}. 93 */ ConfigId(String name)94 public ConfigId(String name) { 95 this(name, null); 96 } 97 98 /** 99 * Two-arg convenience constructor. {@code templateMap} may be null. 100 */ ConfigId(String name, Map<String, String> templateMap)101 public ConfigId(String name, Map<String, String> templateMap) { 102 this.name = name; 103 if (templateMap != null) { 104 this.templateMap.putAll(templateMap); 105 } 106 } 107 108 /** 109 * {@inheritDoc} 110 */ 111 @Override hashCode()112 public int hashCode() { 113 return 2 * ((name == null) ? 0 : name.hashCode()) + 3 * templateMap.hashCode(); 114 } 115 matches(Object a, Object b)116 private boolean matches(Object a, Object b) { 117 if (a == null && b == null) 118 return true; 119 if (a == null || b == null) 120 return false; 121 return a.equals(b); 122 } 123 124 /** 125 * {@inheritDoc} 126 */ 127 @Override equals(Object other)128 public boolean equals(Object other) { 129 if (other == null) 130 return false; 131 if (!(other instanceof ConfigId)) 132 return false; 133 134 final ConfigId otherConf = (ConfigId) other; 135 return matches(name, otherConf.name) && matches(templateMap, otherConf.templateMap); 136 } 137 } 138 139 /** 140 * A {@link IClassPathFilter} for configuration XML files. 141 */ 142 private class ConfigClasspathFilter implements IClassPathFilter { 143 144 private String mPrefix = null; 145 ConfigClasspathFilter(String prefix)146 public ConfigClasspathFilter(String prefix) { 147 mPrefix = getConfigPrefix(); 148 if (prefix != null) { 149 mPrefix += prefix; 150 } 151 CLog.d("Searching the '%s' config path", mPrefix); 152 } 153 154 /** 155 * {@inheritDoc} 156 */ 157 @Override accept(String pathName)158 public boolean accept(String pathName) { 159 // only accept entries that match the pattern, and that we don't already know about 160 final ConfigId pathId = new ConfigId(pathName); 161 String extension = FileUtil.getExtension(pathName); 162 return pathName.startsWith(mPrefix) 163 && SUPPORTED_EXTENSIONS.contains(extension) 164 && !mConfigDefMap.containsKey(pathId); 165 } 166 167 /** 168 * {@inheritDoc} 169 */ 170 @Override transform(String pathName)171 public String transform(String pathName) { 172 // strip off CONFIG_PREFIX and config extension 173 int pathStartIndex = getConfigPrefix().length(); 174 String extension = FileUtil.getExtension(pathName); 175 int pathEndIndex = pathName.length() - extension.length(); 176 return pathName.substring(pathStartIndex, pathEndIndex); 177 } 178 } 179 180 /** 181 * A {@link Comparator} for {@link ConfigurationDef} that sorts by 182 * {@link ConfigurationDef#getName()}. 183 */ 184 private static class ConfigDefComparator implements Comparator<ConfigurationDef> { 185 186 /** 187 * {@inheritDoc} 188 */ 189 @Override compare(ConfigurationDef d1, ConfigurationDef d2)190 public int compare(ConfigurationDef d1, ConfigurationDef d2) { 191 return d1.getName().compareTo(d2.getName()); 192 } 193 194 } 195 196 /** 197 * Get a list of {@link File} of the test cases directories 198 * 199 * <p>The wrapper function is for unit test to mock the system calls. 200 * 201 * @return a list of {@link File} of directories of the test cases folder of build output, based 202 * on the value of environment variables. 203 */ 204 @VisibleForTesting getExternalTestCasesDirs()205 List<File> getExternalTestCasesDirs() { 206 return SystemUtil.getExternalTestCasesDirs(); 207 } 208 209 /** 210 * Get the path to the config file for a test case. 211 * 212 * <p>The given name in a test config can be the name of a test case located in an out directory 213 * defined in the following environment variables: 214 * 215 * <p>ANDROID_TARGET_OUT_TESTCASES 216 * 217 * <p>ANDROID_HOST_OUT_TESTCASES 218 * 219 * <p>This method tries to locate the test config name in these directories. If no config is 220 * found, return null. 221 * 222 * @param name Name of a config file. 223 * @return A File object of the config file for the given test case. 224 */ 225 @VisibleForTesting getTestCaseConfigPath(String name)226 File getTestCaseConfigPath(String name) { 227 String[] possibleConfigFileNames = {name}; 228 if (Strings.isNullOrEmpty(FileUtil.getExtension(name))) { 229 possibleConfigFileNames = new String[SUPPORTED_EXTENSIONS.size()]; 230 int i = 0; 231 for (String supportedExtension : SUPPORTED_EXTENSIONS) { 232 possibleConfigFileNames[i] = (name + supportedExtension); 233 i++; 234 } 235 } 236 237 for (File testCasesDir : getExternalTestCasesDirs()) { 238 for (String configFileName : possibleConfigFileNames) { 239 File config = FileUtil.findFile(testCasesDir, configFileName); 240 if (config != null) { 241 CLog.d("Using config: %s/%s", testCasesDir.getAbsoluteFile(), configFileName); 242 return config; 243 } 244 } 245 } 246 return null; 247 } 248 249 /** 250 * Implementation of {@link IConfigDefLoader} that tracks the included configurations from one 251 * root config, and throws an exception on circular includes. 252 */ 253 protected class ConfigLoader implements IConfigDefLoader { 254 255 private final boolean mIsGlobalConfig; 256 private DirectedGraph<String> mConfigGraph = new DirectedGraph<String>(); 257 ConfigLoader(boolean isGlobalConfig)258 public ConfigLoader(boolean isGlobalConfig) { 259 mIsGlobalConfig = isGlobalConfig; 260 } 261 262 /** 263 * {@inheritDoc} 264 */ 265 @Override getConfigurationDef(String name, Map<String, String> templateMap)266 public ConfigurationDef getConfigurationDef(String name, Map<String, String> templateMap) 267 throws ConfigurationException { 268 269 String configName = findConfigName(name, null); 270 final ConfigId configId = new ConfigId(name, templateMap); 271 ConfigurationDef def = mConfigDefMap.get(configId); 272 273 if (def == null || def.isStale()) { 274 def = new ConfigurationDef(configName); 275 loadConfiguration(configName, def, null, templateMap, null); 276 mConfigDefMap.put(configId, def); 277 } else { 278 if (templateMap != null) { 279 // Clearing the map before returning the cached config to 280 // avoid seeing them as unused. 281 templateMap.clear(); 282 } 283 } 284 return def; 285 } 286 287 /** Returns true if it is a config file found inside the classpath. */ isBundledConfig(String name)288 protected boolean isBundledConfig(String name) { 289 InputStream configStream = getBundledConfigStream(name); 290 return configStream != null; 291 } 292 293 /** 294 * Get the absolute path of a local config file. 295 * 296 * @param root parent path of config file 297 * @param name config file 298 * @return absolute path for local config file. 299 * @throws ConfigurationException 300 */ getAbsolutePath(String root, String name)301 private String getAbsolutePath(String root, String name) throws ConfigurationException { 302 File file = new File(name); 303 if (!file.isAbsolute()) { 304 if (root == null) { 305 // if root directory was not specified, get the current 306 // working directory. 307 root = System.getProperty("user.dir"); 308 } 309 file = new File(root, name); 310 } 311 try { 312 return file.getCanonicalPath(); 313 } catch (IOException e) { 314 throw new ConfigurationException(String.format( 315 "Failure when trying to determine local file canonical path %s", e)); 316 } 317 } 318 319 /** 320 * Find config's name based on its name and its parent name. This is used to properly handle 321 * bundle configs and local configs. 322 * 323 * @param name config's name 324 * @param parentName config's parent's name. 325 * @return the config's full name. 326 * @throws ConfigurationException 327 */ findConfigName(String name, String parentName)328 protected String findConfigName(String name, String parentName) 329 throws ConfigurationException { 330 if (isBundledConfig(name)) { 331 return name; 332 } 333 if (parentName == null || isBundledConfig(parentName)) { 334 // Search files for config. 335 String configName = getAbsolutePath(null, name); 336 File localConfig = new File(configName); 337 if (!localConfig.exists()) { 338 localConfig = getTestCaseConfigPath(name); 339 } 340 if (localConfig != null) { 341 return localConfig.getAbsolutePath(); 342 } 343 // Can not find local config. 344 if (parentName == null) { 345 throw new ConfigurationException( 346 String.format("Can not find local config %s.", name)); 347 348 } else { 349 throw new ConfigurationException( 350 String.format( 351 "Bundled config '%s' is including a config '%s' that's neither " 352 + "local nor bundled.", 353 parentName, name)); 354 } 355 } 356 try { 357 // Local configs' include should be relative to their parent's path. 358 String parentRoot = new File(parentName).getParentFile().getCanonicalPath(); 359 return getAbsolutePath(parentRoot, name); 360 } catch (IOException e) { 361 throw new ConfigurationException(e.getMessage(), e.getCause()); 362 } 363 } 364 365 /** 366 * Configs that are bundled inside the tradefed.jar can only include other configs also 367 * bundled inside tradefed.jar. However, local (external) configs can include both local 368 * (external) and bundled configs. 369 */ 370 @Override loadIncludedConfiguration( ConfigurationDef def, String parentName, String name, String deviceTagObject, Map<String, String> templateMap, Set<String> templateSeen)371 public void loadIncludedConfiguration( 372 ConfigurationDef def, 373 String parentName, 374 String name, 375 String deviceTagObject, 376 Map<String, String> templateMap, 377 Set<String> templateSeen) 378 throws ConfigurationException { 379 380 String config_name = findConfigName(name, parentName); 381 mConfigGraph.addEdge(parentName, config_name); 382 // If the inclusion of configurations is a cycle we throw an exception. 383 if (!mConfigGraph.isDag()) { 384 CLog.e("%s", mConfigGraph); 385 throw new ConfigurationException(String.format( 386 "Circular configuration include: config '%s' is already included", 387 config_name)); 388 } 389 loadConfiguration(config_name, def, deviceTagObject, templateMap, templateSeen); 390 } 391 392 /** 393 * Loads a configuration. 394 * 395 * @param name the name of a built-in configuration to load or a file path to configuration 396 * file to load 397 * @param def the loaded {@link ConfigurationDef} 398 * @param deviceTagObject name of the current deviceTag if we are loading from a config 399 * inside an <include>. Null otherwise. 400 * @param templateMap map from template-include names to their respective concrete 401 * configuration files 402 * @param templateSeen set of template placeholder name already encountered 403 * @throws ConfigurationException if a configuration with given name/file path cannot be 404 * loaded or parsed 405 */ loadConfiguration( String name, ConfigurationDef def, String deviceTagObject, Map<String, String> templateMap, Set<String> templateSeen)406 void loadConfiguration( 407 String name, 408 ConfigurationDef def, 409 String deviceTagObject, 410 Map<String, String> templateMap, 411 Set<String> templateSeen) 412 throws ConfigurationException { 413 BufferedInputStream bufStream = getConfigStream(name); 414 String extension = FileUtil.getExtension(name); 415 switch (extension) { 416 case ".xml": 417 case ".config": 418 case "": 419 ConfigurationXmlParser parser = 420 new ConfigurationXmlParser(this, deviceTagObject); 421 parser.parse(def, name, bufStream, templateMap, templateSeen); 422 break; 423 case ".tf_yaml": 424 ConfigurationYamlParser yamlParser = new ConfigurationYamlParser(); 425 yamlParser.parse(def, name, bufStream, false); 426 break; 427 default: 428 throw new ConfigurationException( 429 String.format("The config format for %s is not supported.", name)); 430 } 431 trackConfig(name, def); 432 } 433 434 /** 435 * Track config for dynamic loading. Right now only local files are supported. 436 * 437 * @param name config's name 438 * @param def config's def. 439 */ trackConfig(String name, ConfigurationDef def)440 protected void trackConfig(String name, ConfigurationDef def) { 441 // Track local config source files 442 if (!isBundledConfig(name)) { 443 def.registerSource(new File(name)); 444 } 445 } 446 447 /** 448 * Should track the config's life cycle or not. 449 * 450 * @param name config's name 451 * @return <code>true</code> if the config is trackable, otherwise <code>false</code>. 452 */ isTrackableConfig(String name)453 protected boolean isTrackableConfig(String name) { 454 return !isBundledConfig(name); 455 } 456 457 /** 458 * {@inheritDoc} 459 */ 460 @Override isGlobalConfig()461 public boolean isGlobalConfig() { 462 return mIsGlobalConfig; 463 } 464 465 } 466 ConfigurationFactory()467 protected ConfigurationFactory() { 468 mConfigDefMap = new Hashtable<ConfigId, ConfigurationDef>(); 469 } 470 471 /** 472 * Get the singleton {@link IConfigurationFactory} instance. 473 */ getInstance()474 public static IConfigurationFactory getInstance() { 475 if (sInstance == null) { 476 sInstance = new ConfigurationFactory(); 477 } 478 return sInstance; 479 } 480 481 /** 482 * Retrieve the {@link ConfigurationDef} for the given name 483 * 484 * @param name the name of a built-in configuration to load or a file path to configuration file 485 * to load 486 * @return {@link ConfigurationDef} 487 * @throws ConfigurationException if an error occurred loading the config 488 */ getConfigurationDef( String name, boolean isGlobal, Map<String, String> templateMap)489 protected ConfigurationDef getConfigurationDef( 490 String name, boolean isGlobal, Map<String, String> templateMap) 491 throws ConfigurationException { 492 return new ConfigLoader(isGlobal).getConfigurationDef(name, templateMap); 493 } 494 495 /** 496 * {@inheritDoc} 497 */ 498 @Override createConfigurationFromArgs(String[] arrayArgs)499 public IConfiguration createConfigurationFromArgs(String[] arrayArgs) 500 throws ConfigurationException { 501 return createConfigurationFromArgs(arrayArgs, null); 502 } 503 504 /** 505 * {@inheritDoc} 506 */ 507 @Override createConfigurationFromArgs(String[] arrayArgs, List<String> unconsumedArgs)508 public IConfiguration createConfigurationFromArgs(String[] arrayArgs, 509 List<String> unconsumedArgs) throws ConfigurationException { 510 return createConfigurationFromArgs(arrayArgs, unconsumedArgs, null); 511 } 512 513 /** 514 * {@inheritDoc} 515 */ 516 @Override createConfigurationFromArgs(String[] arrayArgs, List<String> unconsumedArgs, IKeyStoreClient keyStoreClient)517 public IConfiguration createConfigurationFromArgs(String[] arrayArgs, 518 List<String> unconsumedArgs, IKeyStoreClient keyStoreClient) 519 throws ConfigurationException { 520 if (arrayArgs.length == 0) { 521 throw new ConfigurationException("Configuration to run was not specified"); 522 } 523 524 List<String> listArgs = new ArrayList<String>(arrayArgs.length); 525 // FIXME: Update parsing to not care about arg order. 526 String[] reorderedArrayArgs = reorderArgs(arrayArgs); 527 IConfiguration config = 528 internalCreateConfigurationFromArgs( 529 reorderedArrayArgs, listArgs, keyStoreClient, null); 530 config.setCommandLine(arrayArgs); 531 if (listArgs.contains("--" + CommandOptions.DRY_RUN_OPTION) 532 || listArgs.contains("--" + CommandOptions.NOISY_DRY_RUN_OPTION)) { 533 // In case of dry-run, we replace the KeyStore by a dry-run one. 534 CLog.w("dry-run detected, we are using a dryrun keystore"); 535 keyStoreClient = new DryRunKeyStore(); 536 } 537 final List<String> tmpUnconsumedArgs = config.setOptionsFromCommandLineArgs( 538 listArgs, keyStoreClient); 539 540 if (unconsumedArgs == null && tmpUnconsumedArgs.size() > 0) { 541 // (unconsumedArgs == null) is taken as a signal that the caller 542 // expects all args to 543 // be processed. 544 throw new ConfigurationException(String.format( 545 "Invalid arguments provided. Unprocessed arguments: %s", tmpUnconsumedArgs)); 546 } else if (unconsumedArgs != null) { 547 // Return the unprocessed args 548 unconsumedArgs.addAll(tmpUnconsumedArgs); 549 } 550 551 return config; 552 } 553 554 /** {@inheritDoc} */ 555 @Override createPartialConfigurationFromArgs( String[] arrayArgs, IKeyStoreClient keyStoreClient, Set<String> allowedObjects)556 public IConfiguration createPartialConfigurationFromArgs( 557 String[] arrayArgs, IKeyStoreClient keyStoreClient, Set<String> allowedObjects) 558 throws ConfigurationException { 559 if (arrayArgs.length == 0) { 560 throw new ConfigurationException("Configuration to run was not specified"); 561 } 562 563 List<String> listArgs = new ArrayList<String>(arrayArgs.length); 564 String[] reorderedArrayArgs = reorderArgs(arrayArgs); 565 IConfiguration config = 566 internalCreateConfigurationFromArgs( 567 reorderedArrayArgs, listArgs, keyStoreClient, allowedObjects); 568 config.setCommandLine(arrayArgs); 569 List<String> leftOver = 570 config.setBestEffortOptionsFromCommandLineArgs(listArgs, keyStoreClient); 571 CLog.d("Non-applied arguments: %s", leftOver); 572 return config; 573 } 574 575 /** 576 * Creates a {@link Configuration} from the name given in arguments. 577 * 578 * <p>Note will not populate configuration with values from options 579 * 580 * @param arrayArgs the full list of command line arguments, including the config name 581 * @param optionArgsRef an empty list, that will be populated with the option arguments left to 582 * be interpreted 583 * @param keyStoreClient {@link IKeyStoreClient} keystore client to use if any. 584 * @param allowedObjects config object that are allowed to be created. 585 * @return An {@link IConfiguration} object representing the configuration that was loaded 586 * @throws ConfigurationException 587 */ internalCreateConfigurationFromArgs( String[] arrayArgs, List<String> optionArgsRef, IKeyStoreClient keyStoreClient, Set<String> allowedObjects)588 private IConfiguration internalCreateConfigurationFromArgs( 589 String[] arrayArgs, 590 List<String> optionArgsRef, 591 IKeyStoreClient keyStoreClient, 592 Set<String> allowedObjects) 593 throws ConfigurationException { 594 final List<String> listArgs = new ArrayList<>(Arrays.asList(arrayArgs)); 595 // first arg is config name 596 final String configName = listArgs.remove(0); 597 598 Map<String, String> uniqueMap = 599 extractTemplates(configName, listArgs, optionArgsRef, keyStoreClient); 600 ConfigurationDef configDef = getConfigurationDef(configName, false, uniqueMap); 601 if (!uniqueMap.isEmpty()) { 602 // remove the bad ConfigDef from the cache. 603 for (ConfigId cid : mConfigDefMap.keySet()) { 604 if (mConfigDefMap.get(cid) == configDef) { 605 CLog.d("Cleaning the cache for this configdef"); 606 mConfigDefMap.remove(cid); 607 break; 608 } 609 } 610 throw new ConfigurationException( 611 String.format("Unused template:map parameters: %s", uniqueMap.toString())); 612 } 613 return configDef.createConfiguration(allowedObjects); 614 } 615 extractTemplates( String configName, List<String> listArgs, List<String> optionArgsRef, IKeyStoreClient keyStoreClient)616 private Map<String, String> extractTemplates( 617 String configName, 618 List<String> listArgs, 619 List<String> optionArgsRef, 620 IKeyStoreClient keyStoreClient) 621 throws ConfigurationException { 622 final String extension = FileUtil.getExtension(configName); 623 switch (extension) { 624 case ".xml": 625 case ".config": 626 case "": 627 final ConfigurationXmlParserSettings parserSettings = 628 new ConfigurationXmlParserSettings(); 629 final ArgsOptionParser templateArgParser = new ArgsOptionParser(parserSettings); 630 if (keyStoreClient != null) { 631 templateArgParser.setKeyStore(keyStoreClient); 632 } 633 optionArgsRef.addAll(templateArgParser.parseBestEffort(listArgs)); 634 // Check that the same template is not attempted to be loaded twice. 635 for (String key : parserSettings.templateMap.keySet()) { 636 if (parserSettings.templateMap.get(key).size() > 1) { 637 throw new ConfigurationException( 638 String.format( 639 "More than one template specified for key '%s'", key)); 640 } 641 } 642 return parserSettings.templateMap.getUniqueMap(); 643 case ".tf_yaml": 644 // We parse the arguments but don't support template for YAML 645 final ArgsOptionParser allArgsParser = new ArgsOptionParser(); 646 if (keyStoreClient != null) { 647 allArgsParser.setKeyStore(keyStoreClient); 648 } 649 optionArgsRef.addAll(allArgsParser.parseBestEffort(listArgs)); 650 return new HashMap<>(); 651 default: 652 return new HashMap<>(); 653 } 654 } 655 656 /** 657 * {@inheritDoc} 658 */ 659 @Override createGlobalConfigurationFromArgs(String[] arrayArgs, List<String> remainingArgs)660 public IGlobalConfiguration createGlobalConfigurationFromArgs(String[] arrayArgs, 661 List<String> remainingArgs) throws ConfigurationException { 662 List<String> listArgs = new ArrayList<String>(arrayArgs.length); 663 IGlobalConfiguration config = internalCreateGlobalConfigurationFromArgs(arrayArgs, 664 listArgs); 665 remainingArgs.addAll(config.setOptionsFromCommandLineArgs(listArgs)); 666 667 return config; 668 } 669 670 /** 671 * Creates a {@link GlobalConfiguration} from the name given in arguments. 672 * <p/> 673 * Note will not populate configuration with values from options 674 * 675 * @param arrayArgs the full list of command line arguments, including the config name 676 * @param optionArgsRef an empty list, that will be populated with the 677 * remaining option arguments 678 * @return a {@link IGlobalConfiguration} created from the args 679 * @throws ConfigurationException 680 */ internalCreateGlobalConfigurationFromArgs(String[] arrayArgs, List<String> optionArgsRef)681 private IGlobalConfiguration internalCreateGlobalConfigurationFromArgs(String[] arrayArgs, 682 List<String> optionArgsRef) throws ConfigurationException { 683 if (arrayArgs.length == 0) { 684 throw new ConfigurationException("Configuration to run was not specified"); 685 } 686 optionArgsRef.addAll(Arrays.asList(arrayArgs)); 687 // first arg is config name 688 final String configName = optionArgsRef.remove(0); 689 ConfigurationDef configDef = getConfigurationDef(configName, true, null); 690 IGlobalConfiguration config = configDef.createGlobalConfiguration(); 691 config.setOriginalConfig(configName); 692 config.setConfigurationFactory(this); 693 return config; 694 } 695 696 /** 697 * {@inheritDoc} 698 */ 699 @Override printHelp(PrintStream out)700 public void printHelp(PrintStream out) { 701 try { 702 loadAllConfigs(true); 703 } catch (ConfigurationException e) { 704 // ignore, should never happen 705 } 706 // sort the configs by name before displaying 707 SortedSet<ConfigurationDef> configDefs = new TreeSet<ConfigurationDef>( 708 new ConfigDefComparator()); 709 configDefs.addAll(mConfigDefMap.values()); 710 for (ConfigurationDef def : configDefs) { 711 out.printf(" %s: %s", def.getName(), def.getDescription()); 712 out.println(); 713 } 714 } 715 716 /** 717 * {@inheritDoc} 718 */ 719 @Override getConfigList()720 public List<String> getConfigList() { 721 return getConfigList(null, true); 722 } 723 724 /** {@inheritDoc} */ 725 @Override getConfigList(String subPath, boolean loadFromEnv)726 public List<String> getConfigList(String subPath, boolean loadFromEnv) { 727 Set<String> configNames = getConfigSetFromClasspath(subPath); 728 if (loadFromEnv) { 729 // list config on variable path too 730 configNames.addAll(getConfigNamesFromTestCases(subPath)); 731 } 732 // sort the configs by name before adding to list 733 SortedSet<String> configDefs = new TreeSet<String>(); 734 configDefs.addAll(configNames); 735 List<String> configs = new ArrayList<String>(); 736 configs.addAll(configDefs); 737 return configs; 738 } 739 740 /** 741 * Private helper to get the full set of configurations. 742 */ getConfigSetFromClasspath(String subPath)743 private Set<String> getConfigSetFromClasspath(String subPath) { 744 ClassPathScanner cpScanner = new ClassPathScanner(); 745 return cpScanner.getClassPathEntries(new ConfigClasspathFilter(subPath)); 746 } 747 748 /** 749 * Helper to get the test config files from test cases directories from build output. 750 * 751 * @param subPath where to look for configuration. Can be null. 752 */ 753 @VisibleForTesting getConfigNamesFromTestCases(String subPath)754 Set<String> getConfigNamesFromTestCases(String subPath) { 755 return ConfigurationUtil.getConfigNamesFromDirs(subPath, getExternalTestCasesDirs()); 756 } 757 758 @VisibleForTesting getConfigSetFromClasspathFromJar(String subPath)759 Map<String, String> getConfigSetFromClasspathFromJar(String subPath) { 760 ClassPathScanner cpScanner = new ClassPathScanner(); 761 return cpScanner.getClassPathEntriesFromJar(new ConfigClasspathFilter(subPath)); 762 } 763 764 /** 765 * Loads all configurations found in classpath and test cases directories. 766 * 767 * @param discardExceptions true if any ConfigurationException should be ignored. 768 * @throws ConfigurationException 769 */ loadAllConfigs(boolean discardExceptions)770 public void loadAllConfigs(boolean discardExceptions) throws ConfigurationException { 771 ByteArrayOutputStream baos = new ByteArrayOutputStream(); 772 PrintStream ps = new PrintStream(baos); 773 boolean failed = false; 774 Set<String> configNames = getConfigSetFromClasspath(null); 775 // TODO: split the configs into two lists, one from the jar packages and one from test 776 // cases directories. 777 configNames.addAll(getConfigNamesFromTestCases(null)); 778 for (String configName : configNames) { 779 final ConfigId configId = new ConfigId(configName); 780 try { 781 ConfigurationDef configDef = attemptLoad(configId, null); 782 mConfigDefMap.put(configId, configDef); 783 } catch (ConfigurationException e) { 784 ps.printf("Failed to load %s: %s", configName, e.getMessage()); 785 ps.println(); 786 failed = true; 787 } 788 } 789 if (failed) { 790 if (discardExceptions) { 791 CLog.e("Failure loading configs"); 792 CLog.e(baos.toString()); 793 } else { 794 throw new ConfigurationException(baos.toString()); 795 } 796 } 797 } 798 799 /** 800 * Helper to load a configuration. 801 */ attemptLoad(ConfigId configId, Map<String, String> templateMap)802 private ConfigurationDef attemptLoad(ConfigId configId, Map<String, String> templateMap) 803 throws ConfigurationException { 804 ConfigurationDef configDef = null; 805 try { 806 configDef = getConfigurationDef(configId.name, false, templateMap); 807 return configDef; 808 } catch (TemplateResolutionError tre) { 809 // When a template does not have a default, we try again with known good template 810 // to make sure file formatting at the very least is fine. 811 Map<String, String> fakeTemplateMap = new HashMap<String, String>(); 812 if (templateMap != null) { 813 fakeTemplateMap.putAll(templateMap); 814 } 815 fakeTemplateMap.put(tre.getTemplateKey(), DRY_RUN_TEMPLATE_CONFIG); 816 // We go recursively in case there are several template to dry run. 817 return attemptLoad(configId, fakeTemplateMap); 818 } 819 } 820 821 /** 822 * {@inheritDoc} 823 */ 824 @Override printHelpForConfig(String[] args, boolean importantOnly, PrintStream out)825 public void printHelpForConfig(String[] args, boolean importantOnly, PrintStream out) { 826 try { 827 IConfiguration config = 828 internalCreateConfigurationFromArgs( 829 args, new ArrayList<String>(args.length), null, null); 830 config.printCommandUsage(importantOnly, out); 831 } catch (ConfigurationException e) { 832 // config must not be specified. Print generic help 833 printHelp(out); 834 } 835 } 836 837 /** 838 * {@inheritDoc} 839 */ 840 @Override dumpConfig(String configName, PrintStream out)841 public void dumpConfig(String configName, PrintStream out) { 842 try { 843 InputStream configStream = getConfigStream(configName); 844 StreamUtil.copyStreams(configStream, out); 845 } catch (ConfigurationException e) { 846 Log.e(LOG_TAG, e); 847 } catch (IOException e) { 848 Log.e(LOG_TAG, e); 849 } 850 } 851 852 /** 853 * Return the path prefix of config xml files on classpath 854 * 855 * <p>Exposed so unit tests can mock. 856 * 857 * @return {@link String} path with trailing / 858 */ getConfigPrefix()859 protected String getConfigPrefix() { 860 return CONFIG_PREFIX; 861 } 862 863 /** 864 * Loads an InputStream for given config name 865 * 866 * @param name the configuration name to load 867 * @return a {@link BufferedInputStream} for reading config contents 868 * @throws ConfigurationException if config could not be found 869 */ getConfigStream(String name)870 protected BufferedInputStream getConfigStream(String name) throws ConfigurationException { 871 InputStream configStream = getBundledConfigStream(name); 872 if (configStream == null) { 873 // now try to load from file 874 try { 875 configStream = new FileInputStream(name); 876 } catch (FileNotFoundException e) { 877 throw new ConfigurationException(String.format("Could not find configuration '%s'", 878 name)); 879 } 880 } 881 // buffer input for performance - just in case config file is large 882 return new BufferedInputStream(configStream); 883 } 884 getBundledConfigStream(String name)885 protected InputStream getBundledConfigStream(String name) { 886 String extension = FileUtil.getExtension(name); 887 if (Strings.isNullOrEmpty(extension)) { 888 // If the default name doesn't have an extension, search all possible extensions. 889 for (String supportExtension : SUPPORTED_EXTENSIONS) { 890 InputStream res = 891 getClass() 892 .getResourceAsStream( 893 String.format( 894 "/%s%s%s", 895 getConfigPrefix(), name, supportExtension)); 896 if (res != null) { 897 return res; 898 } 899 } 900 return null; 901 } 902 // Check directly with extension if it has one. 903 return getClass().getResourceAsStream(String.format("/%s%s", getConfigPrefix(), name)); 904 } 905 906 /** 907 * Utility method that checks that all configs can be loaded, parsed, and 908 * all option values set. 909 * Only exposed so that depending project can validate their configs. 910 * Should not be exposed in the console. 911 * 912 * @throws ConfigurationException if one or more configs failed to load 913 */ loadAndPrintAllConfigs()914 public void loadAndPrintAllConfigs() throws ConfigurationException { 915 loadAllConfigs(false); 916 boolean failed = false; 917 ByteArrayOutputStream baos = new ByteArrayOutputStream(); 918 PrintStream ps = new PrintStream(baos); 919 920 for (ConfigurationDef def : mConfigDefMap.values()) { 921 try { 922 def.createConfiguration().printCommandUsage(false, 923 new PrintStream(StreamUtil.nullOutputStream())); 924 } catch (ConfigurationException e) { 925 if (e.getCause() != null && 926 e.getCause() instanceof ClassNotFoundException) { 927 ClassNotFoundException cnfe = (ClassNotFoundException) e.getCause(); 928 String className = cnfe.getLocalizedMessage(); 929 // Some Cts configs are shipped with Trade Federation, we exclude those from 930 // the failure since these packages are not available for loading. 931 if (className != null && className.startsWith("com.android.cts.")) { 932 CLog.w("Could not confirm %s: %s because not part of Trade Federation " 933 + "packages.", def.getName(), e.getMessage()); 934 continue; 935 } 936 } else if (Pattern.matches(CONFIG_ERROR_PATTERN, e.getMessage())) { 937 // If options are inside configuration object tag we are able to validate them 938 if (!e.getMessage().contains("com.android.") && 939 !e.getMessage().contains("com.google.android.")) { 940 // We cannot confirm if an option is indeed missing since a template of 941 // option only is possible to avoid repetition in configuration with the 942 // same base. 943 CLog.w("Could not confirm %s: %s", def.getName(), e.getMessage()); 944 continue; 945 } 946 } 947 ps.printf("Failed to print %s: %s", def.getName(), e.getMessage()); 948 ps.println(); 949 failed = true; 950 } 951 } 952 if (failed) { 953 throw new ConfigurationException(baos.toString()); 954 } 955 } 956 957 /** 958 * Exposed for testing. Return a copy of the Map. 959 */ getMapConfig()960 protected Map<ConfigId, ConfigurationDef> getMapConfig() { 961 // We return a copy to ensure it is not modified outside 962 return new HashMap<ConfigId, ConfigurationDef>(mConfigDefMap); 963 } 964 965 /** In some particular case, we need to clear the map. */ 966 @VisibleForTesting clearMapConfig()967 public void clearMapConfig() { 968 mConfigDefMap.clear(); 969 } 970 971 /** Reorder the args so that template:map args are all moved to the front. */ 972 @VisibleForTesting reorderArgs(String[] args)973 protected String[] reorderArgs(String[] args) { 974 List<String> nonTemplateArgs = new ArrayList<String>(); 975 List<String> reorderedArgs = new ArrayList<String>(); 976 String[] reorderedArgsArray = new String[args.length]; 977 String arg; 978 979 // First arg is the config. 980 if (args.length > 0) { 981 reorderedArgs.add(args[0]); 982 } 983 984 // Split out the template and non-template args so we can add 985 // non-template args at the end while maintaining their order. 986 for (int i = 1; i < args.length; i++) { 987 arg = args[i]; 988 if (arg.equals("--template:map")) { 989 // We need to account for these two types of template:map args. 990 // --template:map tm=tm1 991 // --template:map tm tm1 992 reorderedArgs.add(arg); 993 for (int j = i + 1; j < args.length; j++) { 994 if (args[j].startsWith("-")) { 995 break; 996 } else { 997 reorderedArgs.add(args[j]); 998 i++; 999 } 1000 } 1001 } else { 1002 nonTemplateArgs.add(arg); 1003 } 1004 } 1005 reorderedArgs.addAll(nonTemplateArgs); 1006 return reorderedArgs.toArray(reorderedArgsArray); 1007 } 1008 } 1009