1 /*
2  * Copyright (C) 2017 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 package com.android.tradefed.testtype.suite;
17 
18 import com.android.tradefed.config.Configuration;
19 import com.android.tradefed.config.ConfigurationException;
20 import com.android.tradefed.config.IConfiguration;
21 import com.android.tradefed.config.IDeviceConfiguration;
22 import com.android.tradefed.config.OptionCopier;
23 import com.android.tradefed.invoker.TestInformation;
24 import com.android.tradefed.log.LogUtil.CLog;
25 import com.android.tradefed.targetprep.ITargetPreparer;
26 import com.android.tradefed.targetprep.multi.IMultiTargetPreparer;
27 import com.android.tradefed.testtype.IAbiReceiver;
28 import com.android.tradefed.testtype.IRemoteTest;
29 import com.android.tradefed.testtype.IShardableTest;
30 
31 import java.lang.reflect.InvocationTargetException;
32 import java.util.ArrayList;
33 import java.util.Collection;
34 import java.util.LinkedHashMap;
35 import java.util.List;
36 import java.util.Map;
37 import java.util.Map.Entry;
38 
39 /**
40  * Helper to split a list of modules represented by {@link IConfiguration} into a list of execution
41  * units represented by {@link ModuleDefinition}.
42  *
43  * <p>Each configuration may generate 1 or more {@link ModuleDefinition} depending on its options
44  * and test types:
45  *
46  * <ul>
47  *   <li>A non-shardable {@link IConfiguration} will generate a single {@link ModuleDefinition}.
48  *   <li>A shardable {@link IConfiguration} will generate a number of ModuleDefinition linked to the
49  *       {@link IRemoteTest} properties:
50  *       <ul>
51  *         <li>A non - {@link IShardableTest} will generate a single ModuleDefinition.
52  *         <li>A {@link IShardableTest} generates one ModuleDefinition per tests returned by {@link
53  *             IShardableTest#split()}.
54  *       </ul>
55  *
56  * </ul>
57  */
58 public class ModuleSplitter {
59 
60     /**
61      * Create a List of executable unit {@link ModuleDefinition}s based on the map of configuration
62      * that was loaded.
63      *
64      * @param testInfo the current {@link TestInformation} to proceed with sharding.
65      * @param runConfig {@link LinkedHashMap} loaded from {@link ITestSuite#loadTests()}.
66      * @param shardCount a shard count hint to help with sharding.
67      * @param dynamicModule Whether or not module can be shared in pool or must be independent
68      *     (strict sharding).
69      * @param intraModuleSharding Whether or not to shard within the modules.
70      * @return List of {@link ModuleDefinition}
71      */
splitConfiguration( TestInformation testInfo, LinkedHashMap<String, IConfiguration> runConfig, int shardCount, boolean dynamicModule, boolean intraModuleSharding)72     public static List<ModuleDefinition> splitConfiguration(
73             TestInformation testInfo,
74             LinkedHashMap<String, IConfiguration> runConfig,
75             int shardCount,
76             boolean dynamicModule,
77             boolean intraModuleSharding) {
78         if (dynamicModule) {
79             // We maximize the sharding for dynamic to reduce time difference between first and
80             // last shard as much as possible. Overhead is low due to our test pooling.
81             shardCount *= 2;
82         }
83         List<ModuleDefinition> runModules = new ArrayList<>();
84         for (Entry<String, IConfiguration> configMap : runConfig.entrySet()) {
85             // Check that it's a valid configuration for suites, throw otherwise.
86             ValidateSuiteConfigHelper.validateConfig(configMap.getValue());
87 
88             try {
89                 createAndAddModule(
90                         testInfo,
91                         runModules,
92                         configMap.getKey(),
93                         configMap.getValue(),
94                         shardCount,
95                         dynamicModule,
96                         intraModuleSharding);
97             } catch (RuntimeException e) {
98                 CLog.e("Exception while creating module for '%s'", configMap.getKey());
99                 throw e;
100             }
101         }
102         return runModules;
103     }
104 
createAndAddModule( TestInformation testInfo, List<ModuleDefinition> currentList, String moduleName, IConfiguration config, int shardCount, boolean dynamicModule, boolean intraModuleSharding)105     private static void createAndAddModule(
106             TestInformation testInfo,
107             List<ModuleDefinition> currentList,
108             String moduleName,
109             IConfiguration config,
110             int shardCount,
111             boolean dynamicModule,
112             boolean intraModuleSharding) {
113         List<IRemoteTest> tests = config.getTests();
114         // Get rid of the IRemoteTest reference on the shared configuration. It will not be used
115         // to run.
116         config.setTests(new ArrayList<>());
117         // If this particular configuration module is declared as 'not shardable' we take it whole
118         // but still split the individual IRemoteTest in a pool.
119         if (!intraModuleSharding
120                 || config.getConfigurationDescription().isNotShardable()
121                 || (!dynamicModule
122                         && config.getConfigurationDescription().isNotStrictShardable())) {
123             for (int i = 0; i < tests.size(); i++) {
124                 if (dynamicModule) {
125                     ModuleDefinition module =
126                             new ModuleDefinition(
127                                     moduleName,
128                                     tests,
129                                     clonePreparersMap(config),
130                                     clonePreparers(config.getMultiTargetPreparers()),
131                                     config);
132                     currentList.add(module);
133                 } else {
134                     addModuleToListFromSingleTest(currentList, tests.get(i), moduleName, config);
135                 }
136             }
137             clearPreparersFromConfig(config);
138             return;
139         }
140 
141         // If configuration is possibly shardable we attempt to shard it.
142         for (IRemoteTest test : tests) {
143             if (test instanceof IShardableTest) {
144                 Collection<IRemoteTest> shardedTests =
145                         ((IShardableTest) test).split(shardCount, testInfo);
146                 if (shardedTests != null) {
147                     // Test did shard we put the shard pool in ModuleDefinition which has a polling
148                     // behavior on the pool.
149                     if (dynamicModule) {
150                         for (int i = 0; i < shardCount; i++) {
151                             ModuleDefinition module =
152                                     new ModuleDefinition(
153                                             moduleName,
154                                             shardedTests,
155                                             clonePreparersMap(config),
156                                             clonePreparers(config.getMultiTargetPreparers()),
157                                             config);
158                             currentList.add(module);
159                         }
160                     } else {
161                         // We create independent modules with each sharded test.
162                         for (IRemoteTest moduleTest : shardedTests) {
163                             addModuleToListFromSingleTest(
164                                     currentList, moduleTest, moduleName, config);
165                         }
166                     }
167                     continue;
168                 }
169             }
170             // test is not shardable or did not shard
171             addModuleToListFromSingleTest(currentList, test, moduleName, config);
172         }
173         clearPreparersFromConfig(config);
174     }
175 
176     /**
177      * Helper to add a new {@link ModuleDefinition} to our list of Modules from a single {@link
178      * IRemoteTest}.
179      */
addModuleToListFromSingleTest( List<ModuleDefinition> currentList, IRemoteTest test, String moduleName, IConfiguration config)180     private static void addModuleToListFromSingleTest(
181             List<ModuleDefinition> currentList,
182             IRemoteTest test,
183             String moduleName,
184             IConfiguration config) {
185         List<IRemoteTest> testList = new ArrayList<>();
186         testList.add(test);
187         ModuleDefinition module =
188                 new ModuleDefinition(
189                         moduleName,
190                         testList,
191                         clonePreparersMap(config),
192                         clonePreparers(config.getMultiTargetPreparers()),
193                         config);
194         currentList.add(module);
195     }
196 
197     /**
198      * Deep clone a list of {@link ITargetPreparer} or {@link IMultiTargetPreparer}. We are ensured
199      * to find a default constructor with no arguments since that's the expectation from Tradefed
200      * when loading configuration. Cloning preparers is required since they may be stateful and we
201      * cannot share instance across devices.
202      */
clonePreparers(List<T> preparerList)203     private static <T> List<T> clonePreparers(List<T> preparerList) {
204         List<T> clones = new ArrayList<>();
205         for (T prep : preparerList) {
206             try {
207                 @SuppressWarnings("unchecked")
208                 T clone = (T) prep.getClass().getDeclaredConstructor().newInstance();
209                 OptionCopier.copyOptions(prep, clone);
210                 // Ensure we copy the Abi too.
211                 if (clone instanceof IAbiReceiver) {
212                     ((IAbiReceiver) clone).setAbi(((IAbiReceiver) prep).getAbi());
213                 }
214                 clones.add(clone);
215             } catch (InstantiationException
216                     | IllegalAccessException
217                     | ConfigurationException
218                     | InvocationTargetException
219                     | NoSuchMethodException e) {
220                 throw new RuntimeException(e);
221             }
222         }
223         return clones;
224     }
225 
226     /** Deep cloning of potentially multi-device preparers. */
clonePreparersMap(IConfiguration config)227     private static Map<String, List<ITargetPreparer>> clonePreparersMap(IConfiguration config) {
228         Map<String, List<ITargetPreparer>> res = new LinkedHashMap<>();
229         for (IDeviceConfiguration holder : config.getDeviceConfig()) {
230             List<ITargetPreparer> preparers = new ArrayList<>();
231             res.put(holder.getDeviceName(), preparers);
232             preparers.addAll(clonePreparers(holder.getTargetPreparers()));
233         }
234         return res;
235     }
236 
clearPreparersFromConfig(IConfiguration config)237     private static void clearPreparersFromConfig(IConfiguration config) {
238         try {
239             for (IDeviceConfiguration holder : config.getDeviceConfig()) {
240                 holder.removeObjectType(Configuration.TARGET_PREPARER_TYPE_NAME);
241             }
242             config.setMultiTargetPreparers(new ArrayList<>());
243         } catch (ConfigurationException e) {
244             throw new RuntimeException(e);
245         }
246     }
247 }
248