1 /*
2  * Copyright (C) 2018 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 package com.android.game.qualification.testtype;
18 
19 import com.android.game.qualification.ApkInfo;
20 import com.android.game.qualification.CertificationRequirements;
21 import com.android.game.qualification.GameCoreConfiguration;
22 import com.android.game.qualification.GameCoreConfigurationXmlParser;
23 import com.android.game.qualification.metric.BaseGameQualificationMetricCollector;
24 import com.android.game.qualification.reporter.GameQualificationResultReporter;
25 import com.android.game.qualification.test.PerformanceTest;
26 import com.android.tradefed.config.IConfiguration;
27 import com.android.tradefed.config.IConfigurationReceiver;
28 import com.android.tradefed.config.Option;
29 import com.android.tradefed.device.DeviceNotAvailableException;
30 import com.android.tradefed.device.ITestDevice;
31 import com.android.tradefed.device.metric.IMetricCollector;
32 import com.android.tradefed.device.metric.IMetricCollectorReceiver;
33 import com.android.tradefed.invoker.IInvocationContext;
34 import com.android.tradefed.log.LogUtil.CLog;
35 import com.android.tradefed.metrics.proto.MetricMeasurement;
36 import com.android.tradefed.result.ITestInvocationListener;
37 import com.android.tradefed.result.TestDescription;
38 import com.android.tradefed.testtype.IDeviceTest;
39 import com.android.tradefed.testtype.IInvocationContextReceiver;
40 import com.android.tradefed.testtype.IRemoteTest;
41 import com.android.tradefed.testtype.IShardableTest;
42 import com.android.tradefed.testtype.ITestFilterReceiver;
43 
44 import com.google.common.io.ByteStreams;
45 
46 import org.junit.AssumptionViolatedException;
47 import org.xml.sax.SAXException;
48 
49 import java.io.File;
50 import java.io.FileNotFoundException;
51 import java.io.FileOutputStream;
52 import java.io.IOException;
53 import java.io.InputStream;
54 import java.io.OutputStream;
55 import java.util.ArrayList;
56 import java.util.Collection;
57 import java.util.HashMap;
58 import java.util.HashSet;
59 import java.util.List;
60 import java.util.Set;
61 
62 import javax.xml.parsers.ParserConfigurationException;
63 
64 public class GameQualificationHostsideController implements
65         IShardableTest,
66         IDeviceTest,
67         IMetricCollectorReceiver,
68         IInvocationContextReceiver,
69         IConfigurationReceiver,
70         ITestFilterReceiver {
71     // Package and class of the device side test.
72     public static final String PACKAGE = "com.android.game.qualification.device";
73     public static final String CLASS = PACKAGE + ".GameQualificationTest";
74 
75     private ITestDevice mDevice;
76     private IConfiguration mConfiguration = null;
77     private GameCoreConfiguration mGameCoreConfiguration = null;
78     private List<ApkInfo> mApks = null;
79     private File mApkInfoFile;
80     private Collection<IMetricCollector> mCollectors;
81     private GameQualificationResultReporter mResultReporter;
82     private IInvocationContext mContext;
83     private ArrayList<BaseGameQualificationMetricCollector> mAGQMetricCollectors;
84     private Set<String> mIncludeFilters = new HashSet<>();
85     private Set<String> mExcludeFilters = new HashSet<>();
86 
87     @Override
setDevice(ITestDevice device)88     public void setDevice(ITestDevice device) {
89         mDevice = device;
90     }
91 
92     @Override
getDevice()93     public ITestDevice getDevice() {
94         return mDevice;
95     }
96 
97     @Option(name = "apk-info",
98             description = "An XML file describing the list of APKs for qualifications.",
99             importance = Option.Importance.ALWAYS)
100     private String mApkInfoFileName;
101 
102     @Option(name = "apk-dir",
103             description =
104                     "Directory contains the APKs for qualifications.  If --apk-info is not "
105                             + "specified and a file named 'apk-info.xml' exists in --apk-dir, that "
106                             + "file will be used as the apk-info.",
107             importance = Option.Importance.ALWAYS)
108     private String mApkDir;
109 
getApkDir()110     private String getApkDir() {
111         if (mApkDir == null) {
112             String out = System.getenv("ANDROID_PRODUCT_OUT");
113             if (out != null) {
114                 CLog.i("--apk-dir was not set, looking in ANDROID_PRODUCT_OUT(%s) for apk files.",
115                         out);
116                 mApkDir = out + "/data/app";
117             } else {
118                 CLog.i("--apk-dir and ANDROID_PRODUCT_OUT was not set, looking in current "
119                         + "directory(%s) for apk files.",
120                         new File(".").getAbsolutePath());
121                 mApkDir = ".";
122             }
123         }
124         return mApkDir;
125     }
126 
127     @Override
addIncludeFilter(String filter)128     public void addIncludeFilter(String filter) {
129         mIncludeFilters.add(filter);
130     }
131 
132     @Override
addAllIncludeFilters(Set<String> filters)133     public void addAllIncludeFilters(Set<String> filters) {
134         mIncludeFilters.addAll(filters);
135     }
136 
137     @Override
addExcludeFilter(String filter)138     public void addExcludeFilter(String filter) {
139         mExcludeFilters.add(filter);
140     }
141 
142     @Override
addAllExcludeFilters(Set<String> filters)143     public void addAllExcludeFilters(Set<String> filters) {
144         mExcludeFilters.addAll(filters);
145     }
146 
147     @Override
getIncludeFilters()148     public Set<String> getIncludeFilters() {
149         return mIncludeFilters;
150     }
151 
152     @Override
getExcludeFilters()153     public Set<String> getExcludeFilters() {
154         return mExcludeFilters;
155     }
156 
157     @Override
clearIncludeFilters()158     public void clearIncludeFilters() {
159         mIncludeFilters.clear();
160     }
161 
162     @Override
clearExcludeFilters()163     public void clearExcludeFilters() {
164         mExcludeFilters.clear();
165     }
166 
167     @Override
setMetricCollectors(List<IMetricCollector> list)168     public void setMetricCollectors(List<IMetricCollector> list) {
169         mCollectors = list;
170         mAGQMetricCollectors = new ArrayList<>();
171         for (IMetricCollector collector : list) {
172             if (collector instanceof BaseGameQualificationMetricCollector) {
173                 mAGQMetricCollectors.add((BaseGameQualificationMetricCollector) collector);
174             }
175         }
176     }
177 
178 
179     @Override
setInvocationContext(IInvocationContext iInvocationContext)180     public void setInvocationContext(IInvocationContext iInvocationContext) {
181         mContext = iInvocationContext;
182     }
183 
184     @Override
setConfiguration(IConfiguration configuration)185     public void setConfiguration(IConfiguration configuration) {
186         mConfiguration = configuration;
187     }
188 
189     @Override
split(int shardCountHint)190     public Collection<IRemoteTest> split(int shardCountHint) {
191         initApkList();
192         List<IRemoteTest> shards = new ArrayList<>();
193         for(int i = 0; i < shardCountHint; i++) {
194             if (i >= mApks.size()) {
195                 break;
196             }
197             List<ApkInfo> apkInfo = new ArrayList<>();
198             for(int j = i; j < mApks.size(); j += shardCountHint) {
199                 apkInfo.add(mApks.get(j));
200             }
201             GameQualificationHostsideController shard = new GameQualificationHostsideController();
202             shard.mApks = apkInfo;
203             shard.mApkDir = getApkDir();
204             shard.mApkInfoFileName = mApkInfoFileName;
205             shard.mApkInfoFile = mApkInfoFile;
206 
207             shards.add(shard);
208         }
209         return shards;
210     }
211 
212     @Override
run(ITestInvocationListener listener)213     public void run(ITestInvocationListener listener) throws DeviceNotAvailableException {
214         // Configuration can null if trigger by TEST_MAPPING.
215         if (mConfiguration == null) {
216             return;
217         }
218         // Find result reporter
219         if (mResultReporter == null) {
220             for (ITestInvocationListener testListener
221                     : mConfiguration.getTestInvocationListeners()) {
222                 if (testListener instanceof GameQualificationResultReporter) {
223                     mResultReporter = (GameQualificationResultReporter) testListener;
224                 }
225             }
226         }
227 
228         assert !(mAGQMetricCollectors.isEmpty());
229         for (IMetricCollector collector : mCollectors) {
230             listener = collector.init(mContext, listener);
231         }
232 
233         for (BaseGameQualificationMetricCollector collector : mAGQMetricCollectors) {
234             collector.setDevice(getDevice());
235         }
236 
237         HashMap<String, MetricMeasurement.Metric> runMetrics = new HashMap<>();
238 
239         initApkList();
240         getDevice().pushFile(mApkInfoFile, ApkInfo.APK_LIST_LOCATION);
241 
242         long startTime = System.currentTimeMillis();
243         listener.testRunStarted("gamequalification", mApks.size());
244 
245         for (ApkInfo apk : mApks) {
246             PerformanceTest test =
247                     new PerformanceTest(
248                             getDevice(),
249                             listener,
250                             mAGQMetricCollectors,
251                             apk,
252                             getApkDir(),
253                             mApkInfoFile.getParentFile());
254             for (BaseGameQualificationMetricCollector collector : mAGQMetricCollectors) {
255                 collector.setApkInfo(apk);
256                 collector.setCertificationRequirements(
257                         mGameCoreConfiguration.findCertificationRequirements(apk.getName()));
258             }
259             for (PerformanceTest.Test t : PerformanceTest.Test.values()) {
260                 TestDescription identifier = new TestDescription(CLASS, t.getName() + "[" + apk.getName() + "]");
261                 if (t.isEnableCollectors()) {
262                     for (BaseGameQualificationMetricCollector collector : mAGQMetricCollectors) {
263                         collector.enable();
264                     }
265                     if (mResultReporter != null) {
266                         CertificationRequirements req =
267                                 mGameCoreConfiguration.findCertificationRequirements(apk.getName());
268                         if (req != null) {
269                             mResultReporter.putRequirements(identifier, req);
270                         }
271                     }
272                 }
273                 listener.testStarted(identifier);
274                 try {
275                     t.getMethod().run(test);
276                 } catch(AssumptionViolatedException e) {
277                     listener.testAssumptionFailure(identifier, e.getMessage());
278                 } catch (Error | Exception e) {
279                     test.failed();
280                     listener.testFailed(identifier, e.getMessage());
281                 }
282                 listener.testEnded(identifier, new HashMap<String, MetricMeasurement.Metric>());
283 
284                 if (t.isEnableCollectors()) {
285                     for (BaseGameQualificationMetricCollector collector : mAGQMetricCollectors) {
286                         if (collector.hasError()) {
287                             listener.testFailed(identifier, collector.getErrorMessage());
288                         }
289                         collector.disable();
290                     }
291                 }
292             }
293         }
294         listener.testRunEnded(System.currentTimeMillis() - startTime, runMetrics);
295     }
296 
initApkList()297     private void initApkList() {
298         if (mApks != null) {
299             return;
300         }
301 
302         // Find an apk info file.  The priorities are:
303         // 1. Use the specified apk-info if available.
304         // 2. Use 'apk-info.xml' if there is one in the apk-dir directory.
305         // 3. Use the default apk-info.xml in res.
306         if (mApkInfoFileName != null) {
307             mApkInfoFile = new File(mApkInfoFileName);
308         } else {
309             mApkInfoFile = new File(getApkDir(), "apk-info.xml");
310 
311             if (!mApkInfoFile.exists()) {
312                 String resource = "/com/android/game/qualification/apk-info.xml";
313                 try(InputStream inputStream = ApkInfo.class.getResourceAsStream(resource)) {
314                     if (inputStream == null) {
315                         throw new FileNotFoundException("Unable to find resource: " + resource);
316                     }
317                     mApkInfoFile = File.createTempFile("apk-info", ".xml");
318                     try (OutputStream ostream = new FileOutputStream(mApkInfoFile)) {
319                         ByteStreams.copy(inputStream, ostream);
320                     }
321                     mApkInfoFile.deleteOnExit();
322                 } catch (IOException e) {
323                     throw new RuntimeException(e);
324                 }
325             }
326         }
327         GameCoreConfigurationXmlParser parser = new GameCoreConfigurationXmlParser();
328         try {
329             mGameCoreConfiguration = parser.parse(mApkInfoFile);
330             mApks = mGameCoreConfiguration.getApkInfo();
331         } catch (IOException | ParserConfigurationException | SAXException e) {
332             throw new RuntimeException(e);
333         }
334     }
335 }
336