1 /*
2  * Copyright (C) 2018 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 package com.android.game.qualification.test;
17 
18 import static org.junit.Assert.assertFalse;
19 import static org.junit.Assert.assertNotNull;
20 import static org.junit.Assert.fail;
21 
22 import com.android.annotations.VisibleForTesting;
23 import com.android.ddmlib.testrunner.RemoteAndroidTestRunner;
24 import com.android.game.qualification.ApkInfo;
25 import com.android.game.qualification.ResultData;
26 import com.android.game.qualification.metric.BaseGameQualificationMetricCollector;
27 import com.android.game.qualification.proto.ResultDataProto;
28 import com.android.game.qualification.testtype.GameQualificationHostsideController;
29 import com.android.tradefed.device.DeviceNotAvailableException;
30 import com.android.tradefed.device.ITestDevice;
31 import com.android.tradefed.log.LogUtil.CLog;
32 import com.android.tradefed.result.CollectingTestListener;
33 import com.android.tradefed.result.ITestInvocationListener;
34 import com.android.tradefed.result.InputStreamSource;
35 import com.android.tradefed.result.LogDataType;
36 
37 import com.google.common.io.ByteStreams;
38 import com.google.common.io.Files;
39 
40 import org.junit.Assume;
41 
42 import java.awt.image.BufferedImage;
43 import java.io.ByteArrayOutputStream;
44 import java.io.File;
45 import java.io.FileInputStream;
46 import java.io.IOException;
47 import java.io.InputStream;
48 import java.nio.charset.StandardCharsets;
49 import java.util.Collection;
50 import java.util.concurrent.TimeUnit;
51 
52 import javax.imageio.ImageIO;
53 
54 /**
55  * Performance test designed to be used with {@link GameQualificationHostsideController}
56  *
57  * Tests must be enumerated with the {@link Test} enum and unlike junit tests, they will be run
58  * sequentially.
59  */
60 public class PerformanceTest {
61     private static final String AJUR_RUNNER = "androidx.test.runner.AndroidJUnitRunner";
62     private static final long DEFAULT_TEST_TIMEOUT_MS = 30 * 60 * 1000L; //30min
63     private static final long DEFAULT_MAX_TIMEOUT_TO_OUTPUT_MS = 30 * 60 * 1000L; //30min
64 
65     private ApkInfo mApk;
66     private String mApkDir;
67     private ITestDevice mDevice;
68     private Collection<BaseGameQualificationMetricCollector> mCollectors;
69     private ITestInvocationListener mListener;
70     private File mWorkingDirectory;
71     private boolean allTestsPassed = true;
72 
73     public interface TestMethod {
run(PerformanceTest test)74         void run(PerformanceTest test) throws Exception;
75     }
76 
77     public enum Test {
78         SETUP("setUp", PerformanceTest::setUp, false),
79         RUN("run", PerformanceTest::run, true),
80         SCREENSHOT("screenshotTest", PerformanceTest::testScreenshot, false),
81         TEARDOWN("tearDown", PerformanceTest::tearDown, false);
82 
83         private String mName;
84         private TestMethod mMethod;
85         private boolean mEnableCollectors;
86 
Test(String name, TestMethod method, boolean enableCollectors)87         Test(String name, TestMethod method, boolean enableCollectors) {
88             mName = name;
89             mMethod = method;
90             mEnableCollectors = enableCollectors;
91         }
92 
getName()93         public String getName() {
94             return mName;
95         }
96 
getMethod()97         public TestMethod getMethod() {
98             return mMethod;
99         }
100 
isEnableCollectors()101         public boolean isEnableCollectors() {
102             return mEnableCollectors;
103         }
104     }
105 
PerformanceTest( ITestDevice device, ITestInvocationListener listener, Collection<BaseGameQualificationMetricCollector> collectors, ApkInfo apk, String apkDir, File workingDirectory)106     public PerformanceTest(
107             ITestDevice device,
108             ITestInvocationListener listener,
109             Collection<BaseGameQualificationMetricCollector> collectors,
110             ApkInfo apk,
111             String apkDir,
112             File workingDirectory) {
113         mApk = apk;
114         mApkDir = apkDir;
115         mDevice = device;
116         mCollectors = collectors;
117         mListener = listener;
118         mWorkingDirectory = workingDirectory;
119     }
120 
failed()121     public void failed() {
122         this.allTestsPassed = false;
123     }
124 
125     // BEGIN TESTS
126 
setUp()127     private void setUp() throws DeviceNotAvailableException, IOException, InterruptedException {
128         if (mApk.getScript() != null) {
129             String cmd = mApk.getScript();
130             CLog.i(
131                     "Executing command: " + cmd + "\n"
132                             + "Working directory: " + mWorkingDirectory.getPath());
133             ProcessBuilder pb = new ProcessBuilder("sh", "-c", cmd);
134             pb.environment().put("ANDROID_SERIAL", mDevice.getSerialNumber());
135             pb.directory(mWorkingDirectory);
136             pb.redirectErrorStream(true);
137 
138             Process p = pb.start();
139             boolean finished = p.waitFor(30, TimeUnit.MINUTES);
140             if (!finished || p.exitValue() != 0) {
141                 ByteArrayOutputStream os = new ByteArrayOutputStream();
142                 ByteStreams.copy(p.getInputStream(), os);
143                 String output = os.toString(StandardCharsets.UTF_8.name());
144                 if (!finished) {
145                     output += "\n***TIMEOUT waiting for script to complete.***";
146                     p.destroy();
147                 }
148                 fail("Execution of setup script returned non-zero value:\n" + output);
149             }
150         }
151 
152         File apkFile = findApk(mApk.getFileName());
153         assertNotNull(
154                 String.format(
155                         "Missing APK.  Unable to find %s in %s.\n",
156                         mApk.getFileName(),
157                         mApkDir),
158                 apkFile);
159         CLog.i("Installing %s on %s.", apkFile.getName(), mDevice.getSerialNumber());
160         mDevice.installPackage(apkFile, true);
161     }
162 
run()163     private void run() throws DeviceNotAvailableException {
164         Assume.assumeTrue(allTestsPassed);
165         // APK Test.
166         assertFalse(
167                 "Unable to unlock device: " + mDevice.getDeviceDescriptor(),
168                 mDevice.getKeyguardState().isKeyguardShowing());
169 
170         File apkFile = findApk(mApk.getFileName());
171         assertNotNull(
172                 String.format(
173                         "Missing APK.  Unable to find %s in %s.\n",
174                         mApk.getFileName(),
175                         mApkDir),
176                 apkFile);
177 
178 
179         CollectingTestListener listener = new CollectingTestListener();
180         runDeviceTests(
181                 GameQualificationHostsideController.PACKAGE,
182                 GameQualificationHostsideController.CLASS,
183                 "run[" + mApk.getName() + "]",
184                 listener);
185         ResultDataProto.Result resultData = retrieveResultData();
186         for (BaseGameQualificationMetricCollector collector : mCollectors) {
187             collector.setDeviceResultData(resultData);
188         }
189         assertFalse(listener.hasFailedTests());
190     }
191 
testScreenshot()192     private void testScreenshot() throws IOException, DeviceNotAvailableException {
193         Assume.assumeTrue(allTestsPassed);
194         try (InputStreamSource screenSource = mDevice.getScreenshot()) {
195             mListener.testLog(
196                     String.format("screenshot-%s", mApk.getName()),
197                     LogDataType.PNG,
198                     screenSource);
199             try (InputStream stream = screenSource.createInputStream()) {
200                 stream.reset();
201                 assertFalse(
202                         "A screenshot was taken just after metric collection and it was black.",
203                         isImageBlack(stream));
204             }
205         } catch (IOException e) {
206             throw new IOException("Failed reading screenshot data:\n" + e.getMessage());
207         }
208     }
209 
tearDown()210     private void tearDown() throws DeviceNotAvailableException {
211         mDevice.uninstallPackage(mApk.getPackageName());
212     }
213 
214     // END TESTS
215 
216     /** Find an apk in the apk-dir directory */
findApk(String filename)217     private File findApk(String filename) {
218         File file = new File(mApkDir, filename);
219         if (file.exists()) {
220             return file;
221         }
222         // If a default sample app is named Sample.apk, it is outputted to
223         // $ANDROID_PRODUCT_OUT/data/app/Sample/Sample.apk.
224         file = new File(mApkDir, Files.getNameWithoutExtension(filename) + "/" + filename);
225         if (file.exists()) {
226             return file;
227         }
228         return null;
229     }
230 
231     /** Check if an image is black. */
232     @VisibleForTesting
isImageBlack(InputStream stream)233     static boolean isImageBlack(InputStream stream) throws IOException {
234         BufferedImage img = ImageIO.read(stream);
235         for (int i = 0; i < img.getWidth(); i++) {
236             // Only check the middle portion of the image to avoid status bar.
237             for (int j = img.getHeight() / 4; j < img.getHeight() * 3 / 4; j++) {
238                 int color = img.getRGB(i, j);
239                 // Check if pixel is non-black and not fully transparent.
240                 if ((color & 0x00ffffff) != 0 && (color >> 24) != 0) {
241                     return false;
242                 }
243             }
244         }
245         return true;
246     }
247 
retrieveResultData()248     private ResultDataProto.Result retrieveResultData() throws DeviceNotAvailableException {
249         File resultFile = mDevice.pullFileFromExternal(ResultData.RESULT_FILE_LOCATION);
250 
251         if (resultFile != null) {
252             try (InputStream inputStream = new FileInputStream(resultFile)) {
253                 return ResultDataProto.Result.parseFrom(inputStream);
254             } catch (IOException e) {
255                 throw new RuntimeException(e);
256             }
257         }
258         return null;
259     }
260 
261     /**
262      * Method to run an installed instrumentation package.
263      *
264      * @param pkgName the name of the package to run.
265      * @param testClassName the name of the test class to run.
266      * @param testMethodName the name of the method to run.
267      */
runDeviceTests(String pkgName, String testClassName, String testMethodName, CollectingTestListener listener)268     private void runDeviceTests(String pkgName, String testClassName, String testMethodName, CollectingTestListener listener)
269             throws DeviceNotAvailableException {
270         RemoteAndroidTestRunner testRunner =
271                 new RemoteAndroidTestRunner(pkgName, AJUR_RUNNER, mDevice.getIDevice());
272 
273         testRunner.setMethodName(testClassName, testMethodName);
274 
275         testRunner.addInstrumentationArg(
276                 "timeout_msec", Long.toString(DEFAULT_TEST_TIMEOUT_MS));
277         testRunner.setMaxTimeout(DEFAULT_MAX_TIMEOUT_TO_OUTPUT_MS, TimeUnit.MILLISECONDS);
278 
279         mDevice.runInstrumentationTests(testRunner, listener);
280     }
281 }
282