1 /* 2 * Copyright (C) 2017 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package android.media.cts.bitstreams; 17 18 import com.android.compatibility.common.tradefed.build.CompatibilityBuildHelper; 19 import com.android.compatibility.common.tradefed.targetprep.MediaPreparer; 20 import com.android.compatibility.common.util.MetricsReportLog; 21 import com.android.compatibility.common.util.ResultType; 22 import com.android.compatibility.common.util.ResultUnit; 23 import com.android.tradefed.build.IBuildInfo; 24 import com.android.tradefed.config.Option; 25 import com.android.tradefed.config.OptionClass; 26 import com.android.tradefed.device.DeviceNotAvailableException; 27 import com.android.tradefed.device.ITestDevice; 28 import com.android.tradefed.invoker.TestInformation; 29 import com.android.tradefed.log.LogUtil.CLog; 30 import com.android.tradefed.testtype.IAbi; 31 import com.android.tradefed.testtype.IAbiReceiver; 32 import com.android.tradefed.testtype.IBuildReceiver; 33 import com.android.tradefed.testtype.IDeviceTest; 34 import com.android.tradefed.testtype.ITestInformationReceiver; 35 import com.android.tradefed.util.FileUtil; 36 37 import org.junit.Assert; 38 import org.junit.Ignore; 39 import org.junit.Test; 40 import org.xmlpull.v1.XmlPullParser; 41 import org.xmlpull.v1.XmlPullParserException; 42 import org.xmlpull.v1.XmlPullParserFactory; 43 44 import java.io.ByteArrayOutputStream; 45 import java.io.File; 46 import java.io.FileFilter; 47 import java.io.IOException; 48 import java.io.InputStream; 49 import java.io.OutputStream; 50 import java.io.PrintStream; 51 import java.nio.file.Files; 52 import java.util.ArrayDeque; 53 import java.util.ArrayList; 54 import java.util.Collection; 55 import java.util.Collections; 56 import java.util.Deque; 57 import java.util.HashMap; 58 import java.util.Iterator; 59 import java.util.LinkedHashSet; 60 import java.util.List; 61 import java.util.Map; 62 import java.util.Map.Entry; 63 import java.util.Set; 64 import java.util.concurrent.ConcurrentHashMap; 65 import java.util.concurrent.ConcurrentMap; 66 67 /** 68 * Test that verifies video bitstreams decode pixel perfectly 69 */ 70 @OptionClass(alias="media-bitstreams-test") 71 public abstract class MediaBitstreamsTest implements IDeviceTest, IBuildReceiver, IAbiReceiver, ITestInformationReceiver { 72 73 @Option(name = MediaBitstreams.OPT_HOST_BITSTREAMS_PATH, 74 description = "Absolute path of Ittiam bitstreams (host)", 75 mandatory = true) 76 private File mHostBitstreamsPath = getDefaultBitstreamsDir(); 77 78 @Option(name = MediaBitstreams.OPT_DEVICE_BITSTREAMS_PATH, 79 description = "Absolute path of Ittiam bitstreams (device)") 80 private String mDeviceBitstreamsPath = MediaBitstreams.DEFAULT_DEVICE_BITSTEAMS_PATH; 81 82 @Option(name = MediaBitstreams.OPT_DOWNLOAD_BITSTREAMS, 83 description = "Whether to download the bitstreams files") 84 private boolean mDownloadBitstreams = false; 85 86 @Option(name = MediaBitstreams.OPT_UTILIZATION_RATE, 87 description = "Percentage of external storage space used for test") 88 private int mUtilizationRate = 80; 89 90 @Option(name = MediaBitstreams.OPT_NUM_BATCHES, 91 description = "Number of batches to test;" 92 + " each batch uses external storage up to utilization rate") 93 private int mNumBatches = Integer.MAX_VALUE; 94 95 @Option(name = MediaBitstreams.OPT_DEBUG_TARGET_DEVICE, 96 description = "Whether to debug target device under test") 97 private boolean mDebugTargetDevice = false; 98 99 @Option(name = MediaBitstreams.OPT_BITSTREAMS_PREFIX, 100 description = "Only test bitstreams in this sub-directory") 101 private String mPrefix = ""; 102 103 private String mPath = ""; 104 105 private static ConcurrentMap<String, List<ConformanceEntry>> mResults = new ConcurrentHashMap<>(); 106 107 /** 108 * Which subset of bitstreams to test 109 */ 110 enum BitstreamPackage { 111 STANDARD, 112 FULL, 113 } 114 115 private BitstreamPackage mPackage = BitstreamPackage.FULL; 116 private BitstreamPackage mPackageToRun = BitstreamPackage.STANDARD; 117 private boolean mEnforce = false; 118 119 static class ConformanceEntry { 120 final String mPath, mCodecName, mStatus; ConformanceEntry(String path, String codecName, String status)121 ConformanceEntry(String path, String codecName, String status) { 122 mPath = path; 123 mCodecName = codecName; 124 mStatus = status; 125 } 126 @Override toString()127 public String toString() { 128 return String.format("%s,%s,%s", mPath, mCodecName, mStatus); 129 } 130 } 131 132 /** 133 * A helper to access resources in the build. 134 */ 135 private CompatibilityBuildHelper mBuildHelper; 136 137 private IAbi mAbi; 138 private ITestDevice mDevice; 139 private TestInformation mTestInfo; 140 getDefaultBitstreamsDir()141 static File getDefaultBitstreamsDir() { 142 File mediaDir = MediaPreparer.getDefaultMediaDir(); 143 File[] subDirs = mediaDir.listFiles(new FileFilter() { 144 @Override 145 public boolean accept(File child) { 146 return child.isDirectory(); 147 } 148 }); 149 if (subDirs != null && subDirs.length == 1) { 150 File parent = new File(mediaDir, subDirs[0].getName()); 151 return new File(parent, MediaBitstreams.DEFAULT_HOST_BITSTREAMS_PATH); 152 } else { 153 return new File(MediaBitstreams.DEFAULT_HOST_BITSTREAMS_PATH); 154 } 155 } 156 bitstreams(String prefix, BitstreamPackage packageToRun)157 static Collection<Object[]> bitstreams(String prefix, BitstreamPackage packageToRun) { 158 final String dynConfXml = new File("/", MediaBitstreams.DYNAMIC_CONFIG_XML).toString(); 159 try (InputStream is = MediaBitstreamsTest.class.getResourceAsStream(dynConfXml)) { 160 List<Object[]> entries = new ArrayList<>(); 161 XmlPullParser parser = XmlPullParserFactory.newInstance().newPullParser(); 162 parser.setInput(is, null); 163 parser.nextTag(); 164 parser.require(XmlPullParser.START_TAG, null, MediaBitstreams.DYNAMIC_CONFIG); 165 while (parser.next() != XmlPullParser.END_DOCUMENT) { 166 if (parser.getEventType() != XmlPullParser.START_TAG 167 || !MediaBitstreams.DYNAMIC_CONFIG_ENTRY.equals(parser.getName())) { 168 continue; 169 } 170 final String key = MediaBitstreams.DYNAMIC_CONFIG_KEY; 171 String bitstream = parser.getAttributeValue(null, key); 172 if (!bitstream.startsWith(prefix)) { 173 continue; 174 } 175 while (parser.next() != XmlPullParser.END_DOCUMENT) { 176 if (parser.getEventType() != XmlPullParser.START_TAG) { 177 continue; 178 } 179 if (MediaBitstreams.DYNAMIC_CONFIG_VALUE.equals(parser.getName())) { 180 parser.next(); 181 break; 182 } 183 } 184 String format = parser.getText(); 185 String[] kvPairs = format.split(","); 186 BitstreamPackage curPackage = BitstreamPackage.FULL; 187 boolean enforce = false; 188 for (String kvPair : kvPairs) { 189 String[] kv = kvPair.split("="); 190 if (MediaBitstreams.DYNAMIC_CONFIG_PACKAGE.equals(kv[0])) { 191 String packageName = kv[1]; 192 try { 193 curPackage = BitstreamPackage.valueOf(packageName.toUpperCase()); 194 } catch (Exception e) { 195 CLog.w(e); 196 } 197 } else if (MediaBitstreams.DYNAMIC_CONFIG_ENFORCE.equals(kv[0])) { 198 enforce = "true".equals(kv[1]); 199 } 200 } 201 if (curPackage.compareTo(packageToRun) <= 0) { 202 entries.add(new Object[] {prefix, bitstream, curPackage, packageToRun, enforce}); 203 } 204 } 205 return entries; 206 } catch (XmlPullParserException | IOException e) { 207 CLog.e(e); 208 return Collections.emptyList(); 209 } 210 } 211 MediaBitstreamsTest(String prefix, String path, BitstreamPackage pkg, BitstreamPackage packageToRun )212 public MediaBitstreamsTest(String prefix, String path, BitstreamPackage pkg, BitstreamPackage packageToRun 213 ) { 214 this(prefix, path, pkg, packageToRun, false); 215 } 216 MediaBitstreamsTest(String prefix, String path, BitstreamPackage pkg, BitstreamPackage packageToRun, boolean enforce)217 public MediaBitstreamsTest(String prefix, String path, BitstreamPackage pkg, BitstreamPackage packageToRun, 218 boolean enforce) { 219 mPrefix = prefix; 220 mPath = path; 221 mPackage = pkg; 222 mPackageToRun = packageToRun; 223 mEnforce = enforce; 224 } 225 226 @Override setBuild(IBuildInfo buildInfo)227 public void setBuild(IBuildInfo buildInfo) { 228 // Get the build, this is used to access the APK. 229 mBuildHelper = new CompatibilityBuildHelper(buildInfo); 230 } 231 232 @Override setAbi(IAbi abi)233 public void setAbi(IAbi abi) { 234 mAbi = abi; 235 } 236 237 @Override setDevice(ITestDevice device)238 public void setDevice(ITestDevice device) { 239 mDevice = device; 240 } 241 242 @Override getDevice()243 public ITestDevice getDevice() { 244 return mDevice; 245 } 246 247 @Override setTestInformation(TestInformation testInformation)248 public void setTestInformation(TestInformation testInformation) { 249 mTestInfo = testInformation; 250 } 251 252 @Override getTestInformation()253 public TestInformation getTestInformation() { 254 return mTestInfo; 255 } 256 257 /* 258 * Returns true if all necessary media files exist on the device, and false otherwise. 259 * 260 * This method is exposed for unit testing. 261 */ bitstreamsExistOnDevice(ITestDevice device)262 private boolean bitstreamsExistOnDevice(ITestDevice device) 263 throws DeviceNotAvailableException { 264 return device.doesFileExist(mDeviceBitstreamsPath) 265 && device.isDirectory(mDeviceBitstreamsPath); 266 } 267 getCurrentMethod()268 private String getCurrentMethod() { 269 return Thread.currentThread().getStackTrace()[2].getMethodName(); 270 } 271 createReport(String methodName)272 private MetricsReportLog createReport(String methodName) { 273 String className = MediaBitstreamsTest.class.getCanonicalName(); 274 MetricsReportLog report = new MetricsReportLog( 275 mBuildHelper.getBuildInfo(), mAbi.getName(), 276 String.format("%s#%s", className, methodName), 277 MediaBitstreams.K_MODULE + "." + this.getClass().getSimpleName(), 278 "media_bitstreams_conformance", true); 279 return report; 280 } 281 282 /** 283 * @param method test method name in the form class#method 284 * @param p path to bitstream 285 * @param d decoder name 286 * @param s test status: unsupported, true, false, crash, or timeout. 287 */ addConformanceEntry(String method, String p, String d, String s)288 private void addConformanceEntry(String method, String p, String d, String s) { 289 MetricsReportLog report = createReport(method); 290 report.addValue(MediaBitstreams.KEY_PATH, p, ResultType.NEUTRAL, ResultUnit.NONE); 291 report.addValue(MediaBitstreams.KEY_CODEC_NAME, d, ResultType.NEUTRAL, ResultUnit.NONE); 292 report.addValue(MediaBitstreams.KEY_STATUS, s, ResultType.NEUTRAL, ResultUnit.NONE); 293 report.submit(); 294 295 ConformanceEntry ce = new ConformanceEntry(p, d, s); 296 mResults.putIfAbsent(p, new ArrayList<>()); 297 mResults.get(p).add(ce); 298 } 299 getArgs()300 Map<String, String> getArgs() { 301 Map<String, String> args = new HashMap<>(); 302 args.put(MediaBitstreams.OPT_DEBUG_TARGET_DEVICE, Boolean.toString(mDebugTargetDevice)); 303 args.put(MediaBitstreams.OPT_DEVICE_BITSTREAMS_PATH, mDeviceBitstreamsPath); 304 return args; 305 } 306 307 private class ProcessBitstreamsFormats extends ReportProcessor { 308 309 @Override setUp(ITestDevice device)310 void setUp(ITestDevice device) throws DeviceNotAvailableException { 311 if (mDownloadBitstreams || !bitstreamsExistOnDevice(device)) { 312 device.pushDir(mHostBitstreamsPath, mDeviceBitstreamsPath); 313 } 314 } 315 316 @Override getArgs()317 Map<String, String> getArgs() { 318 return MediaBitstreamsTest.this.getArgs(); 319 } 320 321 @Override process(ITestDevice device, String reportPath)322 void process(ITestDevice device, String reportPath) 323 throws DeviceNotAvailableException, IOException { 324 File dynamicConfigFile = mBuildHelper.getTestFile(MediaBitstreams.K_MODULE + ".dynamic"); 325 device.pullFile(reportPath, dynamicConfigFile); 326 CLog.i("Pulled bitstreams formats to %s", dynamicConfigFile.getPath()); 327 } 328 329 } 330 331 private class ProcessBitstreamsValidation extends ReportProcessor { 332 333 Set<String> mBitstreams; 334 Deque<String> mProcessedBitstreams = new ArrayDeque<>(); 335 private final String mMethodName; 336 private final String mBitstreamsListTxt = new File( 337 mDeviceBitstreamsPath, 338 MediaBitstreams.K_BITSTREAMS_LIST_TXT).toString(); 339 private String mLastCrash; 340 ProcessBitstreamsValidation(Set<String> bitstreams, String methodName)341 ProcessBitstreamsValidation(Set<String> bitstreams, String methodName) { 342 mBitstreams = bitstreams; 343 mMethodName = methodName; 344 } 345 getBitstreamsListString()346 private String getBitstreamsListString() { 347 OutputStream baos = new ByteArrayOutputStream(); 348 PrintStream ps = new PrintStream(baos, true); 349 try { 350 for (String b : mBitstreams) { 351 ps.println(b); 352 } 353 return baos.toString(); 354 } finally { 355 ps.close(); 356 } 357 } 358 pushBitstreams(ITestDevice device)359 private void pushBitstreams(ITestDevice device) 360 throws IOException, DeviceNotAvailableException { 361 File tmp = null; 362 try { 363 CLog.i("Pushing %d bitstream(s) from %s to %s", 364 mBitstreams.size(), 365 mHostBitstreamsPath, 366 mDeviceBitstreamsPath); 367 tmp = Files.createTempDirectory(null).toFile(); 368 for (String b : mBitstreams) { 369 String m = MediaBitstreams.getMd5Path(b); 370 for (String f : new String[] {m, b}) { 371 File tmpf = new File(tmp, f); 372 new File(tmpf.getParent()).mkdirs(); 373 FileUtil.copyFile(new File(mHostBitstreamsPath, f), tmpf); 374 } 375 } 376 device.executeShellCommand(String.format("rm -rf %s", mDeviceBitstreamsPath)); 377 device.pushDir(tmp, mDeviceBitstreamsPath); 378 device.pushString(getBitstreamsListString(), mBitstreamsListTxt); 379 } finally { 380 FileUtil.recursiveDelete(tmp); 381 } 382 } 383 384 @Override setUp(ITestDevice device)385 void setUp(ITestDevice device) throws DeviceNotAvailableException, IOException { 386 pushBitstreams(device); 387 } 388 389 @Override getArgs()390 Map<String, String> getArgs() { 391 Map<String, String> args = MediaBitstreamsTest.this.getArgs(); 392 if (mLastCrash != null) { 393 args.put(MediaBitstreams.OPT_LAST_CRASH, mLastCrash); 394 } 395 return args; 396 } 397 parse(ITestDevice device, String reportPath)398 private void parse(ITestDevice device, String reportPath) 399 throws DeviceNotAvailableException { 400 String[] lines = getReportLines(device, reportPath); 401 mProcessedBitstreams.clear(); 402 for (int i = 0; i < lines.length;) { 403 404 String path = lines[i++]; 405 mProcessedBitstreams.add(path); 406 String errMsg; 407 408 boolean failedEarly; 409 if (i < lines.length) { 410 failedEarly = Boolean.parseBoolean(lines[i++]); 411 errMsg = failedEarly ? lines[i++] : ""; 412 } else { 413 failedEarly = true; 414 errMsg = MediaBitstreams.K_NATIVE_CRASH; 415 mLastCrash = MediaBitstreams.generateCrashSignature(path, ""); 416 mProcessedBitstreams.removeLast(); 417 } 418 419 if (failedEarly) { 420 addConformanceEntry(mMethodName, path, null, errMsg); 421 continue; 422 } 423 424 int n = Integer.parseInt(lines[i++]); 425 for (int j = 0; j < n && i < lines.length; j++) { 426 String decoderName = lines[i++]; 427 String result; 428 if (i < lines.length) { 429 result = lines[i++]; 430 } else { 431 result = MediaBitstreams.K_NATIVE_CRASH; 432 mLastCrash = MediaBitstreams.generateCrashSignature(path, decoderName); 433 mProcessedBitstreams.removeLast(); 434 } 435 addConformanceEntry(mMethodName, path, decoderName, result); 436 } 437 438 439 } 440 } 441 442 @Override process(ITestDevice device, String reportPath)443 void process(ITestDevice device, String reportPath) 444 throws DeviceNotAvailableException, IOException { 445 parse(device, reportPath); 446 } 447 448 @Override recover(ITestDevice device, String reportPath)449 boolean recover(ITestDevice device, String reportPath) 450 throws DeviceNotAvailableException, IOException { 451 try { 452 parse(device, reportPath); 453 mBitstreams.removeAll(mProcessedBitstreams); 454 device.pushString(getBitstreamsListString(), mBitstreamsListTxt); 455 return true; 456 } catch (RuntimeException e) { 457 File hostFile = reportPath == null ? null : device.pullFile(reportPath); 458 CLog.e("Error parsing report; saving report to %s", hostFile); 459 CLog.e(e); 460 return false; 461 } 462 } 463 464 } 465 466 @Ignore 467 @Test testGetBitstreamsFormats()468 public void testGetBitstreamsFormats() throws DeviceNotAvailableException, IOException { 469 ReportProcessor processor = new ProcessBitstreamsFormats(); 470 processor.processDeviceReport( 471 getTestInformation(), 472 getDevice(), 473 getCurrentMethod(), 474 MediaBitstreams.KEY_BITSTREAMS_FORMATS_XML); 475 } 476 477 @Test testBitstreamsConformance()478 public void testBitstreamsConformance() { 479 File bitstreamFile = new File(mHostBitstreamsPath, mPath); 480 if (!bitstreamFile.exists()) { 481 // todo(b/65165250): throw Exception once MediaPreparer can auto-download 482 CLog.w(bitstreamFile + " not found; skipping"); 483 return; 484 } 485 486 if (!mResults.containsKey(mPath)) { 487 try { 488 testBitstreamsConformance(mPrefix); 489 } catch (DeviceNotAvailableException | IOException e) { 490 String curMethod = getCurrentMethod(); 491 addConformanceEntry(curMethod, mPath, MediaBitstreams.K_UNAVAILABLE, e.toString()); 492 } 493 } 494 495 if (mEnforce) { 496 if (!mResults.containsKey(mPath)) { 497 Assert.fail("no results captured for " + mPath); 498 } 499 List<ConformanceEntry> entries = mResults.get(mPath); 500 for (ConformanceEntry ce : entries) { 501 if (!"true".equals(ce.mStatus) && !"unsupported".equals(ce.mStatus)) { 502 Assert.fail(ce.toString()); 503 } 504 } 505 } 506 507 } 508 testBitstreamsConformance(String prefix)509 private void testBitstreamsConformance(String prefix) 510 throws DeviceNotAvailableException, IOException { 511 512 ITestDevice device = getDevice(); 513 SupportedBitstreamsProcessor preparer; 514 preparer = new SupportedBitstreamsProcessor(prefix, mDebugTargetDevice); 515 preparer.processDeviceReport( 516 getTestInformation(), 517 device, 518 MediaBitstreams.K_TEST_GET_SUPPORTED_BITSTREAMS, 519 MediaBitstreams.KEY_SUPPORTED_BITSTREAMS_TXT); 520 Collection<Object[]> bitstreams = bitstreams(mPrefix, mPackageToRun); 521 Set<String> supportedBitstreams = preparer.getSupportedBitstreams(); 522 CLog.i("%d supported bitstreams under %s", supportedBitstreams.size(), prefix); 523 524 int n = 0; 525 long size = 0; 526 long limit = device.getExternalStoreFreeSpace() * mUtilizationRate * 1024 / 100; 527 528 String curMethod = getCurrentMethod(); 529 Set<String> toPush = new LinkedHashSet<>(); 530 Iterator<Object[]> iter = bitstreams.iterator(); 531 532 for (int i = 0; i < bitstreams.size(); i++) { 533 534 if (n >= mNumBatches) { 535 break; 536 } 537 538 String p = (String) iter.next()[1]; 539 Map<String, Boolean> decoderCapabilities; 540 decoderCapabilities = preparer.getDecoderCapabilitiesForPath(p); 541 if (decoderCapabilities.isEmpty()) { 542 addConformanceEntry( 543 curMethod, p, 544 MediaBitstreams.K_UNAVAILABLE, 545 MediaBitstreams.K_UNSUPPORTED); 546 } 547 for (Entry<String, Boolean> entry : decoderCapabilities.entrySet()) { 548 Boolean supported = entry.getValue(); 549 if (supported) { 550 File bitstreamFile = new File(mHostBitstreamsPath, p); 551 String md5Path = MediaBitstreams.getMd5Path(p); 552 File md5File = new File(mHostBitstreamsPath, md5Path); 553 if (md5File.exists() && bitstreamFile.exists() && toPush.add(p)) { 554 size += md5File.length(); 555 size += bitstreamFile.length(); 556 } 557 } else { 558 String d = entry.getKey(); 559 addConformanceEntry(curMethod, p, d, MediaBitstreams.K_UNSUPPORTED); 560 } 561 } 562 563 if (size > limit || i + 1 == bitstreams.size()) { 564 ReportProcessor processor; 565 processor = new ProcessBitstreamsValidation(toPush, curMethod); 566 processor.processDeviceReport( 567 getTestInformation(), 568 device, 569 curMethod, 570 MediaBitstreams.KEY_BITSTREAMS_VALIDATION_TXT); 571 toPush.clear(); 572 size = 0; 573 n++; 574 } 575 576 } 577 578 } 579 580 581 } 582