1 /* 2 * Copyright (C) 2013 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package android.media.cts; 18 19 import android.annotation.TargetApi; 20 import android.content.res.AssetFileDescriptor; 21 import android.content.Context; 22 import android.media.MediaCodec; 23 import android.media.MediaCodecInfo; 24 import android.media.MediaCodecInfo.CodecCapabilities; 25 import android.media.MediaCodecInfo.CodecProfileLevel; 26 import android.media.MediaCodecList; 27 import android.media.MediaExtractor; 28 import android.media.MediaFormat; 29 import android.media.MediaMuxer; 30 import android.media.MediaPlayer; 31 import android.os.Environment; 32 import android.platform.test.annotations.AppModeFull; 33 import android.test.ActivityInstrumentationTestCase2; 34 import android.util.Log; 35 import android.view.Surface; 36 37 import android.media.MediaCodecInfo; 38 import android.media.MediaCodecInfo.CodecCapabilities; 39 import android.media.MediaCodecInfo.CodecProfileLevel; 40 41 import android.media.cts.R; 42 43 import java.io.File; 44 import java.io.IOException; 45 import java.nio.ByteBuffer; 46 import java.util.concurrent.atomic.AtomicReference; 47 import java.util.concurrent.CountDownLatch; 48 49 /** 50 * Test for the integration of MediaMuxer and MediaCodec's encoder. 51 * 52 * <p>It uses MediaExtractor to get frames from a test stream, decodes them to a surface, uses a 53 * shader to edit them, encodes them from the resulting surface, and then uses MediaMuxer to write 54 * them into a file. 55 * 56 * <p>It does not currently check whether the result file is correct, but makes sure that nothing 57 * fails along the way. 58 * 59 * <p>It also tests the way the codec config buffers need to be passed from the MediaCodec to the 60 * MediaMuxer. 61 */ 62 @TargetApi(18) 63 @AppModeFull(reason = "Instant apps cannot access the SD card") 64 public class ExtractDecodeEditEncodeMuxTest 65 extends ActivityInstrumentationTestCase2<MediaStubActivity> { 66 67 private static final String TAG = ExtractDecodeEditEncodeMuxTest.class.getSimpleName(); 68 private static final boolean VERBOSE = false; // lots of logging 69 70 /** How long to wait for the next buffer to become available. */ 71 private static final int TIMEOUT_USEC = 10000; 72 73 /** Where to output the test files. */ 74 private static final File OUTPUT_FILENAME_DIR = Environment.getExternalStorageDirectory(); 75 76 // parameters for the video encoder 77 private static final int OUTPUT_VIDEO_BIT_RATE = 2000000; // 2Mbps 78 private static final int OUTPUT_VIDEO_FRAME_RATE = 15; // 15fps 79 private static final int OUTPUT_VIDEO_IFRAME_INTERVAL = 10; // 10 seconds between I-frames 80 private static final int OUTPUT_VIDEO_COLOR_FORMAT = 81 MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface; 82 83 // parameters for the audio encoder 84 // Advanced Audio Coding 85 private static final String OUTPUT_AUDIO_MIME_TYPE = MediaFormat.MIMETYPE_AUDIO_AAC; 86 private static final int OUTPUT_AUDIO_CHANNEL_COUNT = 2; // Must match the input stream. 87 private static final int OUTPUT_AUDIO_BIT_RATE = 128 * 1024; 88 private static final int OUTPUT_AUDIO_AAC_PROFILE = 89 MediaCodecInfo.CodecProfileLevel.AACObjectHE; 90 private static final int OUTPUT_AUDIO_SAMPLE_RATE_HZ = 44100; // Must match the input stream. 91 92 /** 93 * Used for editing the frames. 94 * 95 * <p>Swaps green and blue channels by storing an RBGA color in an RGBA buffer. 96 */ 97 private static final String FRAGMENT_SHADER = 98 "#extension GL_OES_EGL_image_external : require\n" + 99 "precision mediump float;\n" + 100 "varying vec2 vTextureCoord;\n" + 101 "uniform samplerExternalOES sTexture;\n" + 102 "void main() {\n" + 103 " gl_FragColor = texture2D(sTexture, vTextureCoord).rbga;\n" + 104 "}\n"; 105 106 /** Whether to copy the video from the test video. */ 107 private boolean mCopyVideo; 108 /** Whether to copy the audio from the test video. */ 109 private boolean mCopyAudio; 110 /** Whether to verify the audio format. */ 111 private boolean mVerifyAudioFormat; 112 /** Width of the output frames. */ 113 private int mWidth = -1; 114 /** Height of the output frames. */ 115 private int mHeight = -1; 116 117 /** The raw resource used as the input file. */ 118 private int mSourceResId; 119 120 /** The destination file for the encoded output. */ 121 private String mOutputFile; 122 123 private String mOutputVideoMimeType; 124 ExtractDecodeEditEncodeMuxTest()125 public ExtractDecodeEditEncodeMuxTest() { 126 super(MediaStubActivity.class); 127 } 128 testExtractDecodeEditEncodeMuxQCIF()129 public void testExtractDecodeEditEncodeMuxQCIF() throws Throwable { 130 if(!setSize(176, 144)) return; 131 setSource(R.raw.video_480x360_mp4_h264_500kbps_30fps_aac_stereo_128kbps_44100hz); 132 setCopyVideo(); 133 setVideoMimeType(MediaFormat.MIMETYPE_VIDEO_AVC); 134 TestWrapper.runTest(this); 135 } 136 testExtractDecodeEditEncodeMuxQVGA()137 public void testExtractDecodeEditEncodeMuxQVGA() throws Throwable { 138 if(!setSize(320, 240)) return; 139 setSource(R.raw.video_480x360_mp4_h264_500kbps_30fps_aac_stereo_128kbps_44100hz); 140 setCopyVideo(); 141 setVideoMimeType(MediaFormat.MIMETYPE_VIDEO_AVC); 142 TestWrapper.runTest(this); 143 } 144 testExtractDecodeEditEncodeMux720p()145 public void testExtractDecodeEditEncodeMux720p() throws Throwable { 146 if(!setSize(1280, 720)) return; 147 setSource(R.raw.video_480x360_mp4_h264_500kbps_30fps_aac_stereo_128kbps_44100hz); 148 setCopyVideo(); 149 setVideoMimeType(MediaFormat.MIMETYPE_VIDEO_AVC); 150 TestWrapper.runTest(this); 151 } 152 testExtractDecodeEditEncodeMux2160pHevc()153 public void testExtractDecodeEditEncodeMux2160pHevc() throws Throwable { 154 if(!setSize(3840, 2160)) return; 155 setSource(R.raw.video_480x360_mp4_h264_500kbps_30fps_aac_stereo_128kbps_44100hz); 156 setCopyVideo(); 157 setVideoMimeType(MediaFormat.MIMETYPE_VIDEO_HEVC); 158 TestWrapper.runTest(this); 159 } 160 testExtractDecodeEditEncodeMuxAudio()161 public void testExtractDecodeEditEncodeMuxAudio() throws Throwable { 162 if(!setSize(1280, 720)) return; 163 setSource(R.raw.video_480x360_mp4_h264_500kbps_30fps_aac_stereo_128kbps_44100hz); 164 setCopyAudio(); 165 setVerifyAudioFormat(); 166 TestWrapper.runTest(this); 167 } 168 testExtractDecodeEditEncodeMuxAudioVideo()169 public void testExtractDecodeEditEncodeMuxAudioVideo() throws Throwable { 170 if(!setSize(1280, 720)) return; 171 setSource(R.raw.video_480x360_mp4_h264_500kbps_30fps_aac_stereo_128kbps_44100hz); 172 setCopyAudio(); 173 setCopyVideo(); 174 setVerifyAudioFormat(); 175 TestWrapper.runTest(this); 176 } 177 178 /** Wraps testExtractDecodeEditEncodeMux() */ 179 private static class TestWrapper implements Runnable { 180 private Throwable mThrowable; 181 private ExtractDecodeEditEncodeMuxTest mTest; 182 TestWrapper(ExtractDecodeEditEncodeMuxTest test)183 private TestWrapper(ExtractDecodeEditEncodeMuxTest test) { 184 mTest = test; 185 } 186 187 @Override run()188 public void run() { 189 try { 190 mTest.extractDecodeEditEncodeMux(); 191 } catch (Throwable th) { 192 mThrowable = th; 193 } 194 } 195 196 /** 197 * Entry point. 198 */ runTest(ExtractDecodeEditEncodeMuxTest test)199 public static void runTest(ExtractDecodeEditEncodeMuxTest test) throws Throwable { 200 test.setOutputFile(); 201 TestWrapper wrapper = new TestWrapper(test); 202 Thread th = new Thread(wrapper, "codec test"); 203 th.start(); 204 th.join(); 205 if (wrapper.mThrowable != null) { 206 throw wrapper.mThrowable; 207 } 208 } 209 } 210 211 /** 212 * Sets the test to copy the video stream. 213 */ setCopyVideo()214 private void setCopyVideo() { 215 mCopyVideo = true; 216 } 217 218 /** 219 * Sets the test to copy the video stream. 220 */ setCopyAudio()221 private void setCopyAudio() { 222 mCopyAudio = true; 223 } 224 225 /** 226 * Sets the test to verify the output audio format. 227 */ setVerifyAudioFormat()228 private void setVerifyAudioFormat() { 229 mVerifyAudioFormat = true; 230 } 231 232 /** 233 * Sets the desired frame size and returns whether the given resolution is 234 * supported. 235 * 236 * <p>If decoding/encoding using AVC as the codec, checks that the resolution 237 * is supported. For other codecs, always return {@code true}. 238 */ setSize(int width, int height)239 private boolean setSize(int width, int height) { 240 if ((width % 16) != 0 || (height % 16) != 0) { 241 Log.w(TAG, "WARNING: width or height not multiple of 16"); 242 } 243 mWidth = width; 244 mHeight = height; 245 246 // TODO: remove this logic in setSize as it is now handled when configuring codecs 247 return true; 248 } 249 250 /** 251 * Sets the raw resource used as the source video. 252 */ setSource(int resId)253 private void setSource(int resId) { 254 mSourceResId = resId; 255 } 256 257 /** 258 * Sets the name of the output file based on the other parameters. 259 * 260 * <p>Must be called after {@link #setSize(int, int)} and {@link #setSource(int)}. 261 */ setOutputFile()262 private void setOutputFile() { 263 StringBuilder sb = new StringBuilder(); 264 sb.append(OUTPUT_FILENAME_DIR.getAbsolutePath()); 265 sb.append("/cts-media-"); 266 sb.append(getClass().getSimpleName()); 267 assertTrue("should have called setSource() first", mSourceResId != -1); 268 sb.append('-'); 269 sb.append(mSourceResId); 270 if (mCopyVideo) { 271 assertTrue("should have called setSize() first", mWidth != -1); 272 assertTrue("should have called setSize() first", mHeight != -1); 273 sb.append('-'); 274 sb.append("video"); 275 sb.append('-'); 276 sb.append(mWidth); 277 sb.append('x'); 278 sb.append(mHeight); 279 } 280 if (mCopyAudio) { 281 sb.append('-'); 282 sb.append("audio"); 283 } 284 sb.append(".mp4"); 285 mOutputFile = sb.toString(); 286 } 287 setVideoMimeType(String mimeType)288 private void setVideoMimeType(String mimeType) { 289 mOutputVideoMimeType = mimeType; 290 } 291 292 /** 293 * Tests encoding and subsequently decoding video from frames generated into a buffer. 294 * <p> 295 * We encode several frames of a video test pattern using MediaCodec, then decode the output 296 * with MediaCodec and do some simple checks. 297 */ extractDecodeEditEncodeMux()298 private void extractDecodeEditEncodeMux() throws Exception { 299 // Exception that may be thrown during release. 300 Exception exception = null; 301 302 MediaCodecList mcl = new MediaCodecList(MediaCodecList.REGULAR_CODECS); 303 304 // We avoid the device-specific limitations on width and height by using values 305 // that are multiples of 16, which all tested devices seem to be able to handle. 306 MediaFormat outputVideoFormat = 307 MediaFormat.createVideoFormat(mOutputVideoMimeType, mWidth, mHeight); 308 309 // Set some properties. Failing to specify some of these can cause the MediaCodec 310 // configure() call to throw an unhelpful exception. 311 outputVideoFormat.setInteger( 312 MediaFormat.KEY_COLOR_FORMAT, OUTPUT_VIDEO_COLOR_FORMAT); 313 outputVideoFormat.setInteger(MediaFormat.KEY_BIT_RATE, OUTPUT_VIDEO_BIT_RATE); 314 outputVideoFormat.setInteger(MediaFormat.KEY_FRAME_RATE, OUTPUT_VIDEO_FRAME_RATE); 315 outputVideoFormat.setInteger( 316 MediaFormat.KEY_I_FRAME_INTERVAL, OUTPUT_VIDEO_IFRAME_INTERVAL); 317 if (VERBOSE) Log.d(TAG, "video format: " + outputVideoFormat); 318 319 String videoEncoderName = mcl.findEncoderForFormat(outputVideoFormat); 320 if (videoEncoderName == null) { 321 // Don't fail CTS if they don't have an AVC codec (not here, anyway). 322 Log.e(TAG, "Unable to find an appropriate codec for " + outputVideoFormat); 323 return; 324 } 325 if (VERBOSE) Log.d(TAG, "video found codec: " + videoEncoderName); 326 327 MediaFormat outputAudioFormat = 328 MediaFormat.createAudioFormat( 329 OUTPUT_AUDIO_MIME_TYPE, OUTPUT_AUDIO_SAMPLE_RATE_HZ, 330 OUTPUT_AUDIO_CHANNEL_COUNT); 331 outputAudioFormat.setInteger(MediaFormat.KEY_BIT_RATE, OUTPUT_AUDIO_BIT_RATE); 332 // TODO: Bug workaround --- uncomment once fixed. 333 // outputAudioFormat.setInteger(MediaFormat.KEY_AAC_PROFILE, OUTPUT_AUDIO_AAC_PROFILE); 334 335 String audioEncoderName = mcl.findEncoderForFormat(outputAudioFormat); 336 if (audioEncoderName == null) { 337 // Don't fail CTS if they don't have an AAC codec (not here, anyway). 338 Log.e(TAG, "Unable to find an appropriate codec for " + outputAudioFormat); 339 return; 340 } 341 if (VERBOSE) Log.d(TAG, "audio found codec: " + audioEncoderName); 342 343 MediaExtractor videoExtractor = null; 344 MediaExtractor audioExtractor = null; 345 OutputSurface outputSurface = null; 346 MediaCodec videoDecoder = null; 347 MediaCodec audioDecoder = null; 348 MediaCodec videoEncoder = null; 349 MediaCodec audioEncoder = null; 350 MediaMuxer muxer = null; 351 352 InputSurface inputSurface = null; 353 354 try { 355 if (mCopyVideo) { 356 videoExtractor = createExtractor(); 357 int videoInputTrack = getAndSelectVideoTrackIndex(videoExtractor); 358 assertTrue("missing video track in test video", videoInputTrack != -1); 359 MediaFormat inputFormat = videoExtractor.getTrackFormat(videoInputTrack); 360 361 // Create a MediaCodec for the desired codec, then configure it as an encoder with 362 // our desired properties. Request a Surface to use for input. 363 AtomicReference<Surface> inputSurfaceReference = new AtomicReference<Surface>(); 364 videoEncoder = createVideoEncoder( 365 videoEncoderName, outputVideoFormat, inputSurfaceReference); 366 inputSurface = new InputSurface(inputSurfaceReference.get()); 367 inputSurface.makeCurrent(); 368 // Create a MediaCodec for the decoder, based on the extractor's format. 369 outputSurface = new OutputSurface(); 370 outputSurface.changeFragmentShader(FRAGMENT_SHADER); 371 videoDecoder = createVideoDecoder(mcl, inputFormat, outputSurface.getSurface()); 372 } 373 374 if (mCopyAudio) { 375 audioExtractor = createExtractor(); 376 int audioInputTrack = getAndSelectAudioTrackIndex(audioExtractor); 377 assertTrue("missing audio track in test video", audioInputTrack != -1); 378 MediaFormat inputFormat = audioExtractor.getTrackFormat(audioInputTrack); 379 380 // Create a MediaCodec for the desired codec, then configure it as an encoder with 381 // our desired properties. Request a Surface to use for input. 382 audioEncoder = createAudioEncoder(audioEncoderName, outputAudioFormat); 383 // Create a MediaCodec for the decoder, based on the extractor's format. 384 audioDecoder = createAudioDecoder(mcl, inputFormat); 385 } 386 387 // Creates a muxer but do not start or add tracks just yet. 388 muxer = createMuxer(); 389 390 doExtractDecodeEditEncodeMux( 391 videoExtractor, 392 audioExtractor, 393 videoDecoder, 394 videoEncoder, 395 audioDecoder, 396 audioEncoder, 397 muxer, 398 inputSurface, 399 outputSurface); 400 } finally { 401 if (VERBOSE) Log.d(TAG, "releasing extractor, decoder, encoder, and muxer"); 402 // Try to release everything we acquired, even if one of the releases fails, in which 403 // case we save the first exception we got and re-throw at the end (unless something 404 // other exception has already been thrown). This guarantees the first exception thrown 405 // is reported as the cause of the error, everything is (attempted) to be released, and 406 // all other exceptions appear in the logs. 407 try { 408 if (videoExtractor != null) { 409 videoExtractor.release(); 410 } 411 } catch(Exception e) { 412 Log.e(TAG, "error while releasing videoExtractor", e); 413 if (exception == null) { 414 exception = e; 415 } 416 } 417 try { 418 if (audioExtractor != null) { 419 audioExtractor.release(); 420 } 421 } catch(Exception e) { 422 Log.e(TAG, "error while releasing audioExtractor", e); 423 if (exception == null) { 424 exception = e; 425 } 426 } 427 try { 428 if (videoDecoder != null) { 429 videoDecoder.stop(); 430 videoDecoder.release(); 431 } 432 } catch(Exception e) { 433 Log.e(TAG, "error while releasing videoDecoder", e); 434 if (exception == null) { 435 exception = e; 436 } 437 } 438 try { 439 if (outputSurface != null) { 440 outputSurface.release(); 441 } 442 } catch(Exception e) { 443 Log.e(TAG, "error while releasing outputSurface", e); 444 if (exception == null) { 445 exception = e; 446 } 447 } 448 try { 449 if (videoEncoder != null) { 450 videoEncoder.stop(); 451 videoEncoder.release(); 452 } 453 } catch(Exception e) { 454 Log.e(TAG, "error while releasing videoEncoder", e); 455 if (exception == null) { 456 exception = e; 457 } 458 } 459 try { 460 if (audioDecoder != null) { 461 audioDecoder.stop(); 462 audioDecoder.release(); 463 } 464 } catch(Exception e) { 465 Log.e(TAG, "error while releasing audioDecoder", e); 466 if (exception == null) { 467 exception = e; 468 } 469 } 470 try { 471 if (audioEncoder != null) { 472 audioEncoder.stop(); 473 audioEncoder.release(); 474 } 475 } catch(Exception e) { 476 Log.e(TAG, "error while releasing audioEncoder", e); 477 if (exception == null) { 478 exception = e; 479 } 480 } 481 try { 482 if (muxer != null) { 483 muxer.stop(); 484 muxer.release(); 485 } 486 } catch(Exception e) { 487 Log.e(TAG, "error while releasing muxer", e); 488 if (exception == null) { 489 exception = e; 490 } 491 } 492 try { 493 if (inputSurface != null) { 494 inputSurface.release(); 495 } 496 } catch(Exception e) { 497 Log.e(TAG, "error while releasing inputSurface", e); 498 if (exception == null) { 499 exception = e; 500 } 501 } 502 } 503 if (exception != null) { 504 throw exception; 505 } 506 507 MediaExtractor mediaExtractor = null; 508 try { 509 mediaExtractor = new MediaExtractor(); 510 mediaExtractor.setDataSource(mOutputFile); 511 512 assertEquals("incorrect number of tracks", (mCopyAudio ? 1 : 0) + (mCopyVideo ? 1 : 0), 513 mediaExtractor.getTrackCount()); 514 if (mVerifyAudioFormat) { 515 boolean foundAudio = false; 516 for (int i = 0; i < mediaExtractor.getTrackCount(); i++) { 517 MediaFormat trackFormat = mediaExtractor.getTrackFormat(i); 518 if (isAudioFormat(trackFormat)) { 519 foundAudio = true; 520 int expectedSampleRate = OUTPUT_AUDIO_SAMPLE_RATE_HZ; 521 522 // SBR mode halves the sample rate in the format. 523 if (OUTPUT_AUDIO_AAC_PROFILE == 524 MediaCodecInfo.CodecProfileLevel.AACObjectHE) { 525 expectedSampleRate /= 2; 526 } 527 assertEquals("sample rates should match", expectedSampleRate, 528 trackFormat.getInteger(MediaFormat.KEY_SAMPLE_RATE)); 529 } 530 } 531 532 assertTrue("output should have an audio track", foundAudio || !mCopyAudio); 533 } 534 } catch (IOException e) { 535 throw new IllegalStateException("exception verifying output file", e); 536 } finally { 537 if (mediaExtractor != null) { 538 mediaExtractor.release(); 539 } 540 } 541 542 // TODO: Check the generated output file's video format and sample data. 543 544 MediaStubActivity activity = getActivity(); 545 final MediaPlayer mp = new MediaPlayer(); 546 final Exception[] exceptionHolder = { null }; 547 final CountDownLatch playbackEndSignal = new CountDownLatch(1); 548 mp.setOnErrorListener(new MediaPlayer.OnErrorListener() { 549 @Override 550 public boolean onError(MediaPlayer origin, int what, int extra) { 551 exceptionHolder[0] = new RuntimeException("error playing output file: what=" + what 552 + " extra=" + extra); 553 // Returning false would trigger onCompletion() so that 554 // playbackEndSignal.await() can stop waiting. 555 return false; 556 } 557 }); 558 mp.setOnCompletionListener(new MediaPlayer.OnCompletionListener() { 559 @Override 560 public void onCompletion(MediaPlayer origin) { 561 playbackEndSignal.countDown(); 562 } 563 }); 564 try { 565 mp.setDataSource(mOutputFile); 566 mp.setDisplay(activity.getSurfaceHolder()); 567 mp.prepare(); 568 mp.start(); 569 playbackEndSignal.await(); 570 } catch (Exception e) { 571 exceptionHolder[0] = e; 572 } finally { 573 mp.release(); 574 } 575 576 if (exceptionHolder[0] != null) { 577 throw exceptionHolder[0]; 578 } 579 } 580 581 /** 582 * Creates an extractor that reads its frames from {@link #mSourceResId}. 583 */ createExtractor()584 private MediaExtractor createExtractor() throws IOException { 585 MediaExtractor extractor; 586 Context context = getInstrumentation().getTargetContext(); 587 AssetFileDescriptor srcFd = context.getResources().openRawResourceFd(mSourceResId); 588 extractor = new MediaExtractor(); 589 extractor.setDataSource(srcFd.getFileDescriptor(), srcFd.getStartOffset(), 590 srcFd.getLength()); 591 return extractor; 592 } 593 594 /** 595 * Creates a decoder for the given format, which outputs to the given surface. 596 * 597 * @param inputFormat the format of the stream to decode 598 * @param surface into which to decode the frames 599 */ createVideoDecoder( MediaCodecList mcl, MediaFormat inputFormat, Surface surface)600 private MediaCodec createVideoDecoder( 601 MediaCodecList mcl, MediaFormat inputFormat, Surface surface) throws IOException { 602 MediaCodec decoder = MediaCodec.createByCodecName(mcl.findDecoderForFormat(inputFormat)); 603 decoder.configure(inputFormat, surface, null, 0); 604 decoder.start(); 605 return decoder; 606 } 607 608 /** 609 * Creates an encoder for the given format using the specified codec, taking input from a 610 * surface. 611 * 612 * <p>The surface to use as input is stored in the given reference. 613 * 614 * @param codecInfo of the codec to use 615 * @param format of the stream to be produced 616 * @param surfaceReference to store the surface to use as input 617 */ createVideoEncoder( String codecName, MediaFormat format, AtomicReference<Surface> surfaceReference)618 private MediaCodec createVideoEncoder( 619 String codecName, 620 MediaFormat format, 621 AtomicReference<Surface> surfaceReference) 622 throws IOException { 623 MediaCodec encoder = MediaCodec.createByCodecName(codecName); 624 encoder.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE); 625 // Must be called before start() is. 626 surfaceReference.set(encoder.createInputSurface()); 627 encoder.start(); 628 return encoder; 629 } 630 631 /** 632 * Creates a decoder for the given format. 633 * 634 * @param inputFormat the format of the stream to decode 635 */ createAudioDecoder( MediaCodecList mcl, MediaFormat inputFormat)636 private MediaCodec createAudioDecoder( 637 MediaCodecList mcl, MediaFormat inputFormat) throws IOException { 638 MediaCodec decoder = MediaCodec.createByCodecName(mcl.findDecoderForFormat(inputFormat)); 639 decoder.configure(inputFormat, null, null, 0); 640 decoder.start(); 641 return decoder; 642 } 643 644 /** 645 * Creates an encoder for the given format using the specified codec. 646 * 647 * @param codecInfo of the codec to use 648 * @param format of the stream to be produced 649 */ createAudioEncoder(String codecName, MediaFormat format)650 private MediaCodec createAudioEncoder(String codecName, MediaFormat format) 651 throws IOException { 652 MediaCodec encoder = MediaCodec.createByCodecName(codecName); 653 encoder.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE); 654 encoder.start(); 655 return encoder; 656 } 657 658 /** 659 * Creates a muxer to write the encoded frames. 660 * 661 * <p>The muxer is not started as it needs to be started only after all streams have been added. 662 */ createMuxer()663 private MediaMuxer createMuxer() throws IOException { 664 return new MediaMuxer(mOutputFile, MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4); 665 } 666 getAndSelectVideoTrackIndex(MediaExtractor extractor)667 private int getAndSelectVideoTrackIndex(MediaExtractor extractor) { 668 for (int index = 0; index < extractor.getTrackCount(); ++index) { 669 if (VERBOSE) { 670 Log.d(TAG, "format for track " + index + " is " 671 + getMimeTypeFor(extractor.getTrackFormat(index))); 672 } 673 if (isVideoFormat(extractor.getTrackFormat(index))) { 674 extractor.selectTrack(index); 675 return index; 676 } 677 } 678 return -1; 679 } 680 getAndSelectAudioTrackIndex(MediaExtractor extractor)681 private int getAndSelectAudioTrackIndex(MediaExtractor extractor) { 682 for (int index = 0; index < extractor.getTrackCount(); ++index) { 683 if (VERBOSE) { 684 Log.d(TAG, "format for track " + index + " is " 685 + getMimeTypeFor(extractor.getTrackFormat(index))); 686 } 687 if (isAudioFormat(extractor.getTrackFormat(index))) { 688 extractor.selectTrack(index); 689 return index; 690 } 691 } 692 return -1; 693 } 694 695 /** 696 * Does the actual work for extracting, decoding, encoding and muxing. 697 */ doExtractDecodeEditEncodeMux( MediaExtractor videoExtractor, MediaExtractor audioExtractor, MediaCodec videoDecoder, MediaCodec videoEncoder, MediaCodec audioDecoder, MediaCodec audioEncoder, MediaMuxer muxer, InputSurface inputSurface, OutputSurface outputSurface)698 private void doExtractDecodeEditEncodeMux( 699 MediaExtractor videoExtractor, 700 MediaExtractor audioExtractor, 701 MediaCodec videoDecoder, 702 MediaCodec videoEncoder, 703 MediaCodec audioDecoder, 704 MediaCodec audioEncoder, 705 MediaMuxer muxer, 706 InputSurface inputSurface, 707 OutputSurface outputSurface) { 708 ByteBuffer[] videoDecoderInputBuffers = null; 709 ByteBuffer[] videoDecoderOutputBuffers = null; 710 ByteBuffer[] videoEncoderOutputBuffers = null; 711 MediaCodec.BufferInfo videoDecoderOutputBufferInfo = null; 712 MediaCodec.BufferInfo videoEncoderOutputBufferInfo = null; 713 if (mCopyVideo) { 714 videoDecoderInputBuffers = videoDecoder.getInputBuffers(); 715 videoDecoderOutputBuffers = videoDecoder.getOutputBuffers(); 716 videoEncoderOutputBuffers = videoEncoder.getOutputBuffers(); 717 videoDecoderOutputBufferInfo = new MediaCodec.BufferInfo(); 718 videoEncoderOutputBufferInfo = new MediaCodec.BufferInfo(); 719 } 720 ByteBuffer[] audioDecoderInputBuffers = null; 721 ByteBuffer[] audioDecoderOutputBuffers = null; 722 ByteBuffer[] audioEncoderInputBuffers = null; 723 ByteBuffer[] audioEncoderOutputBuffers = null; 724 MediaCodec.BufferInfo audioDecoderOutputBufferInfo = null; 725 MediaCodec.BufferInfo audioEncoderOutputBufferInfo = null; 726 if (mCopyAudio) { 727 audioDecoderInputBuffers = audioDecoder.getInputBuffers(); 728 audioDecoderOutputBuffers = audioDecoder.getOutputBuffers(); 729 audioEncoderInputBuffers = audioEncoder.getInputBuffers(); 730 audioEncoderOutputBuffers = audioEncoder.getOutputBuffers(); 731 audioDecoderOutputBufferInfo = new MediaCodec.BufferInfo(); 732 audioEncoderOutputBufferInfo = new MediaCodec.BufferInfo(); 733 } 734 // We will get these from the decoders when notified of a format change. 735 MediaFormat decoderOutputVideoFormat = null; 736 MediaFormat decoderOutputAudioFormat = null; 737 // We will get these from the encoders when notified of a format change. 738 MediaFormat encoderOutputVideoFormat = null; 739 MediaFormat encoderOutputAudioFormat = null; 740 // We will determine these once we have the output format. 741 int outputVideoTrack = -1; 742 int outputAudioTrack = -1; 743 // Whether things are done on the video side. 744 boolean videoExtractorDone = false; 745 boolean videoDecoderDone = false; 746 boolean videoEncoderDone = false; 747 // Whether things are done on the audio side. 748 boolean audioExtractorDone = false; 749 boolean audioDecoderDone = false; 750 boolean audioEncoderDone = false; 751 // The audio decoder output buffer to process, -1 if none. 752 int pendingAudioDecoderOutputBufferIndex = -1; 753 754 boolean muxing = false; 755 756 int videoExtractedFrameCount = 0; 757 int videoDecodedFrameCount = 0; 758 int videoEncodedFrameCount = 0; 759 760 int audioExtractedFrameCount = 0; 761 int audioDecodedFrameCount = 0; 762 int audioEncodedFrameCount = 0; 763 764 while ((mCopyVideo && !videoEncoderDone) || (mCopyAudio && !audioEncoderDone)) { 765 if (VERBOSE) { 766 Log.d(TAG, String.format( 767 "loop: " 768 769 + "V(%b){" 770 + "extracted:%d(done:%b) " 771 + "decoded:%d(done:%b) " 772 + "encoded:%d(done:%b)} " 773 774 + "A(%b){" 775 + "extracted:%d(done:%b) " 776 + "decoded:%d(done:%b) " 777 + "encoded:%d(done:%b) " 778 + "pending:%d} " 779 780 + "muxing:%b(V:%d,A:%d)", 781 782 mCopyVideo, 783 videoExtractedFrameCount, videoExtractorDone, 784 videoDecodedFrameCount, videoDecoderDone, 785 videoEncodedFrameCount, videoEncoderDone, 786 787 mCopyAudio, 788 audioExtractedFrameCount, audioExtractorDone, 789 audioDecodedFrameCount, audioDecoderDone, 790 audioEncodedFrameCount, audioEncoderDone, 791 pendingAudioDecoderOutputBufferIndex, 792 793 muxing, outputVideoTrack, outputAudioTrack)); 794 } 795 796 // Extract video from file and feed to decoder. 797 // Do not extract video if we have determined the output format but we are not yet 798 // ready to mux the frames. 799 while (mCopyVideo && !videoExtractorDone 800 && (encoderOutputVideoFormat == null || muxing)) { 801 int decoderInputBufferIndex = videoDecoder.dequeueInputBuffer(TIMEOUT_USEC); 802 if (decoderInputBufferIndex == MediaCodec.INFO_TRY_AGAIN_LATER) { 803 if (VERBOSE) Log.d(TAG, "no video decoder input buffer"); 804 break; 805 } 806 if (VERBOSE) { 807 Log.d(TAG, "video decoder: returned input buffer: " + decoderInputBufferIndex); 808 } 809 ByteBuffer decoderInputBuffer = videoDecoderInputBuffers[decoderInputBufferIndex]; 810 int size = videoExtractor.readSampleData(decoderInputBuffer, 0); 811 long presentationTime = videoExtractor.getSampleTime(); 812 int flags = videoExtractor.getSampleFlags(); 813 if (VERBOSE) { 814 Log.d(TAG, "video extractor: returned buffer of size " + size); 815 Log.d(TAG, "video extractor: returned buffer for time " + presentationTime); 816 } 817 videoExtractorDone = !videoExtractor.advance(); 818 if (videoExtractorDone) { 819 if (VERBOSE) Log.d(TAG, "video extractor: EOS"); 820 flags = flags | MediaCodec.BUFFER_FLAG_END_OF_STREAM; 821 } 822 if (size >= 0) { 823 videoDecoder.queueInputBuffer( 824 decoderInputBufferIndex, 825 0, 826 size, 827 presentationTime, 828 flags); 829 videoExtractedFrameCount++; 830 } 831 // We extracted a frame, let's try something else next. 832 break; 833 } 834 835 // Extract audio from file and feed to decoder. 836 // Do not extract audio if we have determined the output format but we are not yet 837 // ready to mux the frames. 838 while (mCopyAudio && !audioExtractorDone 839 && (encoderOutputAudioFormat == null || muxing)) { 840 int decoderInputBufferIndex = audioDecoder.dequeueInputBuffer(TIMEOUT_USEC); 841 if (decoderInputBufferIndex == MediaCodec.INFO_TRY_AGAIN_LATER) { 842 if (VERBOSE) Log.d(TAG, "no audio decoder input buffer"); 843 break; 844 } 845 if (VERBOSE) { 846 Log.d(TAG, "audio decoder: returned input buffer: " + decoderInputBufferIndex); 847 } 848 ByteBuffer decoderInputBuffer = audioDecoderInputBuffers[decoderInputBufferIndex]; 849 int size = audioExtractor.readSampleData(decoderInputBuffer, 0); 850 long presentationTime = audioExtractor.getSampleTime(); 851 if (VERBOSE) { 852 Log.d(TAG, "audio extractor: returned buffer of size " + size); 853 Log.d(TAG, "audio extractor: returned buffer for time " + presentationTime); 854 } 855 if (size >= 0) { 856 audioDecoder.queueInputBuffer( 857 decoderInputBufferIndex, 858 0, 859 size, 860 presentationTime, 861 audioExtractor.getSampleFlags()); 862 } 863 audioExtractorDone = !audioExtractor.advance(); 864 if (audioExtractorDone) { 865 if (VERBOSE) Log.d(TAG, "audio extractor: EOS"); 866 audioDecoder.queueInputBuffer( 867 decoderInputBufferIndex, 868 0, 869 0, 870 0, 871 MediaCodec.BUFFER_FLAG_END_OF_STREAM); 872 } 873 audioExtractedFrameCount++; 874 // We extracted a frame, let's try something else next. 875 break; 876 } 877 878 // Poll output frames from the video decoder and feed the encoder. 879 while (mCopyVideo && !videoDecoderDone 880 && (encoderOutputVideoFormat == null || muxing)) { 881 int decoderOutputBufferIndex = 882 videoDecoder.dequeueOutputBuffer( 883 videoDecoderOutputBufferInfo, TIMEOUT_USEC); 884 if (decoderOutputBufferIndex == MediaCodec.INFO_TRY_AGAIN_LATER) { 885 if (VERBOSE) Log.d(TAG, "no video decoder output buffer"); 886 break; 887 } 888 if (decoderOutputBufferIndex == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) { 889 if (VERBOSE) Log.d(TAG, "video decoder: output buffers changed"); 890 videoDecoderOutputBuffers = videoDecoder.getOutputBuffers(); 891 break; 892 } 893 if (decoderOutputBufferIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) { 894 decoderOutputVideoFormat = videoDecoder.getOutputFormat(); 895 if (VERBOSE) { 896 Log.d(TAG, "video decoder: output format changed: " 897 + decoderOutputVideoFormat); 898 } 899 break; 900 } 901 if (VERBOSE) { 902 Log.d(TAG, "video decoder: returned output buffer: " 903 + decoderOutputBufferIndex); 904 Log.d(TAG, "video decoder: returned buffer of size " 905 + videoDecoderOutputBufferInfo.size); 906 } 907 ByteBuffer decoderOutputBuffer = 908 videoDecoderOutputBuffers[decoderOutputBufferIndex]; 909 if ((videoDecoderOutputBufferInfo.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) 910 != 0) { 911 if (VERBOSE) Log.d(TAG, "video decoder: codec config buffer"); 912 videoDecoder.releaseOutputBuffer(decoderOutputBufferIndex, false); 913 break; 914 } 915 if (VERBOSE) { 916 Log.d(TAG, "video decoder: returned buffer for time " 917 + videoDecoderOutputBufferInfo.presentationTimeUs); 918 } 919 boolean render = videoDecoderOutputBufferInfo.size != 0; 920 videoDecoder.releaseOutputBuffer(decoderOutputBufferIndex, render); 921 if (render) { 922 if (VERBOSE) Log.d(TAG, "output surface: await new image"); 923 outputSurface.awaitNewImage(); 924 // Edit the frame and send it to the encoder. 925 if (VERBOSE) Log.d(TAG, "output surface: draw image"); 926 outputSurface.drawImage(); 927 inputSurface.setPresentationTime( 928 videoDecoderOutputBufferInfo.presentationTimeUs * 1000); 929 if (VERBOSE) Log.d(TAG, "input surface: swap buffers"); 930 inputSurface.swapBuffers(); 931 if (VERBOSE) Log.d(TAG, "video encoder: notified of new frame"); 932 videoDecodedFrameCount++; 933 } 934 if ((videoDecoderOutputBufferInfo.flags 935 & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) { 936 if (VERBOSE) Log.d(TAG, "video decoder: EOS"); 937 videoDecoderDone = true; 938 videoEncoder.signalEndOfInputStream(); 939 } 940 // We extracted a pending frame, let's try something else next. 941 break; 942 } 943 944 // Poll output frames from the audio decoder. 945 // Do not poll if we already have a pending buffer to feed to the encoder. 946 while (mCopyAudio && !audioDecoderDone && pendingAudioDecoderOutputBufferIndex == -1 947 && (encoderOutputAudioFormat == null || muxing)) { 948 int decoderOutputBufferIndex = 949 audioDecoder.dequeueOutputBuffer( 950 audioDecoderOutputBufferInfo, TIMEOUT_USEC); 951 if (decoderOutputBufferIndex == MediaCodec.INFO_TRY_AGAIN_LATER) { 952 if (VERBOSE) Log.d(TAG, "no audio decoder output buffer"); 953 break; 954 } 955 if (decoderOutputBufferIndex == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) { 956 if (VERBOSE) Log.d(TAG, "audio decoder: output buffers changed"); 957 audioDecoderOutputBuffers = audioDecoder.getOutputBuffers(); 958 break; 959 } 960 if (decoderOutputBufferIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) { 961 decoderOutputAudioFormat = audioDecoder.getOutputFormat(); 962 if (VERBOSE) { 963 Log.d(TAG, "audio decoder: output format changed: " 964 + decoderOutputAudioFormat); 965 } 966 break; 967 } 968 if (VERBOSE) { 969 Log.d(TAG, "audio decoder: returned output buffer: " 970 + decoderOutputBufferIndex); 971 } 972 if (VERBOSE) { 973 Log.d(TAG, "audio decoder: returned buffer of size " 974 + audioDecoderOutputBufferInfo.size); 975 } 976 ByteBuffer decoderOutputBuffer = 977 audioDecoderOutputBuffers[decoderOutputBufferIndex]; 978 if ((audioDecoderOutputBufferInfo.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) 979 != 0) { 980 if (VERBOSE) Log.d(TAG, "audio decoder: codec config buffer"); 981 audioDecoder.releaseOutputBuffer(decoderOutputBufferIndex, false); 982 break; 983 } 984 if (VERBOSE) { 985 Log.d(TAG, "audio decoder: returned buffer for time " 986 + audioDecoderOutputBufferInfo.presentationTimeUs); 987 } 988 if (VERBOSE) { 989 Log.d(TAG, "audio decoder: output buffer is now pending: " 990 + pendingAudioDecoderOutputBufferIndex); 991 } 992 pendingAudioDecoderOutputBufferIndex = decoderOutputBufferIndex; 993 audioDecodedFrameCount++; 994 // We extracted a pending frame, let's try something else next. 995 break; 996 } 997 998 // Feed the pending decoded audio buffer to the audio encoder. 999 while (mCopyAudio && pendingAudioDecoderOutputBufferIndex != -1) { 1000 if (VERBOSE) { 1001 Log.d(TAG, "audio decoder: attempting to process pending buffer: " 1002 + pendingAudioDecoderOutputBufferIndex); 1003 } 1004 int encoderInputBufferIndex = audioEncoder.dequeueInputBuffer(TIMEOUT_USEC); 1005 if (encoderInputBufferIndex == MediaCodec.INFO_TRY_AGAIN_LATER) { 1006 if (VERBOSE) Log.d(TAG, "no audio encoder input buffer"); 1007 break; 1008 } 1009 if (VERBOSE) { 1010 Log.d(TAG, "audio encoder: returned input buffer: " + encoderInputBufferIndex); 1011 } 1012 ByteBuffer encoderInputBuffer = audioEncoderInputBuffers[encoderInputBufferIndex]; 1013 int size = audioDecoderOutputBufferInfo.size; 1014 long presentationTime = audioDecoderOutputBufferInfo.presentationTimeUs; 1015 if (VERBOSE) { 1016 Log.d(TAG, "audio decoder: processing pending buffer: " 1017 + pendingAudioDecoderOutputBufferIndex); 1018 } 1019 if (VERBOSE) { 1020 Log.d(TAG, "audio decoder: pending buffer of size " + size); 1021 Log.d(TAG, "audio decoder: pending buffer for time " + presentationTime); 1022 } 1023 if (size >= 0) { 1024 ByteBuffer decoderOutputBuffer = 1025 audioDecoderOutputBuffers[pendingAudioDecoderOutputBufferIndex] 1026 .duplicate(); 1027 decoderOutputBuffer.position(audioDecoderOutputBufferInfo.offset); 1028 decoderOutputBuffer.limit(audioDecoderOutputBufferInfo.offset + size); 1029 encoderInputBuffer.position(0); 1030 encoderInputBuffer.put(decoderOutputBuffer); 1031 1032 audioEncoder.queueInputBuffer( 1033 encoderInputBufferIndex, 1034 0, 1035 size, 1036 presentationTime, 1037 audioDecoderOutputBufferInfo.flags); 1038 } 1039 audioDecoder.releaseOutputBuffer(pendingAudioDecoderOutputBufferIndex, false); 1040 pendingAudioDecoderOutputBufferIndex = -1; 1041 if ((audioDecoderOutputBufferInfo.flags 1042 & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) { 1043 if (VERBOSE) Log.d(TAG, "audio decoder: EOS"); 1044 audioDecoderDone = true; 1045 } 1046 // We enqueued a pending frame, let's try something else next. 1047 break; 1048 } 1049 1050 // Poll frames from the video encoder and send them to the muxer. 1051 while (mCopyVideo && !videoEncoderDone 1052 && (encoderOutputVideoFormat == null || muxing)) { 1053 int encoderOutputBufferIndex = videoEncoder.dequeueOutputBuffer( 1054 videoEncoderOutputBufferInfo, TIMEOUT_USEC); 1055 if (encoderOutputBufferIndex == MediaCodec.INFO_TRY_AGAIN_LATER) { 1056 if (VERBOSE) Log.d(TAG, "no video encoder output buffer"); 1057 break; 1058 } 1059 if (encoderOutputBufferIndex == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) { 1060 if (VERBOSE) Log.d(TAG, "video encoder: output buffers changed"); 1061 videoEncoderOutputBuffers = videoEncoder.getOutputBuffers(); 1062 break; 1063 } 1064 if (encoderOutputBufferIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) { 1065 if (VERBOSE) Log.d(TAG, "video encoder: output format changed"); 1066 if (outputVideoTrack >= 0) { 1067 fail("video encoder changed its output format again?"); 1068 } 1069 encoderOutputVideoFormat = videoEncoder.getOutputFormat(); 1070 break; 1071 } 1072 assertTrue("should have added track before processing output", muxing); 1073 if (VERBOSE) { 1074 Log.d(TAG, "video encoder: returned output buffer: " 1075 + encoderOutputBufferIndex); 1076 Log.d(TAG, "video encoder: returned buffer of size " 1077 + videoEncoderOutputBufferInfo.size); 1078 } 1079 ByteBuffer encoderOutputBuffer = 1080 videoEncoderOutputBuffers[encoderOutputBufferIndex]; 1081 if ((videoEncoderOutputBufferInfo.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) 1082 != 0) { 1083 if (VERBOSE) Log.d(TAG, "video encoder: codec config buffer"); 1084 // Simply ignore codec config buffers. 1085 videoEncoder.releaseOutputBuffer(encoderOutputBufferIndex, false); 1086 break; 1087 } 1088 if (VERBOSE) { 1089 Log.d(TAG, "video encoder: returned buffer for time " 1090 + videoEncoderOutputBufferInfo.presentationTimeUs); 1091 } 1092 if (videoEncoderOutputBufferInfo.size != 0) { 1093 muxer.writeSampleData( 1094 outputVideoTrack, encoderOutputBuffer, videoEncoderOutputBufferInfo); 1095 videoEncodedFrameCount++; 1096 } 1097 if ((videoEncoderOutputBufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) 1098 != 0) { 1099 if (VERBOSE) Log.d(TAG, "video encoder: EOS"); 1100 videoEncoderDone = true; 1101 } 1102 videoEncoder.releaseOutputBuffer(encoderOutputBufferIndex, false); 1103 // We enqueued an encoded frame, let's try something else next. 1104 break; 1105 } 1106 1107 // Poll frames from the audio encoder and send them to the muxer. 1108 while (mCopyAudio && !audioEncoderDone 1109 && (encoderOutputAudioFormat == null || muxing)) { 1110 int encoderOutputBufferIndex = audioEncoder.dequeueOutputBuffer( 1111 audioEncoderOutputBufferInfo, TIMEOUT_USEC); 1112 if (encoderOutputBufferIndex == MediaCodec.INFO_TRY_AGAIN_LATER) { 1113 if (VERBOSE) Log.d(TAG, "no audio encoder output buffer"); 1114 break; 1115 } 1116 if (encoderOutputBufferIndex == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) { 1117 if (VERBOSE) Log.d(TAG, "audio encoder: output buffers changed"); 1118 audioEncoderOutputBuffers = audioEncoder.getOutputBuffers(); 1119 break; 1120 } 1121 if (encoderOutputBufferIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) { 1122 if (VERBOSE) Log.d(TAG, "audio encoder: output format changed"); 1123 if (outputAudioTrack >= 0) { 1124 fail("audio encoder changed its output format again?"); 1125 } 1126 1127 encoderOutputAudioFormat = audioEncoder.getOutputFormat(); 1128 break; 1129 } 1130 assertTrue("should have added track before processing output", muxing); 1131 if (VERBOSE) { 1132 Log.d(TAG, "audio encoder: returned output buffer: " 1133 + encoderOutputBufferIndex); 1134 Log.d(TAG, "audio encoder: returned buffer of size " 1135 + audioEncoderOutputBufferInfo.size); 1136 } 1137 ByteBuffer encoderOutputBuffer = 1138 audioEncoderOutputBuffers[encoderOutputBufferIndex]; 1139 if ((audioEncoderOutputBufferInfo.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) 1140 != 0) { 1141 if (VERBOSE) Log.d(TAG, "audio encoder: codec config buffer"); 1142 // Simply ignore codec config buffers. 1143 audioEncoder.releaseOutputBuffer(encoderOutputBufferIndex, false); 1144 break; 1145 } 1146 if (VERBOSE) { 1147 Log.d(TAG, "audio encoder: returned buffer for time " 1148 + audioEncoderOutputBufferInfo.presentationTimeUs); 1149 } 1150 if (audioEncoderOutputBufferInfo.size != 0) { 1151 muxer.writeSampleData( 1152 outputAudioTrack, encoderOutputBuffer, audioEncoderOutputBufferInfo); 1153 } 1154 if ((audioEncoderOutputBufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) 1155 != 0) { 1156 if (VERBOSE) Log.d(TAG, "audio encoder: EOS"); 1157 audioEncoderDone = true; 1158 } 1159 audioEncoder.releaseOutputBuffer(encoderOutputBufferIndex, false); 1160 audioEncodedFrameCount++; 1161 // We enqueued an encoded frame, let's try something else next. 1162 break; 1163 } 1164 1165 if (!muxing 1166 && (!mCopyAudio || encoderOutputAudioFormat != null) 1167 && (!mCopyVideo || encoderOutputVideoFormat != null)) { 1168 if (mCopyVideo) { 1169 Log.d(TAG, "muxer: adding video track."); 1170 outputVideoTrack = muxer.addTrack(encoderOutputVideoFormat); 1171 } 1172 if (mCopyAudio) { 1173 Log.d(TAG, "muxer: adding audio track."); 1174 outputAudioTrack = muxer.addTrack(encoderOutputAudioFormat); 1175 } 1176 Log.d(TAG, "muxer: starting"); 1177 muxer.start(); 1178 muxing = true; 1179 } 1180 } 1181 1182 // Basic validation checks. 1183 if (mCopyVideo) { 1184 assertEquals("encoded and decoded video frame counts should match", 1185 videoDecodedFrameCount, videoEncodedFrameCount); 1186 assertTrue("decoded frame count should be less than extracted frame count", 1187 videoDecodedFrameCount <= videoExtractedFrameCount); 1188 } 1189 if (mCopyAudio) { 1190 assertEquals("no frame should be pending", -1, pendingAudioDecoderOutputBufferIndex); 1191 } 1192 } 1193 isVideoFormat(MediaFormat format)1194 private static boolean isVideoFormat(MediaFormat format) { 1195 return getMimeTypeFor(format).startsWith("video/"); 1196 } 1197 isAudioFormat(MediaFormat format)1198 private static boolean isAudioFormat(MediaFormat format) { 1199 return getMimeTypeFor(format).startsWith("audio/"); 1200 } 1201 getMimeTypeFor(MediaFormat format)1202 private static String getMimeTypeFor(MediaFormat format) { 1203 return format.getString(MediaFormat.KEY_MIME); 1204 } 1205 1206 /** 1207 * Returns the first codec capable of encoding the specified MIME type, or null if no match was 1208 * found. 1209 */ selectCodec(String mimeType)1210 private static MediaCodecInfo selectCodec(String mimeType) { 1211 int numCodecs = MediaCodecList.getCodecCount(); 1212 for (int i = 0; i < numCodecs; i++) { 1213 MediaCodecInfo codecInfo = MediaCodecList.getCodecInfoAt(i); 1214 1215 if (codecInfo.isAlias()) { 1216 continue; 1217 } 1218 if (!codecInfo.isEncoder()) { 1219 continue; 1220 } 1221 1222 String[] types = codecInfo.getSupportedTypes(); 1223 for (int j = 0; j < types.length; j++) { 1224 if (types[j].equalsIgnoreCase(mimeType)) { 1225 return codecInfo; 1226 } 1227 } 1228 } 1229 return null; 1230 } 1231 } 1232