1 /* 2 * Copyright (C) 2012 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package android.media; 18 19 import android.annotation.IntDef; 20 import android.annotation.NonNull; 21 import android.annotation.Nullable; 22 import android.compat.annotation.UnsupportedAppUsage; 23 import android.graphics.ImageFormat; 24 import android.graphics.Rect; 25 import android.graphics.SurfaceTexture; 26 import android.media.MediaCodecInfo.CodecCapabilities; 27 import android.os.Build; 28 import android.os.Bundle; 29 import android.os.Handler; 30 import android.os.IHwBinder; 31 import android.os.Looper; 32 import android.os.Message; 33 import android.os.PersistableBundle; 34 import android.view.Surface; 35 36 import java.io.IOException; 37 import java.lang.annotation.Retention; 38 import java.lang.annotation.RetentionPolicy; 39 import java.nio.ByteBuffer; 40 import java.nio.ByteOrder; 41 import java.nio.ReadOnlyBufferException; 42 import java.util.Arrays; 43 import java.util.HashMap; 44 import java.util.Map; 45 import java.util.concurrent.locks.Lock; 46 import java.util.concurrent.locks.ReentrantLock; 47 48 /** 49 MediaCodec class can be used to access low-level media codecs, i.e. encoder/decoder components. 50 It is part of the Android low-level multimedia support infrastructure (normally used together 51 with {@link MediaExtractor}, {@link MediaSync}, {@link MediaMuxer}, {@link MediaCrypto}, 52 {@link MediaDrm}, {@link Image}, {@link Surface}, and {@link AudioTrack}.) 53 <p> 54 <center><object style="width: 540px; height: 205px;" type="image/svg+xml" 55 data="../../../images/media/mediacodec_buffers.svg"><img 56 src="../../../images/media/mediacodec_buffers.png" style="width: 540px; height: 205px" 57 alt="MediaCodec buffer flow diagram"></object></center> 58 <p> 59 In broad terms, a codec processes input data to generate output data. It processes data 60 asynchronously and uses a set of input and output buffers. At a simplistic level, you request 61 (or receive) an empty input buffer, fill it up with data and send it to the codec for 62 processing. The codec uses up the data and transforms it into one of its empty output buffers. 63 Finally, you request (or receive) a filled output buffer, consume its contents and release it 64 back to the codec. 65 66 <h3>Data Types</h3> 67 <p> 68 Codecs operate on three kinds of data: compressed data, raw audio data and raw video data. 69 All three kinds of data can be processed using {@link ByteBuffer ByteBuffers}, but you should use 70 a {@link Surface} for raw video data to improve codec performance. Surface uses native video 71 buffers without mapping or copying them to ByteBuffers; thus, it is much more efficient. 72 You normally cannot access the raw video data when using a Surface, but you can use the 73 {@link ImageReader} class to access unsecured decoded (raw) video frames. This may still be more 74 efficient than using ByteBuffers, as some native buffers may be mapped into {@linkplain 75 ByteBuffer#isDirect direct} ByteBuffers. When using ByteBuffer mode, you can access raw video 76 frames using the {@link Image} class and {@link #getInputImage getInput}/{@link #getOutputImage 77 OutputImage(int)}. 78 79 <h4>Compressed Buffers</h4> 80 <p> 81 Input buffers (for decoders) and output buffers (for encoders) contain compressed data according 82 to the {@linkplain MediaFormat#KEY_MIME format's type}. For video types this is normally a single 83 compressed video frame. For audio data this is normally a single access unit (an encoded audio 84 segment typically containing a few milliseconds of audio as dictated by the format type), but 85 this requirement is slightly relaxed in that a buffer may contain multiple encoded access units 86 of audio. In either case, buffers do not start or end on arbitrary byte boundaries, but rather on 87 frame/access unit boundaries unless they are flagged with {@link #BUFFER_FLAG_PARTIAL_FRAME}. 88 89 <h4>Raw Audio Buffers</h4> 90 <p> 91 Raw audio buffers contain entire frames of PCM audio data, which is one sample for each channel 92 in channel order. Each PCM audio sample is either a 16 bit signed integer or a float, 93 in native byte order. 94 Raw audio buffers in the float PCM encoding are only possible 95 if the MediaFormat's {@linkplain MediaFormat#KEY_PCM_ENCODING} 96 is set to {@linkplain AudioFormat#ENCODING_PCM_FLOAT} during MediaCodec 97 {@link #configure configure(…)} 98 and confirmed by {@link #getOutputFormat} for decoders 99 or {@link #getInputFormat} for encoders. 100 A sample method to check for float PCM in the MediaFormat is as follows: 101 102 <pre class=prettyprint> 103 static boolean isPcmFloat(MediaFormat format) { 104 return format.getInteger(MediaFormat.KEY_PCM_ENCODING, AudioFormat.ENCODING_PCM_16BIT) 105 == AudioFormat.ENCODING_PCM_FLOAT; 106 }</pre> 107 108 In order to extract, in a short array, 109 one channel of a buffer containing 16 bit signed integer audio data, 110 the following code may be used: 111 112 <pre class=prettyprint> 113 // Assumes the buffer PCM encoding is 16 bit. 114 short[] getSamplesForChannel(MediaCodec codec, int bufferId, int channelIx) { 115 ByteBuffer outputBuffer = codec.getOutputBuffer(bufferId); 116 MediaFormat format = codec.getOutputFormat(bufferId); 117 ShortBuffer samples = outputBuffer.order(ByteOrder.nativeOrder()).asShortBuffer(); 118 int numChannels = format.getInteger(MediaFormat.KEY_CHANNEL_COUNT); 119 if (channelIx < 0 || channelIx >= numChannels) { 120 return null; 121 } 122 short[] res = new short[samples.remaining() / numChannels]; 123 for (int i = 0; i < res.length; ++i) { 124 res[i] = samples.get(i * numChannels + channelIx); 125 } 126 return res; 127 }</pre> 128 129 <h4>Raw Video Buffers</h4> 130 <p> 131 In ByteBuffer mode video buffers are laid out according to their {@linkplain 132 MediaFormat#KEY_COLOR_FORMAT color format}. You can get the supported color formats as an array 133 from {@link #getCodecInfo}{@code .}{@link MediaCodecInfo#getCapabilitiesForType 134 getCapabilitiesForType(…)}{@code .}{@link CodecCapabilities#colorFormats colorFormats}. 135 Video codecs may support three kinds of color formats: 136 <ul> 137 <li><strong>native raw video format:</strong> This is marked by {@link 138 CodecCapabilities#COLOR_FormatSurface} and it can be used with an input or output Surface.</li> 139 <li><strong>flexible YUV buffers</strong> (such as {@link 140 CodecCapabilities#COLOR_FormatYUV420Flexible}): These can be used with an input/output Surface, 141 as well as in ByteBuffer mode, by using {@link #getInputImage getInput}/{@link #getOutputImage 142 OutputImage(int)}.</li> 143 <li><strong>other, specific formats:</strong> These are normally only supported in ByteBuffer 144 mode. Some color formats are vendor specific. Others are defined in {@link CodecCapabilities}. 145 For color formats that are equivalent to a flexible format, you can still use {@link 146 #getInputImage getInput}/{@link #getOutputImage OutputImage(int)}.</li> 147 </ul> 148 <p> 149 All video codecs support flexible YUV 4:2:0 buffers since {@link 150 android.os.Build.VERSION_CODES#LOLLIPOP_MR1}. 151 152 <h4>Accessing Raw Video ByteBuffers on Older Devices</h4> 153 <p> 154 Prior to {@link android.os.Build.VERSION_CODES#LOLLIPOP} and {@link Image} support, you need to 155 use the {@link MediaFormat#KEY_STRIDE} and {@link MediaFormat#KEY_SLICE_HEIGHT} output format 156 values to understand the layout of the raw output buffers. 157 <p class=note> 158 Note that on some devices the slice-height is advertised as 0. This could mean either that the 159 slice-height is the same as the frame height, or that the slice-height is the frame height 160 aligned to some value (usually a power of 2). Unfortunately, there is no standard and simple way 161 to tell the actual slice height in this case. Furthermore, the vertical stride of the {@code U} 162 plane in planar formats is also not specified or defined, though usually it is half of the slice 163 height. 164 <p> 165 The {@link MediaFormat#KEY_WIDTH} and {@link MediaFormat#KEY_HEIGHT} keys specify the size of the 166 video frames; however, for most encondings the video (picture) only occupies a portion of the 167 video frame. This is represented by the 'crop rectangle'. 168 <p> 169 You need to use the following keys to get the crop rectangle of raw output images from the 170 {@linkplain #getOutputFormat output format}. If these keys are not present, the video occupies the 171 entire video frame.The crop rectangle is understood in the context of the output frame 172 <em>before</em> applying any {@linkplain MediaFormat#KEY_ROTATION rotation}. 173 <table style="width: 0%"> 174 <thead> 175 <tr> 176 <th>Format Key</th> 177 <th>Type</th> 178 <th>Description</th> 179 </tr> 180 </thead> 181 <tbody> 182 <tr> 183 <td>{@code "crop-left"}</td> 184 <td>Integer</td> 185 <td>The left-coordinate (x) of the crop rectangle</td> 186 </tr><tr> 187 <td>{@code "crop-top"}</td> 188 <td>Integer</td> 189 <td>The top-coordinate (y) of the crop rectangle</td> 190 </tr><tr> 191 <td>{@code "crop-right"}</td> 192 <td>Integer</td> 193 <td>The right-coordinate (x) <strong>MINUS 1</strong> of the crop rectangle</td> 194 </tr><tr> 195 <td>{@code "crop-bottom"}</td> 196 <td>Integer</td> 197 <td>The bottom-coordinate (y) <strong>MINUS 1</strong> of the crop rectangle</td> 198 </tr><tr> 199 <td colspan=3> 200 The right and bottom coordinates can be understood as the coordinates of the right-most 201 valid column/bottom-most valid row of the cropped output image. 202 </td> 203 </tr> 204 </tbody> 205 </table> 206 <p> 207 The size of the video frame (before rotation) can be calculated as such: 208 <pre class=prettyprint> 209 MediaFormat format = decoder.getOutputFormat(…); 210 int width = format.getInteger(MediaFormat.KEY_WIDTH); 211 if (format.containsKey("crop-left") && format.containsKey("crop-right")) { 212 width = format.getInteger("crop-right") + 1 - format.getInteger("crop-left"); 213 } 214 int height = format.getInteger(MediaFormat.KEY_HEIGHT); 215 if (format.containsKey("crop-top") && format.containsKey("crop-bottom")) { 216 height = format.getInteger("crop-bottom") + 1 - format.getInteger("crop-top"); 217 } 218 </pre> 219 <p class=note> 220 Also note that the meaning of {@link BufferInfo#offset BufferInfo.offset} was not consistent across 221 devices. On some devices the offset pointed to the top-left pixel of the crop rectangle, while on 222 most devices it pointed to the top-left pixel of the entire frame. 223 224 <h3>States</h3> 225 <p> 226 During its life a codec conceptually exists in one of three states: Stopped, Executing or 227 Released. The Stopped collective state is actually the conglomeration of three states: 228 Uninitialized, Configured and Error, whereas the Executing state conceptually progresses through 229 three sub-states: Flushed, Running and End-of-Stream. 230 <p> 231 <center><object style="width: 516px; height: 353px;" type="image/svg+xml" 232 data="../../../images/media/mediacodec_states.svg"><img 233 src="../../../images/media/mediacodec_states.png" style="width: 519px; height: 356px" 234 alt="MediaCodec state diagram"></object></center> 235 <p> 236 When you create a codec using one of the factory methods, the codec is in the Uninitialized 237 state. First, you need to configure it via {@link #configure configure(…)}, which brings 238 it to the Configured state, then call {@link #start} to move it to the Executing state. In this 239 state you can process data through the buffer queue manipulation described above. 240 <p> 241 The Executing state has three sub-states: Flushed, Running and End-of-Stream. Immediately after 242 {@link #start} the codec is in the Flushed sub-state, where it holds all the buffers. As soon 243 as the first input buffer is dequeued, the codec moves to the Running sub-state, where it spends 244 most of its life. When you queue an input buffer with the {@linkplain #BUFFER_FLAG_END_OF_STREAM 245 end-of-stream marker}, the codec transitions to the End-of-Stream sub-state. In this state the 246 codec no longer accepts further input buffers, but still generates output buffers until the 247 end-of-stream is reached on the output. You can move back to the Flushed sub-state at any time 248 while in the Executing state using {@link #flush}. 249 <p> 250 Call {@link #stop} to return the codec to the Uninitialized state, whereupon it may be configured 251 again. When you are done using a codec, you must release it by calling {@link #release}. 252 <p> 253 On rare occasions the codec may encounter an error and move to the Error state. This is 254 communicated using an invalid return value from a queuing operation, or sometimes via an 255 exception. Call {@link #reset} to make the codec usable again. You can call it from any state to 256 move the codec back to the Uninitialized state. Otherwise, call {@link #release} to move to the 257 terminal Released state. 258 259 <h3>Creation</h3> 260 <p> 261 Use {@link MediaCodecList} to create a MediaCodec for a specific {@link MediaFormat}. When 262 decoding a file or a stream, you can get the desired format from {@link 263 MediaExtractor#getTrackFormat MediaExtractor.getTrackFormat}. Inject any specific features that 264 you want to add using {@link MediaFormat#setFeatureEnabled MediaFormat.setFeatureEnabled}, then 265 call {@link MediaCodecList#findDecoderForFormat MediaCodecList.findDecoderForFormat} to get the 266 name of a codec that can handle that specific media format. Finally, create the codec using 267 {@link #createByCodecName}. 268 <p class=note> 269 <strong>Note:</strong> On {@link android.os.Build.VERSION_CODES#LOLLIPOP}, the format to 270 {@code MediaCodecList.findDecoder}/{@code EncoderForFormat} must not contain a {@linkplain 271 MediaFormat#KEY_FRAME_RATE frame rate}. Use 272 <code class=prettyprint>format.setString(MediaFormat.KEY_FRAME_RATE, null)</code> 273 to clear any existing frame rate setting in the format. 274 <p> 275 You can also create the preferred codec for a specific MIME type using {@link 276 #createDecoderByType createDecoder}/{@link #createEncoderByType EncoderByType(String)}. 277 This, however, cannot be used to inject features, and may create a codec that cannot handle the 278 specific desired media format. 279 280 <h4>Creating secure decoders</h4> 281 <p> 282 On versions {@link android.os.Build.VERSION_CODES#KITKAT_WATCH} and earlier, secure codecs might 283 not be listed in {@link MediaCodecList}, but may still be available on the system. Secure codecs 284 that exist can be instantiated by name only, by appending {@code ".secure"} to the name of a 285 regular codec (the name of all secure codecs must end in {@code ".secure"}.) {@link 286 #createByCodecName} will throw an {@code IOException} if the codec is not present on the system. 287 <p> 288 From {@link android.os.Build.VERSION_CODES#LOLLIPOP} onwards, you should use the {@link 289 CodecCapabilities#FEATURE_SecurePlayback} feature in the media format to create a secure decoder. 290 291 <h3>Initialization</h3> 292 <p> 293 After creating the codec, you can set a callback using {@link #setCallback setCallback} if you 294 want to process data asynchronously. Then, {@linkplain #configure configure} the codec using the 295 specific media format. This is when you can specify the output {@link Surface} for video 296 producers – codecs that generate raw video data (e.g. video decoders). This is also when 297 you can set the decryption parameters for secure codecs (see {@link MediaCrypto}). Finally, since 298 some codecs can operate in multiple modes, you must specify whether you want it to work as a 299 decoder or an encoder. 300 <p> 301 Since {@link android.os.Build.VERSION_CODES#LOLLIPOP}, you can query the resulting input and 302 output format in the Configured state. You can use this to verify the resulting configuration, 303 e.g. color formats, before starting the codec. 304 <p> 305 If you want to process raw input video buffers natively with a video consumer – a codec 306 that processes raw video input, such as a video encoder – create a destination Surface for 307 your input data using {@link #createInputSurface} after configuration. Alternately, set up the 308 codec to use a previously created {@linkplain #createPersistentInputSurface persistent input 309 surface} by calling {@link #setInputSurface}. 310 311 <h4 id=CSD><a name="CSD"></a>Codec-specific Data</h4> 312 <p> 313 Some formats, notably AAC audio and MPEG4, H.264 and H.265 video formats require the actual data 314 to be prefixed by a number of buffers containing setup data, or codec specific data. When 315 processing such compressed formats, this data must be submitted to the codec after {@link 316 #start} and before any frame data. Such data must be marked using the flag {@link 317 #BUFFER_FLAG_CODEC_CONFIG} in a call to {@link #queueInputBuffer queueInputBuffer}. 318 <p> 319 Codec-specific data can also be included in the format passed to {@link #configure configure} in 320 ByteBuffer entries with keys "csd-0", "csd-1", etc. These keys are always included in the track 321 {@link MediaFormat} obtained from the {@link MediaExtractor#getTrackFormat MediaExtractor}. 322 Codec-specific data in the format is automatically submitted to the codec upon {@link #start}; 323 you <strong>MUST NOT</strong> submit this data explicitly. If the format did not contain codec 324 specific data, you can choose to submit it using the specified number of buffers in the correct 325 order, according to the format requirements. In case of H.264 AVC, you can also concatenate all 326 codec-specific data and submit it as a single codec-config buffer. 327 <p> 328 Android uses the following codec-specific data buffers. These are also required to be set in 329 the track format for proper {@link MediaMuxer} track configuration. Each parameter set and the 330 codec-specific-data sections marked with (<sup>*</sup>) must start with a start code of 331 {@code "\x00\x00\x00\x01"}. 332 <p> 333 <style>td.NA { background: #ccc; } .mid > tr > td { vertical-align: middle; }</style> 334 <table> 335 <thead> 336 <th>Format</th> 337 <th>CSD buffer #0</th> 338 <th>CSD buffer #1</th> 339 <th>CSD buffer #2</th> 340 </thead> 341 <tbody class=mid> 342 <tr> 343 <td>AAC</td> 344 <td>Decoder-specific information from ESDS<sup>*</sup></td> 345 <td class=NA>Not Used</td> 346 <td class=NA>Not Used</td> 347 </tr> 348 <tr> 349 <td>VORBIS</td> 350 <td>Identification header</td> 351 <td>Setup header</td> 352 <td class=NA>Not Used</td> 353 </tr> 354 <tr> 355 <td>OPUS</td> 356 <td>Identification header</td> 357 <td>Pre-skip in nanosecs<br> 358 (unsigned 64-bit {@linkplain ByteOrder#nativeOrder native-order} integer.)<br> 359 This overrides the pre-skip value in the identification header.</td> 360 <td>Seek Pre-roll in nanosecs<br> 361 (unsigned 64-bit {@linkplain ByteOrder#nativeOrder native-order} integer.)</td> 362 </tr> 363 <tr> 364 <td>FLAC</td> 365 <td>"fLaC", the FLAC stream marker in ASCII,<br> 366 followed by the STREAMINFO block (the mandatory metadata block),<br> 367 optionally followed by any number of other metadata blocks</td> 368 <td class=NA>Not Used</td> 369 <td class=NA>Not Used</td> 370 </tr> 371 <tr> 372 <td>MPEG-4</td> 373 <td>Decoder-specific information from ESDS<sup>*</sup></td> 374 <td class=NA>Not Used</td> 375 <td class=NA>Not Used</td> 376 </tr> 377 <tr> 378 <td>H.264 AVC</td> 379 <td>SPS (Sequence Parameter Sets<sup>*</sup>)</td> 380 <td>PPS (Picture Parameter Sets<sup>*</sup>)</td> 381 <td class=NA>Not Used</td> 382 </tr> 383 <tr> 384 <td>H.265 HEVC</td> 385 <td>VPS (Video Parameter Sets<sup>*</sup>) +<br> 386 SPS (Sequence Parameter Sets<sup>*</sup>) +<br> 387 PPS (Picture Parameter Sets<sup>*</sup>)</td> 388 <td class=NA>Not Used</td> 389 <td class=NA>Not Used</td> 390 </tr> 391 <tr> 392 <td>VP9</td> 393 <td>VP9 <a href="http://wiki.webmproject.org/vp9-codecprivate">CodecPrivate</a> Data 394 (optional)</td> 395 <td class=NA>Not Used</td> 396 <td class=NA>Not Used</td> 397 </tr> 398 </tbody> 399 </table> 400 401 <p class=note> 402 <strong>Note:</strong> care must be taken if the codec is flushed immediately or shortly 403 after start, before any output buffer or output format change has been returned, as the codec 404 specific data may be lost during the flush. You must resubmit the data using buffers marked with 405 {@link #BUFFER_FLAG_CODEC_CONFIG} after such flush to ensure proper codec operation. 406 <p> 407 Encoders (or codecs that generate compressed data) will create and return the codec specific data 408 before any valid output buffer in output buffers marked with the {@linkplain 409 #BUFFER_FLAG_CODEC_CONFIG codec-config flag}. Buffers containing codec-specific-data have no 410 meaningful timestamps. 411 412 <h3>Data Processing</h3> 413 <p> 414 Each codec maintains a set of input and output buffers that are referred to by a buffer-ID in 415 API calls. After a successful call to {@link #start} the client "owns" neither input nor output 416 buffers. In synchronous mode, call {@link #dequeueInputBuffer dequeueInput}/{@link 417 #dequeueOutputBuffer OutputBuffer(…)} to obtain (get ownership of) an input or output 418 buffer from the codec. In asynchronous mode, you will automatically receive available buffers via 419 the {@link Callback#onInputBufferAvailable MediaCodec.Callback.onInput}/{@link 420 Callback#onOutputBufferAvailable OutputBufferAvailable(…)} callbacks. 421 <p> 422 Upon obtaining an input buffer, fill it with data and submit it to the codec using {@link 423 #queueInputBuffer queueInputBuffer} – or {@link #queueSecureInputBuffer 424 queueSecureInputBuffer} if using decryption. Do not submit multiple input buffers with the same 425 timestamp (unless it is <a href="#CSD">codec-specific data</a> marked as such). 426 <p> 427 The codec in turn will return a read-only output buffer via the {@link 428 Callback#onOutputBufferAvailable onOutputBufferAvailable} callback in asynchronous mode, or in 429 response to a {@link #dequeueOutputBuffer dequeueOutputBuffer} call in synchronous mode. After the 430 output buffer has been processed, call one of the {@link #releaseOutputBuffer 431 releaseOutputBuffer} methods to return the buffer to the codec. 432 <p> 433 While you are not required to resubmit/release buffers immediately to the codec, holding onto 434 input and/or output buffers may stall the codec, and this behavior is device dependent. 435 <strong>Specifically, it is possible that a codec may hold off on generating output buffers until 436 <em>all</em> outstanding buffers have been released/resubmitted.</strong> Therefore, try to 437 hold onto to available buffers as little as possible. 438 <p> 439 Depending on the API version, you can process data in three ways: 440 <table> 441 <thead> 442 <tr> 443 <th>Processing Mode</th> 444 <th>API version <= 20<br>Jelly Bean/KitKat</th> 445 <th>API version >= 21<br>Lollipop and later</th> 446 </tr> 447 </thead> 448 <tbody> 449 <tr> 450 <td>Synchronous API using buffer arrays</td> 451 <td>Supported</td> 452 <td>Deprecated</td> 453 </tr> 454 <tr> 455 <td>Synchronous API using buffers</td> 456 <td class=NA>Not Available</td> 457 <td>Supported</td> 458 </tr> 459 <tr> 460 <td>Asynchronous API using buffers</td> 461 <td class=NA>Not Available</td> 462 <td>Supported</td> 463 </tr> 464 </tbody> 465 </table> 466 467 <h4>Asynchronous Processing using Buffers</h4> 468 <p> 469 Since {@link android.os.Build.VERSION_CODES#LOLLIPOP}, the preferred method is to process data 470 asynchronously by setting a callback before calling {@link #configure configure}. Asynchronous 471 mode changes the state transitions slightly, because you must call {@link #start} after {@link 472 #flush} to transition the codec to the Running sub-state and start receiving input buffers. 473 Similarly, upon an initial call to {@code start} the codec will move directly to the Running 474 sub-state and start passing available input buffers via the callback. 475 <p> 476 <center><object style="width: 516px; height: 353px;" type="image/svg+xml" 477 data="../../../images/media/mediacodec_async_states.svg"><img 478 src="../../../images/media/mediacodec_async_states.png" style="width: 516px; height: 353px" 479 alt="MediaCodec state diagram for asynchronous operation"></object></center> 480 <p> 481 MediaCodec is typically used like this in asynchronous mode: 482 <pre class=prettyprint> 483 MediaCodec codec = MediaCodec.createByCodecName(name); 484 MediaFormat mOutputFormat; // member variable 485 codec.setCallback(new MediaCodec.Callback() { 486 {@literal @Override} 487 void onInputBufferAvailable(MediaCodec mc, int inputBufferId) { 488 ByteBuffer inputBuffer = codec.getInputBuffer(inputBufferId); 489 // fill inputBuffer with valid data 490 … 491 codec.queueInputBuffer(inputBufferId, …); 492 } 493 494 {@literal @Override} 495 void onOutputBufferAvailable(MediaCodec mc, int outputBufferId, …) { 496 ByteBuffer outputBuffer = codec.getOutputBuffer(outputBufferId); 497 MediaFormat bufferFormat = codec.getOutputFormat(outputBufferId); // option A 498 // bufferFormat is equivalent to mOutputFormat 499 // outputBuffer is ready to be processed or rendered. 500 … 501 codec.releaseOutputBuffer(outputBufferId, …); 502 } 503 504 {@literal @Override} 505 void onOutputFormatChanged(MediaCodec mc, MediaFormat format) { 506 // Subsequent data will conform to new format. 507 // Can ignore if using getOutputFormat(outputBufferId) 508 mOutputFormat = format; // option B 509 } 510 511 {@literal @Override} 512 void onError(…) { 513 … 514 } 515 }); 516 codec.configure(format, …); 517 mOutputFormat = codec.getOutputFormat(); // option B 518 codec.start(); 519 // wait for processing to complete 520 codec.stop(); 521 codec.release();</pre> 522 523 <h4>Synchronous Processing using Buffers</h4> 524 <p> 525 Since {@link android.os.Build.VERSION_CODES#LOLLIPOP}, you should retrieve input and output 526 buffers using {@link #getInputBuffer getInput}/{@link #getOutputBuffer OutputBuffer(int)} and/or 527 {@link #getInputImage getInput}/{@link #getOutputImage OutputImage(int)} even when using the 528 codec in synchronous mode. This allows certain optimizations by the framework, e.g. when 529 processing dynamic content. This optimization is disabled if you call {@link #getInputBuffers 530 getInput}/{@link #getOutputBuffers OutputBuffers()}. 531 532 <p class=note> 533 <strong>Note:</strong> do not mix the methods of using buffers and buffer arrays at the same 534 time. Specifically, only call {@code getInput}/{@code OutputBuffers} directly after {@link 535 #start} or after having dequeued an output buffer ID with the value of {@link 536 #INFO_OUTPUT_FORMAT_CHANGED}. 537 <p> 538 MediaCodec is typically used like this in synchronous mode: 539 <pre> 540 MediaCodec codec = MediaCodec.createByCodecName(name); 541 codec.configure(format, …); 542 MediaFormat outputFormat = codec.getOutputFormat(); // option B 543 codec.start(); 544 for (;;) { 545 int inputBufferId = codec.dequeueInputBuffer(timeoutUs); 546 if (inputBufferId >= 0) { 547 ByteBuffer inputBuffer = codec.getInputBuffer(…); 548 // fill inputBuffer with valid data 549 … 550 codec.queueInputBuffer(inputBufferId, …); 551 } 552 int outputBufferId = codec.dequeueOutputBuffer(…); 553 if (outputBufferId >= 0) { 554 ByteBuffer outputBuffer = codec.getOutputBuffer(outputBufferId); 555 MediaFormat bufferFormat = codec.getOutputFormat(outputBufferId); // option A 556 // bufferFormat is identical to outputFormat 557 // outputBuffer is ready to be processed or rendered. 558 … 559 codec.releaseOutputBuffer(outputBufferId, …); 560 } else if (outputBufferId == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) { 561 // Subsequent data will conform to new format. 562 // Can ignore if using getOutputFormat(outputBufferId) 563 outputFormat = codec.getOutputFormat(); // option B 564 } 565 } 566 codec.stop(); 567 codec.release();</pre> 568 569 <h4>Synchronous Processing using Buffer Arrays (deprecated)</h4> 570 <p> 571 In versions {@link android.os.Build.VERSION_CODES#KITKAT_WATCH} and before, the set of input and 572 output buffers are represented by the {@code ByteBuffer[]} arrays. After a successful call to 573 {@link #start}, retrieve the buffer arrays using {@link #getInputBuffers getInput}/{@link 574 #getOutputBuffers OutputBuffers()}. Use the buffer ID-s as indices into these arrays (when 575 non-negative), as demonstrated in the sample below. Note that there is no inherent correlation 576 between the size of the arrays and the number of input and output buffers used by the system, 577 although the array size provides an upper bound. 578 <pre> 579 MediaCodec codec = MediaCodec.createByCodecName(name); 580 codec.configure(format, …); 581 codec.start(); 582 ByteBuffer[] inputBuffers = codec.getInputBuffers(); 583 ByteBuffer[] outputBuffers = codec.getOutputBuffers(); 584 for (;;) { 585 int inputBufferId = codec.dequeueInputBuffer(…); 586 if (inputBufferId >= 0) { 587 // fill inputBuffers[inputBufferId] with valid data 588 … 589 codec.queueInputBuffer(inputBufferId, …); 590 } 591 int outputBufferId = codec.dequeueOutputBuffer(…); 592 if (outputBufferId >= 0) { 593 // outputBuffers[outputBufferId] is ready to be processed or rendered. 594 … 595 codec.releaseOutputBuffer(outputBufferId, …); 596 } else if (outputBufferId == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) { 597 outputBuffers = codec.getOutputBuffers(); 598 } else if (outputBufferId == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) { 599 // Subsequent data will conform to new format. 600 MediaFormat format = codec.getOutputFormat(); 601 } 602 } 603 codec.stop(); 604 codec.release();</pre> 605 606 <h4>End-of-stream Handling</h4> 607 <p> 608 When you reach the end of the input data, you must signal it to the codec by specifying the 609 {@link #BUFFER_FLAG_END_OF_STREAM} flag in the call to {@link #queueInputBuffer 610 queueInputBuffer}. You can do this on the last valid input buffer, or by submitting an additional 611 empty input buffer with the end-of-stream flag set. If using an empty buffer, the timestamp will 612 be ignored. 613 <p> 614 The codec will continue to return output buffers until it eventually signals the end of the 615 output stream by specifying the same end-of-stream flag in the {@link BufferInfo} set in {@link 616 #dequeueOutputBuffer dequeueOutputBuffer} or returned via {@link Callback#onOutputBufferAvailable 617 onOutputBufferAvailable}. This can be set on the last valid output buffer, or on an empty buffer 618 after the last valid output buffer. The timestamp of such empty buffer should be ignored. 619 <p> 620 Do not submit additional input buffers after signaling the end of the input stream, unless the 621 codec has been flushed, or stopped and restarted. 622 623 <h4>Using an Output Surface</h4> 624 <p> 625 The data processing is nearly identical to the ByteBuffer mode when using an output {@link 626 Surface}; however, the output buffers will not be accessible, and are represented as {@code null} 627 values. E.g. {@link #getOutputBuffer getOutputBuffer}/{@link #getOutputImage Image(int)} will 628 return {@code null} and {@link #getOutputBuffers} will return an array containing only {@code 629 null}-s. 630 <p> 631 When using an output Surface, you can select whether or not to render each output buffer on the 632 surface. You have three choices: 633 <ul> 634 <li><strong>Do not render the buffer:</strong> Call {@link #releaseOutputBuffer(int, boolean) 635 releaseOutputBuffer(bufferId, false)}.</li> 636 <li><strong>Render the buffer with the default timestamp:</strong> Call {@link 637 #releaseOutputBuffer(int, boolean) releaseOutputBuffer(bufferId, true)}.</li> 638 <li><strong>Render the buffer with a specific timestamp:</strong> Call {@link 639 #releaseOutputBuffer(int, long) releaseOutputBuffer(bufferId, timestamp)}.</li> 640 </ul> 641 <p> 642 Since {@link android.os.Build.VERSION_CODES#M}, the default timestamp is the {@linkplain 643 BufferInfo#presentationTimeUs presentation timestamp} of the buffer (converted to nanoseconds). 644 It was not defined prior to that. 645 <p> 646 Also since {@link android.os.Build.VERSION_CODES#M}, you can change the output Surface 647 dynamically using {@link #setOutputSurface setOutputSurface}. 648 <p> 649 When rendering output to a Surface, the Surface may be configured to drop excessive frames (that 650 are not consumed by the Surface in a timely manner). Or it may be configured to not drop excessive 651 frames. In the latter mode if the Surface is not consuming output frames fast enough, it will 652 eventually block the decoder. Prior to {@link android.os.Build.VERSION_CODES#Q} the exact behavior 653 was undefined, with the exception that View surfaces (SuerfaceView or TextureView) always dropped 654 excessive frames. Since {@link android.os.Build.VERSION_CODES#Q} the default behavior is to drop 655 excessive frames. Applications can opt out of this behavior for non-View surfaces (such as 656 ImageReader or SurfaceTexture) by targeting SDK {@link android.os.Build.VERSION_CODES#Q} and 657 setting the key {@code "allow-frame-drop"} to {@code 0} in their configure format. 658 659 <h4>Transformations When Rendering onto Surface</h4> 660 661 If the codec is configured into Surface mode, any crop rectangle, {@linkplain 662 MediaFormat#KEY_ROTATION rotation} and {@linkplain #setVideoScalingMode video scaling 663 mode} will be automatically applied with one exception: 664 <p class=note> 665 Prior to the {@link android.os.Build.VERSION_CODES#M} release, software decoders may not 666 have applied the rotation when being rendered onto a Surface. Unfortunately, there is no standard 667 and simple way to identify software decoders, or if they apply the rotation other than by trying 668 it out. 669 <p> 670 There are also some caveats. 671 <p class=note> 672 Note that the pixel aspect ratio is not considered when displaying the output onto the 673 Surface. This means that if you are using {@link #VIDEO_SCALING_MODE_SCALE_TO_FIT} mode, you 674 must position the output Surface so that it has the proper final display aspect ratio. Conversely, 675 you can only use {@link #VIDEO_SCALING_MODE_SCALE_TO_FIT_WITH_CROPPING} mode for content with 676 square pixels (pixel aspect ratio or 1:1). 677 <p class=note> 678 Note also that as of {@link android.os.Build.VERSION_CODES#N} release, {@link 679 #VIDEO_SCALING_MODE_SCALE_TO_FIT_WITH_CROPPING} mode may not work correctly for videos rotated 680 by 90 or 270 degrees. 681 <p class=note> 682 When setting the video scaling mode, note that it must be reset after each time the output 683 buffers change. Since the {@link #INFO_OUTPUT_BUFFERS_CHANGED} event is deprecated, you can 684 do this after each time the output format changes. 685 686 <h4>Using an Input Surface</h4> 687 <p> 688 When using an input Surface, there are no accessible input buffers, as buffers are automatically 689 passed from the input surface to the codec. Calling {@link #dequeueInputBuffer 690 dequeueInputBuffer} will throw an {@code IllegalStateException}, and {@link #getInputBuffers} 691 returns a bogus {@code ByteBuffer[]} array that <strong>MUST NOT</strong> be written into. 692 <p> 693 Call {@link #signalEndOfInputStream} to signal end-of-stream. The input surface will stop 694 submitting data to the codec immediately after this call. 695 <p> 696 697 <h3>Seeking & Adaptive Playback Support</h3> 698 <p> 699 Video decoders (and in general codecs that consume compressed video data) behave differently 700 regarding seek and format change whether or not they support and are configured for adaptive 701 playback. You can check if a decoder supports {@linkplain 702 CodecCapabilities#FEATURE_AdaptivePlayback adaptive playback} via {@link 703 CodecCapabilities#isFeatureSupported CodecCapabilities.isFeatureSupported(String)}. Adaptive 704 playback support for video decoders is only activated if you configure the codec to decode onto a 705 {@link Surface}. 706 707 <h4 id=KeyFrames><a name="KeyFrames"></a>Stream Boundary and Key Frames</h4> 708 <p> 709 It is important that the input data after {@link #start} or {@link #flush} starts at a suitable 710 stream boundary: the first frame must a key frame. A <em>key frame</em> can be decoded 711 completely on its own (for most codecs this means an I-frame), and no frames that are to be 712 displayed after a key frame refer to frames before the key frame. 713 <p> 714 The following table summarizes suitable key frames for various video formats. 715 <table> 716 <thead> 717 <tr> 718 <th>Format</th> 719 <th>Suitable key frame</th> 720 </tr> 721 </thead> 722 <tbody class=mid> 723 <tr> 724 <td>VP9/VP8</td> 725 <td>a suitable intraframe where no subsequent frames refer to frames prior to this frame.<br> 726 <i>(There is no specific name for such key frame.)</i></td> 727 </tr> 728 <tr> 729 <td>H.265 HEVC</td> 730 <td>IDR or CRA</td> 731 </tr> 732 <tr> 733 <td>H.264 AVC</td> 734 <td>IDR</td> 735 </tr> 736 <tr> 737 <td>MPEG-4<br>H.263<br>MPEG-2</td> 738 <td>a suitable I-frame where no subsequent frames refer to frames prior to this frame.<br> 739 <i>(There is no specific name for such key frame.)</td> 740 </tr> 741 </tbody> 742 </table> 743 744 <h4>For decoders that do not support adaptive playback (including when not decoding onto a 745 Surface)</h4> 746 <p> 747 In order to start decoding data that is not adjacent to previously submitted data (i.e. after a 748 seek) you <strong>MUST</strong> flush the decoder. Since all output buffers are immediately 749 revoked at the point of the flush, you may want to first signal then wait for the end-of-stream 750 before you call {@code flush}. It is important that the input data after a flush starts at a 751 suitable stream boundary/key frame. 752 <p class=note> 753 <strong>Note:</strong> the format of the data submitted after a flush must not change; {@link 754 #flush} does not support format discontinuities; for that, a full {@link #stop} - {@link 755 #configure configure(…)} - {@link #start} cycle is necessary. 756 757 <p class=note> 758 <strong>Also note:</strong> if you flush the codec too soon after {@link #start} – 759 generally, before the first output buffer or output format change is received – you 760 will need to resubmit the codec-specific-data to the codec. See the <a 761 href="#CSD">codec-specific-data section</a> for more info. 762 763 <h4>For decoders that support and are configured for adaptive playback</h4> 764 <p> 765 In order to start decoding data that is not adjacent to previously submitted data (i.e. after a 766 seek) it is <em>not necessary</em> to flush the decoder; however, input data after the 767 discontinuity must start at a suitable stream boundary/key frame. 768 <p> 769 For some video formats - namely H.264, H.265, VP8 and VP9 - it is also possible to change the 770 picture size or configuration mid-stream. To do this you must package the entire new 771 codec-specific configuration data together with the key frame into a single buffer (including 772 any start codes), and submit it as a <strong>regular</strong> input buffer. 773 <p> 774 You will receive an {@link #INFO_OUTPUT_FORMAT_CHANGED} return value from {@link 775 #dequeueOutputBuffer dequeueOutputBuffer} or a {@link Callback#onOutputBufferAvailable 776 onOutputFormatChanged} callback just after the picture-size change takes place and before any 777 frames with the new size have been returned. 778 <p class=note> 779 <strong>Note:</strong> just as the case for codec-specific data, be careful when calling 780 {@link #flush} shortly after you have changed the picture size. If you have not received 781 confirmation of the picture size change, you will need to repeat the request for the new picture 782 size. 783 784 <h3>Error handling</h3> 785 <p> 786 The factory methods {@link #createByCodecName createByCodecName} and {@link #createDecoderByType 787 createDecoder}/{@link #createEncoderByType EncoderByType} throw {@code IOException} on failure 788 which you must catch or declare to pass up. MediaCodec methods throw {@code 789 IllegalStateException} when the method is called from a codec state that does not allow it; this 790 is typically due to incorrect application API usage. Methods involving secure buffers may throw 791 {@link CryptoException}, which has further error information obtainable from {@link 792 CryptoException#getErrorCode}. 793 <p> 794 Internal codec errors result in a {@link CodecException}, which may be due to media content 795 corruption, hardware failure, resource exhaustion, and so forth, even when the application is 796 correctly using the API. The recommended action when receiving a {@code CodecException} 797 can be determined by calling {@link CodecException#isRecoverable} and {@link 798 CodecException#isTransient}: 799 <ul> 800 <li><strong>recoverable errors:</strong> If {@code isRecoverable()} returns true, then call 801 {@link #stop}, {@link #configure configure(…)}, and {@link #start} to recover.</li> 802 <li><strong>transient errors:</strong> If {@code isTransient()} returns true, then resources are 803 temporarily unavailable and the method may be retried at a later time.</li> 804 <li><strong>fatal errors:</strong> If both {@code isRecoverable()} and {@code isTransient()} 805 return false, then the {@code CodecException} is fatal and the codec must be {@linkplain #reset 806 reset} or {@linkplain #release released}.</li> 807 </ul> 808 <p> 809 Both {@code isRecoverable()} and {@code isTransient()} do not return true at the same time. 810 811 <h2 id=History><a name="History"></a>Valid API Calls and API History</h2> 812 <p> 813 This sections summarizes the valid API calls in each state and the API history of the MediaCodec 814 class. For API version numbers, see {@link android.os.Build.VERSION_CODES}. 815 816 <style> 817 .api > tr > th, .api > tr > td { text-align: center; padding: 4px 4px; } 818 .api > tr > th { vertical-align: bottom; } 819 .api > tr > td { vertical-align: middle; } 820 .sml > tr > th, .sml > tr > td { text-align: center; padding: 2px 4px; } 821 .fn { text-align: left; } 822 .fn > code > a { font: 14px/19px Roboto Condensed, sans-serif; } 823 .deg45 { 824 white-space: nowrap; background: none; border: none; vertical-align: bottom; 825 width: 30px; height: 83px; 826 } 827 .deg45 > div { 828 transform: skew(-45deg, 0deg) translate(1px, -67px); 829 transform-origin: bottom left 0; 830 width: 30px; height: 20px; 831 } 832 .deg45 > div > div { border: 1px solid #ddd; background: #999; height: 90px; width: 42px; } 833 .deg45 > div > div > div { transform: skew(45deg, 0deg) translate(-55px, 55px) rotate(-45deg); } 834 </style> 835 836 <table align="right" style="width: 0%"> 837 <thead> 838 <tr><th>Symbol</th><th>Meaning</th></tr> 839 </thead> 840 <tbody class=sml> 841 <tr><td>●</td><td>Supported</td></tr> 842 <tr><td>⁕</td><td>Semantics changed</td></tr> 843 <tr><td>○</td><td>Experimental support</td></tr> 844 <tr><td>[ ]</td><td>Deprecated</td></tr> 845 <tr><td>⎋</td><td>Restricted to surface input mode</td></tr> 846 <tr><td>⎆</td><td>Restricted to surface output mode</td></tr> 847 <tr><td>▧</td><td>Restricted to ByteBuffer input mode</td></tr> 848 <tr><td>↩</td><td>Restricted to synchronous mode</td></tr> 849 <tr><td>⇄</td><td>Restricted to asynchronous mode</td></tr> 850 <tr><td>( )</td><td>Can be called, but shouldn't</td></tr> 851 </tbody> 852 </table> 853 854 <table style="width: 100%;"> 855 <thead class=api> 856 <tr> 857 <th class=deg45><div><div style="background:#4285f4"><div>Uninitialized</div></div></div></th> 858 <th class=deg45><div><div style="background:#f4b400"><div>Configured</div></div></div></th> 859 <th class=deg45><div><div style="background:#e67c73"><div>Flushed</div></div></div></th> 860 <th class=deg45><div><div style="background:#0f9d58"><div>Running</div></div></div></th> 861 <th class=deg45><div><div style="background:#f7cb4d"><div>End of Stream</div></div></div></th> 862 <th class=deg45><div><div style="background:#db4437"><div>Error</div></div></div></th> 863 <th class=deg45><div><div style="background:#666"><div>Released</div></div></div></th> 864 <th></th> 865 <th colspan="8">SDK Version</th> 866 </tr> 867 <tr> 868 <th colspan="7">State</th> 869 <th>Method</th> 870 <th>16</th> 871 <th>17</th> 872 <th>18</th> 873 <th>19</th> 874 <th>20</th> 875 <th>21</th> 876 <th>22</th> 877 <th>23</th> 878 </tr> 879 </thead> 880 <tbody class=api> 881 <tr> 882 <td></td> 883 <td></td> 884 <td></td> 885 <td></td> 886 <td></td> 887 <td></td> 888 <td></td> 889 <td class=fn>{@link #createByCodecName createByCodecName}</td> 890 <td>●</td> 891 <td>●</td> 892 <td>●</td> 893 <td>●</td> 894 <td>●</td> 895 <td>●</td> 896 <td>●</td> 897 <td>●</td> 898 </tr> 899 <tr> 900 <td></td> 901 <td></td> 902 <td></td> 903 <td></td> 904 <td></td> 905 <td></td> 906 <td></td> 907 <td class=fn>{@link #createDecoderByType createDecoderByType}</td> 908 <td>●</td> 909 <td>●</td> 910 <td>●</td> 911 <td>●</td> 912 <td>●</td> 913 <td>●</td> 914 <td>●</td> 915 <td>●</td> 916 </tr> 917 <tr> 918 <td></td> 919 <td></td> 920 <td></td> 921 <td></td> 922 <td></td> 923 <td></td> 924 <td></td> 925 <td class=fn>{@link #createEncoderByType createEncoderByType}</td> 926 <td>●</td> 927 <td>●</td> 928 <td>●</td> 929 <td>●</td> 930 <td>●</td> 931 <td>●</td> 932 <td>●</td> 933 <td>●</td> 934 </tr> 935 <tr> 936 <td></td> 937 <td></td> 938 <td></td> 939 <td></td> 940 <td></td> 941 <td></td> 942 <td></td> 943 <td class=fn>{@link #createPersistentInputSurface createPersistentInputSurface}</td> 944 <td></td> 945 <td></td> 946 <td></td> 947 <td></td> 948 <td></td> 949 <td></td> 950 <td></td> 951 <td>●</td> 952 </tr> 953 <tr> 954 <td>16+</td> 955 <td>-</td> 956 <td>-</td> 957 <td>-</td> 958 <td>-</td> 959 <td>-</td> 960 <td>-</td> 961 <td class=fn>{@link #configure configure}</td> 962 <td>●</td> 963 <td>●</td> 964 <td>●</td> 965 <td>●</td> 966 <td>●</td> 967 <td>⁕</td> 968 <td>●</td> 969 <td>●</td> 970 </tr> 971 <tr> 972 <td>-</td> 973 <td>18+</td> 974 <td>-</td> 975 <td>-</td> 976 <td>-</td> 977 <td>-</td> 978 <td>-</td> 979 <td class=fn>{@link #createInputSurface createInputSurface}</td> 980 <td></td> 981 <td></td> 982 <td>⎋</td> 983 <td>⎋</td> 984 <td>⎋</td> 985 <td>⎋</td> 986 <td>⎋</td> 987 <td>⎋</td> 988 </tr> 989 <tr> 990 <td>-</td> 991 <td>-</td> 992 <td>16+</td> 993 <td>16+</td> 994 <td>(16+)</td> 995 <td>-</td> 996 <td>-</td> 997 <td class=fn>{@link #dequeueInputBuffer dequeueInputBuffer}</td> 998 <td>●</td> 999 <td>●</td> 1000 <td>▧</td> 1001 <td>▧</td> 1002 <td>▧</td> 1003 <td>⁕▧↩</td> 1004 <td>▧↩</td> 1005 <td>▧↩</td> 1006 </tr> 1007 <tr> 1008 <td>-</td> 1009 <td>-</td> 1010 <td>16+</td> 1011 <td>16+</td> 1012 <td>16+</td> 1013 <td>-</td> 1014 <td>-</td> 1015 <td class=fn>{@link #dequeueOutputBuffer dequeueOutputBuffer}</td> 1016 <td>●</td> 1017 <td>●</td> 1018 <td>●</td> 1019 <td>●</td> 1020 <td>●</td> 1021 <td>⁕↩</td> 1022 <td>↩</td> 1023 <td>↩</td> 1024 </tr> 1025 <tr> 1026 <td>-</td> 1027 <td>-</td> 1028 <td>16+</td> 1029 <td>16+</td> 1030 <td>16+</td> 1031 <td>-</td> 1032 <td>-</td> 1033 <td class=fn>{@link #flush flush}</td> 1034 <td>●</td> 1035 <td>●</td> 1036 <td>●</td> 1037 <td>●</td> 1038 <td>●</td> 1039 <td>●</td> 1040 <td>●</td> 1041 <td>●</td> 1042 </tr> 1043 <tr> 1044 <td>18+</td> 1045 <td>18+</td> 1046 <td>18+</td> 1047 <td>18+</td> 1048 <td>18+</td> 1049 <td>18+</td> 1050 <td>-</td> 1051 <td class=fn>{@link #getCodecInfo getCodecInfo}</td> 1052 <td></td> 1053 <td></td> 1054 <td>●</td> 1055 <td>●</td> 1056 <td>●</td> 1057 <td>●</td> 1058 <td>●</td> 1059 <td>●</td> 1060 </tr> 1061 <tr> 1062 <td>-</td> 1063 <td>-</td> 1064 <td>(21+)</td> 1065 <td>21+</td> 1066 <td>(21+)</td> 1067 <td>-</td> 1068 <td>-</td> 1069 <td class=fn>{@link #getInputBuffer getInputBuffer}</td> 1070 <td></td> 1071 <td></td> 1072 <td></td> 1073 <td></td> 1074 <td></td> 1075 <td>●</td> 1076 <td>●</td> 1077 <td>●</td> 1078 </tr> 1079 <tr> 1080 <td>-</td> 1081 <td>-</td> 1082 <td>16+</td> 1083 <td>(16+)</td> 1084 <td>(16+)</td> 1085 <td>-</td> 1086 <td>-</td> 1087 <td class=fn>{@link #getInputBuffers getInputBuffers}</td> 1088 <td>●</td> 1089 <td>●</td> 1090 <td>●</td> 1091 <td>●</td> 1092 <td>●</td> 1093 <td>[⁕↩]</td> 1094 <td>[↩]</td> 1095 <td>[↩]</td> 1096 </tr> 1097 <tr> 1098 <td>-</td> 1099 <td>21+</td> 1100 <td>(21+)</td> 1101 <td>(21+)</td> 1102 <td>(21+)</td> 1103 <td>-</td> 1104 <td>-</td> 1105 <td class=fn>{@link #getInputFormat getInputFormat}</td> 1106 <td></td> 1107 <td></td> 1108 <td></td> 1109 <td></td> 1110 <td></td> 1111 <td>●</td> 1112 <td>●</td> 1113 <td>●</td> 1114 </tr> 1115 <tr> 1116 <td>-</td> 1117 <td>-</td> 1118 <td>(21+)</td> 1119 <td>21+</td> 1120 <td>(21+)</td> 1121 <td>-</td> 1122 <td>-</td> 1123 <td class=fn>{@link #getInputImage getInputImage}</td> 1124 <td></td> 1125 <td></td> 1126 <td></td> 1127 <td></td> 1128 <td></td> 1129 <td>○</td> 1130 <td>●</td> 1131 <td>●</td> 1132 </tr> 1133 <tr> 1134 <td>18+</td> 1135 <td>18+</td> 1136 <td>18+</td> 1137 <td>18+</td> 1138 <td>18+</td> 1139 <td>18+</td> 1140 <td>-</td> 1141 <td class=fn>{@link #getName getName}</td> 1142 <td></td> 1143 <td></td> 1144 <td>●</td> 1145 <td>●</td> 1146 <td>●</td> 1147 <td>●</td> 1148 <td>●</td> 1149 <td>●</td> 1150 </tr> 1151 <tr> 1152 <td>-</td> 1153 <td>-</td> 1154 <td>(21+)</td> 1155 <td>21+</td> 1156 <td>21+</td> 1157 <td>-</td> 1158 <td>-</td> 1159 <td class=fn>{@link #getOutputBuffer getOutputBuffer}</td> 1160 <td></td> 1161 <td></td> 1162 <td></td> 1163 <td></td> 1164 <td></td> 1165 <td>●</td> 1166 <td>●</td> 1167 <td>●</td> 1168 </tr> 1169 <tr> 1170 <td>-</td> 1171 <td>-</td> 1172 <td>16+</td> 1173 <td>16+</td> 1174 <td>16+</td> 1175 <td>-</td> 1176 <td>-</td> 1177 <td class=fn>{@link #getOutputBuffers getOutputBuffers}</td> 1178 <td>●</td> 1179 <td>●</td> 1180 <td>●</td> 1181 <td>●</td> 1182 <td>●</td> 1183 <td>[⁕↩]</td> 1184 <td>[↩]</td> 1185 <td>[↩]</td> 1186 </tr> 1187 <tr> 1188 <td>-</td> 1189 <td>21+</td> 1190 <td>16+</td> 1191 <td>16+</td> 1192 <td>16+</td> 1193 <td>-</td> 1194 <td>-</td> 1195 <td class=fn>{@link #getOutputFormat()}</td> 1196 <td>●</td> 1197 <td>●</td> 1198 <td>●</td> 1199 <td>●</td> 1200 <td>●</td> 1201 <td>●</td> 1202 <td>●</td> 1203 <td>●</td> 1204 </tr> 1205 <tr> 1206 <td>-</td> 1207 <td>-</td> 1208 <td>(21+)</td> 1209 <td>21+</td> 1210 <td>21+</td> 1211 <td>-</td> 1212 <td>-</td> 1213 <td class=fn>{@link #getOutputFormat(int)}</td> 1214 <td></td> 1215 <td></td> 1216 <td></td> 1217 <td></td> 1218 <td></td> 1219 <td>●</td> 1220 <td>●</td> 1221 <td>●</td> 1222 </tr> 1223 <tr> 1224 <td>-</td> 1225 <td>-</td> 1226 <td>(21+)</td> 1227 <td>21+</td> 1228 <td>21+</td> 1229 <td>-</td> 1230 <td>-</td> 1231 <td class=fn>{@link #getOutputImage getOutputImage}</td> 1232 <td></td> 1233 <td></td> 1234 <td></td> 1235 <td></td> 1236 <td></td> 1237 <td>○</td> 1238 <td>●</td> 1239 <td>●</td> 1240 </tr> 1241 <tr> 1242 <td>-</td> 1243 <td>-</td> 1244 <td>-</td> 1245 <td>16+</td> 1246 <td>(16+)</td> 1247 <td>-</td> 1248 <td>-</td> 1249 <td class=fn>{@link #queueInputBuffer queueInputBuffer}</td> 1250 <td>●</td> 1251 <td>●</td> 1252 <td>●</td> 1253 <td>●</td> 1254 <td>●</td> 1255 <td>⁕</td> 1256 <td>●</td> 1257 <td>●</td> 1258 </tr> 1259 <tr> 1260 <td>-</td> 1261 <td>-</td> 1262 <td>-</td> 1263 <td>16+</td> 1264 <td>(16+)</td> 1265 <td>-</td> 1266 <td>-</td> 1267 <td class=fn>{@link #queueSecureInputBuffer queueSecureInputBuffer}</td> 1268 <td>●</td> 1269 <td>●</td> 1270 <td>●</td> 1271 <td>●</td> 1272 <td>●</td> 1273 <td>⁕</td> 1274 <td>●</td> 1275 <td>●</td> 1276 </tr> 1277 <tr> 1278 <td>16+</td> 1279 <td>16+</td> 1280 <td>16+</td> 1281 <td>16+</td> 1282 <td>16+</td> 1283 <td>16+</td> 1284 <td>16+</td> 1285 <td class=fn>{@link #release release}</td> 1286 <td>●</td> 1287 <td>●</td> 1288 <td>●</td> 1289 <td>●</td> 1290 <td>●</td> 1291 <td>●</td> 1292 <td>●</td> 1293 <td>●</td> 1294 </tr> 1295 <tr> 1296 <td>-</td> 1297 <td>-</td> 1298 <td>-</td> 1299 <td>16+</td> 1300 <td>16+</td> 1301 <td>-</td> 1302 <td>-</td> 1303 <td class=fn>{@link #releaseOutputBuffer(int, boolean)}</td> 1304 <td>●</td> 1305 <td>●</td> 1306 <td>●</td> 1307 <td>●</td> 1308 <td>●</td> 1309 <td>⁕</td> 1310 <td>●</td> 1311 <td>⁕</td> 1312 </tr> 1313 <tr> 1314 <td>-</td> 1315 <td>-</td> 1316 <td>-</td> 1317 <td>21+</td> 1318 <td>21+</td> 1319 <td>-</td> 1320 <td>-</td> 1321 <td class=fn>{@link #releaseOutputBuffer(int, long)}</td> 1322 <td></td> 1323 <td></td> 1324 <td></td> 1325 <td></td> 1326 <td></td> 1327 <td>⎆</td> 1328 <td>⎆</td> 1329 <td>⎆</td> 1330 </tr> 1331 <tr> 1332 <td>21+</td> 1333 <td>21+</td> 1334 <td>21+</td> 1335 <td>21+</td> 1336 <td>21+</td> 1337 <td>21+</td> 1338 <td>-</td> 1339 <td class=fn>{@link #reset reset}</td> 1340 <td></td> 1341 <td></td> 1342 <td></td> 1343 <td></td> 1344 <td></td> 1345 <td>●</td> 1346 <td>●</td> 1347 <td>●</td> 1348 </tr> 1349 <tr> 1350 <td>21+</td> 1351 <td>-</td> 1352 <td>-</td> 1353 <td>-</td> 1354 <td>-</td> 1355 <td>-</td> 1356 <td>-</td> 1357 <td class=fn>{@link #setCallback(Callback) setCallback}</td> 1358 <td></td> 1359 <td></td> 1360 <td></td> 1361 <td></td> 1362 <td></td> 1363 <td>●</td> 1364 <td>●</td> 1365 <td>{@link #setCallback(Callback, Handler) ⁕}</td> 1366 </tr> 1367 <tr> 1368 <td>-</td> 1369 <td>23+</td> 1370 <td>-</td> 1371 <td>-</td> 1372 <td>-</td> 1373 <td>-</td> 1374 <td>-</td> 1375 <td class=fn>{@link #setInputSurface setInputSurface}</td> 1376 <td></td> 1377 <td></td> 1378 <td></td> 1379 <td></td> 1380 <td></td> 1381 <td></td> 1382 <td></td> 1383 <td>⎋</td> 1384 </tr> 1385 <tr> 1386 <td>23+</td> 1387 <td>23+</td> 1388 <td>23+</td> 1389 <td>23+</td> 1390 <td>23+</td> 1391 <td>(23+)</td> 1392 <td>(23+)</td> 1393 <td class=fn>{@link #setOnFrameRenderedListener setOnFrameRenderedListener}</td> 1394 <td></td> 1395 <td></td> 1396 <td></td> 1397 <td></td> 1398 <td></td> 1399 <td></td> 1400 <td></td> 1401 <td>○ ⎆</td> 1402 </tr> 1403 <tr> 1404 <td>-</td> 1405 <td>23+</td> 1406 <td>23+</td> 1407 <td>23+</td> 1408 <td>23+</td> 1409 <td>-</td> 1410 <td>-</td> 1411 <td class=fn>{@link #setOutputSurface setOutputSurface}</td> 1412 <td></td> 1413 <td></td> 1414 <td></td> 1415 <td></td> 1416 <td></td> 1417 <td></td> 1418 <td></td> 1419 <td>⎆</td> 1420 </tr> 1421 <tr> 1422 <td>19+</td> 1423 <td>19+</td> 1424 <td>19+</td> 1425 <td>19+</td> 1426 <td>19+</td> 1427 <td>(19+)</td> 1428 <td>-</td> 1429 <td class=fn>{@link #setParameters setParameters}</td> 1430 <td></td> 1431 <td></td> 1432 <td></td> 1433 <td>●</td> 1434 <td>●</td> 1435 <td>●</td> 1436 <td>●</td> 1437 <td>●</td> 1438 </tr> 1439 <tr> 1440 <td>-</td> 1441 <td>(16+)</td> 1442 <td>(16+)</td> 1443 <td>16+</td> 1444 <td>(16+)</td> 1445 <td>(16+)</td> 1446 <td>-</td> 1447 <td class=fn>{@link #setVideoScalingMode setVideoScalingMode}</td> 1448 <td>⎆</td> 1449 <td>⎆</td> 1450 <td>⎆</td> 1451 <td>⎆</td> 1452 <td>⎆</td> 1453 <td>⎆</td> 1454 <td>⎆</td> 1455 <td>⎆</td> 1456 </tr> 1457 <tr> 1458 <td>(29+)</td> 1459 <td>29+</td> 1460 <td>29+</td> 1461 <td>29+</td> 1462 <td>(29+)</td> 1463 <td>(29+)</td> 1464 <td>-</td> 1465 <td class=fn>{@link #setAudioPresentation setAudioPresentation}</td> 1466 <td></td> 1467 <td></td> 1468 <td></td> 1469 <td></td> 1470 <td></td> 1471 <td></td> 1472 <td></td> 1473 <td></td> 1474 </tr> 1475 <tr> 1476 <td>-</td> 1477 <td>-</td> 1478 <td>18+</td> 1479 <td>18+</td> 1480 <td>-</td> 1481 <td>-</td> 1482 <td>-</td> 1483 <td class=fn>{@link #signalEndOfInputStream signalEndOfInputStream}</td> 1484 <td></td> 1485 <td></td> 1486 <td>⎋</td> 1487 <td>⎋</td> 1488 <td>⎋</td> 1489 <td>⎋</td> 1490 <td>⎋</td> 1491 <td>⎋</td> 1492 </tr> 1493 <tr> 1494 <td>-</td> 1495 <td>16+</td> 1496 <td>21+(⇄)</td> 1497 <td>-</td> 1498 <td>-</td> 1499 <td>-</td> 1500 <td>-</td> 1501 <td class=fn>{@link #start start}</td> 1502 <td>●</td> 1503 <td>●</td> 1504 <td>●</td> 1505 <td>●</td> 1506 <td>●</td> 1507 <td>⁕</td> 1508 <td>●</td> 1509 <td>●</td> 1510 </tr> 1511 <tr> 1512 <td>-</td> 1513 <td>-</td> 1514 <td>16+</td> 1515 <td>16+</td> 1516 <td>16+</td> 1517 <td>-</td> 1518 <td>-</td> 1519 <td class=fn>{@link #stop stop}</td> 1520 <td>●</td> 1521 <td>●</td> 1522 <td>●</td> 1523 <td>●</td> 1524 <td>●</td> 1525 <td>●</td> 1526 <td>●</td> 1527 <td>●</td> 1528 </tr> 1529 </tbody> 1530 </table> 1531 */ 1532 final public class MediaCodec { 1533 /** 1534 * Per buffer metadata includes an offset and size specifying 1535 * the range of valid data in the associated codec (output) buffer. 1536 */ 1537 public final static class BufferInfo { 1538 /** 1539 * Update the buffer metadata information. 1540 * 1541 * @param newOffset the start-offset of the data in the buffer. 1542 * @param newSize the amount of data (in bytes) in the buffer. 1543 * @param newTimeUs the presentation timestamp in microseconds. 1544 * @param newFlags buffer flags associated with the buffer. This 1545 * should be a combination of {@link #BUFFER_FLAG_KEY_FRAME} and 1546 * {@link #BUFFER_FLAG_END_OF_STREAM}. 1547 */ set( int newOffset, int newSize, long newTimeUs, @BufferFlag int newFlags)1548 public void set( 1549 int newOffset, int newSize, long newTimeUs, @BufferFlag int newFlags) { 1550 offset = newOffset; 1551 size = newSize; 1552 presentationTimeUs = newTimeUs; 1553 flags = newFlags; 1554 } 1555 1556 /** 1557 * The start-offset of the data in the buffer. 1558 */ 1559 public int offset; 1560 1561 /** 1562 * The amount of data (in bytes) in the buffer. If this is {@code 0}, 1563 * the buffer has no data in it and can be discarded. The only 1564 * use of a 0-size buffer is to carry the end-of-stream marker. 1565 */ 1566 public int size; 1567 1568 /** 1569 * The presentation timestamp in microseconds for the buffer. 1570 * This is derived from the presentation timestamp passed in 1571 * with the corresponding input buffer. This should be ignored for 1572 * a 0-sized buffer. 1573 */ 1574 public long presentationTimeUs; 1575 1576 /** 1577 * Buffer flags associated with the buffer. A combination of 1578 * {@link #BUFFER_FLAG_KEY_FRAME} and {@link #BUFFER_FLAG_END_OF_STREAM}. 1579 * 1580 * <p>Encoded buffers that are key frames are marked with 1581 * {@link #BUFFER_FLAG_KEY_FRAME}. 1582 * 1583 * <p>The last output buffer corresponding to the input buffer 1584 * marked with {@link #BUFFER_FLAG_END_OF_STREAM} will also be marked 1585 * with {@link #BUFFER_FLAG_END_OF_STREAM}. In some cases this could 1586 * be an empty buffer, whose sole purpose is to carry the end-of-stream 1587 * marker. 1588 */ 1589 @BufferFlag 1590 public int flags; 1591 1592 /** @hide */ 1593 @NonNull dup()1594 public BufferInfo dup() { 1595 BufferInfo copy = new BufferInfo(); 1596 copy.set(offset, size, presentationTimeUs, flags); 1597 return copy; 1598 } 1599 }; 1600 1601 // The follow flag constants MUST stay in sync with their equivalents 1602 // in MediaCodec.h ! 1603 1604 /** 1605 * This indicates that the (encoded) buffer marked as such contains 1606 * the data for a key frame. 1607 * 1608 * @deprecated Use {@link #BUFFER_FLAG_KEY_FRAME} instead. 1609 */ 1610 public static final int BUFFER_FLAG_SYNC_FRAME = 1; 1611 1612 /** 1613 * This indicates that the (encoded) buffer marked as such contains 1614 * the data for a key frame. 1615 */ 1616 public static final int BUFFER_FLAG_KEY_FRAME = 1; 1617 1618 /** 1619 * This indicated that the buffer marked as such contains codec 1620 * initialization / codec specific data instead of media data. 1621 */ 1622 public static final int BUFFER_FLAG_CODEC_CONFIG = 2; 1623 1624 /** 1625 * This signals the end of stream, i.e. no buffers will be available 1626 * after this, unless of course, {@link #flush} follows. 1627 */ 1628 public static final int BUFFER_FLAG_END_OF_STREAM = 4; 1629 1630 /** 1631 * This indicates that the buffer only contains part of a frame, 1632 * and the decoder should batch the data until a buffer without 1633 * this flag appears before decoding the frame. 1634 */ 1635 public static final int BUFFER_FLAG_PARTIAL_FRAME = 8; 1636 1637 /** 1638 * This indicates that the buffer contains non-media data for the 1639 * muxer to process. 1640 * 1641 * All muxer data should start with a FOURCC header that determines the type of data. 1642 * 1643 * For example, when it contains Exif data sent to a MediaMuxer track of 1644 * {@link MediaFormat#MIMETYPE_IMAGE_ANDROID_HEIC} type, the data must start with 1645 * Exif header ("Exif\0\0"), followed by the TIFF header (See JEITA CP-3451C Section 4.5.2.) 1646 * 1647 * @hide 1648 */ 1649 public static final int BUFFER_FLAG_MUXER_DATA = 16; 1650 1651 /** @hide */ 1652 @IntDef( 1653 flag = true, 1654 value = { 1655 BUFFER_FLAG_SYNC_FRAME, 1656 BUFFER_FLAG_KEY_FRAME, 1657 BUFFER_FLAG_CODEC_CONFIG, 1658 BUFFER_FLAG_END_OF_STREAM, 1659 BUFFER_FLAG_PARTIAL_FRAME, 1660 BUFFER_FLAG_MUXER_DATA, 1661 }) 1662 @Retention(RetentionPolicy.SOURCE) 1663 public @interface BufferFlag {} 1664 1665 private EventHandler mEventHandler; 1666 private EventHandler mOnFrameRenderedHandler; 1667 private EventHandler mCallbackHandler; 1668 private Callback mCallback; 1669 private OnFrameRenderedListener mOnFrameRenderedListener; 1670 private final Object mListenerLock = new Object(); 1671 private MediaCodecInfo mCodecInfo; 1672 private final Object mCodecInfoLock = new Object(); 1673 private MediaCrypto mCrypto; 1674 1675 private static final int EVENT_CALLBACK = 1; 1676 private static final int EVENT_SET_CALLBACK = 2; 1677 private static final int EVENT_FRAME_RENDERED = 3; 1678 1679 private static final int CB_INPUT_AVAILABLE = 1; 1680 private static final int CB_OUTPUT_AVAILABLE = 2; 1681 private static final int CB_ERROR = 3; 1682 private static final int CB_OUTPUT_FORMAT_CHANGE = 4; 1683 1684 private class EventHandler extends Handler { 1685 private MediaCodec mCodec; 1686 EventHandler(@onNull MediaCodec codec, @NonNull Looper looper)1687 public EventHandler(@NonNull MediaCodec codec, @NonNull Looper looper) { 1688 super(looper); 1689 mCodec = codec; 1690 } 1691 1692 @Override handleMessage(@onNull Message msg)1693 public void handleMessage(@NonNull Message msg) { 1694 switch (msg.what) { 1695 case EVENT_CALLBACK: 1696 { 1697 handleCallback(msg); 1698 break; 1699 } 1700 case EVENT_SET_CALLBACK: 1701 { 1702 mCallback = (MediaCodec.Callback) msg.obj; 1703 break; 1704 } 1705 case EVENT_FRAME_RENDERED: 1706 Map<String, Object> map = (Map<String, Object>)msg.obj; 1707 for (int i = 0; ; ++i) { 1708 Object mediaTimeUs = map.get(i + "-media-time-us"); 1709 Object systemNano = map.get(i + "-system-nano"); 1710 OnFrameRenderedListener onFrameRenderedListener; 1711 synchronized (mListenerLock) { 1712 onFrameRenderedListener = mOnFrameRenderedListener; 1713 } 1714 if (mediaTimeUs == null || systemNano == null 1715 || onFrameRenderedListener == null) { 1716 break; 1717 } 1718 onFrameRenderedListener.onFrameRendered( 1719 mCodec, (long)mediaTimeUs, (long)systemNano); 1720 } 1721 break; 1722 default: 1723 { 1724 break; 1725 } 1726 } 1727 } 1728 handleCallback(@onNull Message msg)1729 private void handleCallback(@NonNull Message msg) { 1730 if (mCallback == null) { 1731 return; 1732 } 1733 1734 switch (msg.arg1) { 1735 case CB_INPUT_AVAILABLE: 1736 { 1737 int index = msg.arg2; 1738 synchronized(mBufferLock) { 1739 validateInputByteBuffer(mCachedInputBuffers, index); 1740 } 1741 mCallback.onInputBufferAvailable(mCodec, index); 1742 break; 1743 } 1744 1745 case CB_OUTPUT_AVAILABLE: 1746 { 1747 int index = msg.arg2; 1748 BufferInfo info = (MediaCodec.BufferInfo) msg.obj; 1749 synchronized(mBufferLock) { 1750 validateOutputByteBuffer(mCachedOutputBuffers, index, info); 1751 } 1752 mCallback.onOutputBufferAvailable( 1753 mCodec, index, info); 1754 break; 1755 } 1756 1757 case CB_ERROR: 1758 { 1759 mCallback.onError(mCodec, (MediaCodec.CodecException) msg.obj); 1760 break; 1761 } 1762 1763 case CB_OUTPUT_FORMAT_CHANGE: 1764 { 1765 mCallback.onOutputFormatChanged(mCodec, 1766 new MediaFormat((Map<String, Object>) msg.obj)); 1767 break; 1768 } 1769 1770 default: 1771 { 1772 break; 1773 } 1774 } 1775 } 1776 } 1777 1778 private boolean mHasSurface = false; 1779 1780 /** 1781 * Instantiate the preferred decoder supporting input data of the given mime type. 1782 * 1783 * The following is a partial list of defined mime types and their semantics: 1784 * <ul> 1785 * <li>"video/x-vnd.on2.vp8" - VP8 video (i.e. video in .webm) 1786 * <li>"video/x-vnd.on2.vp9" - VP9 video (i.e. video in .webm) 1787 * <li>"video/avc" - H.264/AVC video 1788 * <li>"video/hevc" - H.265/HEVC video 1789 * <li>"video/mp4v-es" - MPEG4 video 1790 * <li>"video/3gpp" - H.263 video 1791 * <li>"audio/3gpp" - AMR narrowband audio 1792 * <li>"audio/amr-wb" - AMR wideband audio 1793 * <li>"audio/mpeg" - MPEG1/2 audio layer III 1794 * <li>"audio/mp4a-latm" - AAC audio (note, this is raw AAC packets, not packaged in LATM!) 1795 * <li>"audio/vorbis" - vorbis audio 1796 * <li>"audio/g711-alaw" - G.711 alaw audio 1797 * <li>"audio/g711-mlaw" - G.711 ulaw audio 1798 * </ul> 1799 * 1800 * <strong>Note:</strong> It is preferred to use {@link MediaCodecList#findDecoderForFormat} 1801 * and {@link #createByCodecName} to ensure that the resulting codec can handle a 1802 * given format. 1803 * 1804 * @param type The mime type of the input data. 1805 * @throws IOException if the codec cannot be created. 1806 * @throws IllegalArgumentException if type is not a valid mime type. 1807 * @throws NullPointerException if type is null. 1808 */ 1809 @NonNull createDecoderByType(@onNull String type)1810 public static MediaCodec createDecoderByType(@NonNull String type) 1811 throws IOException { 1812 return new MediaCodec(type, true /* nameIsType */, false /* encoder */); 1813 } 1814 1815 /** 1816 * Instantiate the preferred encoder supporting output data of the given mime type. 1817 * 1818 * <strong>Note:</strong> It is preferred to use {@link MediaCodecList#findEncoderForFormat} 1819 * and {@link #createByCodecName} to ensure that the resulting codec can handle a 1820 * given format. 1821 * 1822 * @param type The desired mime type of the output data. 1823 * @throws IOException if the codec cannot be created. 1824 * @throws IllegalArgumentException if type is not a valid mime type. 1825 * @throws NullPointerException if type is null. 1826 */ 1827 @NonNull createEncoderByType(@onNull String type)1828 public static MediaCodec createEncoderByType(@NonNull String type) 1829 throws IOException { 1830 return new MediaCodec(type, true /* nameIsType */, true /* encoder */); 1831 } 1832 1833 /** 1834 * If you know the exact name of the component you want to instantiate 1835 * use this method to instantiate it. Use with caution. 1836 * Likely to be used with information obtained from {@link android.media.MediaCodecList} 1837 * @param name The name of the codec to be instantiated. 1838 * @throws IOException if the codec cannot be created. 1839 * @throws IllegalArgumentException if name is not valid. 1840 * @throws NullPointerException if name is null. 1841 */ 1842 @NonNull createByCodecName(@onNull String name)1843 public static MediaCodec createByCodecName(@NonNull String name) 1844 throws IOException { 1845 return new MediaCodec( 1846 name, false /* nameIsType */, false /* unused */); 1847 } 1848 MediaCodec( @onNull String name, boolean nameIsType, boolean encoder)1849 private MediaCodec( 1850 @NonNull String name, boolean nameIsType, boolean encoder) { 1851 Looper looper; 1852 if ((looper = Looper.myLooper()) != null) { 1853 mEventHandler = new EventHandler(this, looper); 1854 } else if ((looper = Looper.getMainLooper()) != null) { 1855 mEventHandler = new EventHandler(this, looper); 1856 } else { 1857 mEventHandler = null; 1858 } 1859 mCallbackHandler = mEventHandler; 1860 mOnFrameRenderedHandler = mEventHandler; 1861 1862 mBufferLock = new Object(); 1863 1864 // save name used at creation 1865 mNameAtCreation = nameIsType ? null : name; 1866 1867 native_setup(name, nameIsType, encoder); 1868 } 1869 1870 private String mNameAtCreation; 1871 1872 @Override finalize()1873 protected void finalize() { 1874 native_finalize(); 1875 mCrypto = null; 1876 } 1877 1878 /** 1879 * Returns the codec to its initial (Uninitialized) state. 1880 * 1881 * Call this if an {@link MediaCodec.CodecException#isRecoverable unrecoverable} 1882 * error has occured to reset the codec to its initial state after creation. 1883 * 1884 * @throws CodecException if an unrecoverable error has occured and the codec 1885 * could not be reset. 1886 * @throws IllegalStateException if in the Released state. 1887 */ reset()1888 public final void reset() { 1889 freeAllTrackedBuffers(); // free buffers first 1890 native_reset(); 1891 mCrypto = null; 1892 } 1893 native_reset()1894 private native final void native_reset(); 1895 1896 /** 1897 * Free up resources used by the codec instance. 1898 * 1899 * Make sure you call this when you're done to free up any opened 1900 * component instance instead of relying on the garbage collector 1901 * to do this for you at some point in the future. 1902 */ release()1903 public final void release() { 1904 freeAllTrackedBuffers(); // free buffers first 1905 native_release(); 1906 mCrypto = null; 1907 } 1908 native_release()1909 private native final void native_release(); 1910 1911 /** 1912 * If this codec is to be used as an encoder, pass this flag. 1913 */ 1914 public static final int CONFIGURE_FLAG_ENCODE = 1; 1915 1916 /** @hide */ 1917 @IntDef(flag = true, value = { CONFIGURE_FLAG_ENCODE }) 1918 @Retention(RetentionPolicy.SOURCE) 1919 public @interface ConfigureFlag {} 1920 1921 /** 1922 * Configures a component. 1923 * 1924 * @param format The format of the input data (decoder) or the desired 1925 * format of the output data (encoder). Passing {@code null} 1926 * as {@code format} is equivalent to passing an 1927 * {@link MediaFormat#MediaFormat an empty mediaformat}. 1928 * @param surface Specify a surface on which to render the output of this 1929 * decoder. Pass {@code null} as {@code surface} if the 1930 * codec does not generate raw video output (e.g. not a video 1931 * decoder) and/or if you want to configure the codec for 1932 * {@link ByteBuffer} output. 1933 * @param crypto Specify a crypto object to facilitate secure decryption 1934 * of the media data. Pass {@code null} as {@code crypto} for 1935 * non-secure codecs. 1936 * Please note that {@link MediaCodec} does NOT take ownership 1937 * of the {@link MediaCrypto} object; it is the application's 1938 * responsibility to properly cleanup the {@link MediaCrypto} object 1939 * when not in use. 1940 * @param flags Specify {@link #CONFIGURE_FLAG_ENCODE} to configure the 1941 * component as an encoder. 1942 * @throws IllegalArgumentException if the surface has been released (or is invalid), 1943 * or the format is unacceptable (e.g. missing a mandatory key), 1944 * or the flags are not set properly 1945 * (e.g. missing {@link #CONFIGURE_FLAG_ENCODE} for an encoder). 1946 * @throws IllegalStateException if not in the Uninitialized state. 1947 * @throws CryptoException upon DRM error. 1948 * @throws CodecException upon codec error. 1949 */ configure( @ullable MediaFormat format, @Nullable Surface surface, @Nullable MediaCrypto crypto, @ConfigureFlag int flags)1950 public void configure( 1951 @Nullable MediaFormat format, 1952 @Nullable Surface surface, @Nullable MediaCrypto crypto, 1953 @ConfigureFlag int flags) { 1954 configure(format, surface, crypto, null, flags); 1955 } 1956 1957 /** 1958 * Configure a component to be used with a descrambler. 1959 * @param format The format of the input data (decoder) or the desired 1960 * format of the output data (encoder). Passing {@code null} 1961 * as {@code format} is equivalent to passing an 1962 * {@link MediaFormat#MediaFormat an empty mediaformat}. 1963 * @param surface Specify a surface on which to render the output of this 1964 * decoder. Pass {@code null} as {@code surface} if the 1965 * codec does not generate raw video output (e.g. not a video 1966 * decoder) and/or if you want to configure the codec for 1967 * {@link ByteBuffer} output. 1968 * @param flags Specify {@link #CONFIGURE_FLAG_ENCODE} to configure the 1969 * component as an encoder. 1970 * @param descrambler Specify a descrambler object to facilitate secure 1971 * descrambling of the media data, or null for non-secure codecs. 1972 * @throws IllegalArgumentException if the surface has been released (or is invalid), 1973 * or the format is unacceptable (e.g. missing a mandatory key), 1974 * or the flags are not set properly 1975 * (e.g. missing {@link #CONFIGURE_FLAG_ENCODE} for an encoder). 1976 * @throws IllegalStateException if not in the Uninitialized state. 1977 * @throws CryptoException upon DRM error. 1978 * @throws CodecException upon codec error. 1979 */ configure( @ullable MediaFormat format, @Nullable Surface surface, @ConfigureFlag int flags, @Nullable MediaDescrambler descrambler)1980 public void configure( 1981 @Nullable MediaFormat format, @Nullable Surface surface, 1982 @ConfigureFlag int flags, @Nullable MediaDescrambler descrambler) { 1983 configure(format, surface, null, 1984 descrambler != null ? descrambler.getBinder() : null, flags); 1985 } 1986 configure( @ullable MediaFormat format, @Nullable Surface surface, @Nullable MediaCrypto crypto, @Nullable IHwBinder descramblerBinder, @ConfigureFlag int flags)1987 private void configure( 1988 @Nullable MediaFormat format, @Nullable Surface surface, 1989 @Nullable MediaCrypto crypto, @Nullable IHwBinder descramblerBinder, 1990 @ConfigureFlag int flags) { 1991 if (crypto != null && descramblerBinder != null) { 1992 throw new IllegalArgumentException("Can't use crypto and descrambler together!"); 1993 } 1994 1995 String[] keys = null; 1996 Object[] values = null; 1997 1998 if (format != null) { 1999 Map<String, Object> formatMap = format.getMap(); 2000 keys = new String[formatMap.size()]; 2001 values = new Object[formatMap.size()]; 2002 2003 int i = 0; 2004 for (Map.Entry<String, Object> entry: formatMap.entrySet()) { 2005 if (entry.getKey().equals(MediaFormat.KEY_AUDIO_SESSION_ID)) { 2006 int sessionId = 0; 2007 try { 2008 sessionId = (Integer)entry.getValue(); 2009 } 2010 catch (Exception e) { 2011 throw new IllegalArgumentException("Wrong Session ID Parameter!"); 2012 } 2013 keys[i] = "audio-hw-sync"; 2014 values[i] = AudioSystem.getAudioHwSyncForSession(sessionId); 2015 } else { 2016 keys[i] = entry.getKey(); 2017 values[i] = entry.getValue(); 2018 } 2019 ++i; 2020 } 2021 } 2022 2023 mHasSurface = surface != null; 2024 mCrypto = crypto; 2025 2026 native_configure(keys, values, surface, crypto, descramblerBinder, flags); 2027 } 2028 2029 /** 2030 * Dynamically sets the output surface of a codec. 2031 * <p> 2032 * This can only be used if the codec was configured with an output surface. The 2033 * new output surface should have a compatible usage type to the original output surface. 2034 * E.g. codecs may not support switching from a SurfaceTexture (GPU readable) output 2035 * to ImageReader (software readable) output. 2036 * @param surface the output surface to use. It must not be {@code null}. 2037 * @throws IllegalStateException if the codec does not support setting the output 2038 * surface in the current state. 2039 * @throws IllegalArgumentException if the new surface is not of a suitable type for the codec. 2040 */ setOutputSurface(@onNull Surface surface)2041 public void setOutputSurface(@NonNull Surface surface) { 2042 if (!mHasSurface) { 2043 throw new IllegalStateException("codec was not configured for an output surface"); 2044 } 2045 native_setSurface(surface); 2046 } 2047 native_setSurface(@onNull Surface surface)2048 private native void native_setSurface(@NonNull Surface surface); 2049 2050 /** 2051 * Create a persistent input surface that can be used with codecs that normally have an input 2052 * surface, such as video encoders. A persistent input can be reused by subsequent 2053 * {@link MediaCodec} or {@link MediaRecorder} instances, but can only be used by at 2054 * most one codec or recorder instance concurrently. 2055 * <p> 2056 * The application is responsible for calling release() on the Surface when done. 2057 * 2058 * @return an input surface that can be used with {@link #setInputSurface}. 2059 */ 2060 @NonNull createPersistentInputSurface()2061 public static Surface createPersistentInputSurface() { 2062 return native_createPersistentInputSurface(); 2063 } 2064 2065 static class PersistentSurface extends Surface { 2066 @SuppressWarnings("unused") PersistentSurface()2067 PersistentSurface() {} // used by native 2068 2069 @Override release()2070 public void release() { 2071 native_releasePersistentInputSurface(this); 2072 super.release(); 2073 } 2074 2075 private long mPersistentObject; 2076 }; 2077 2078 /** 2079 * Configures the codec (e.g. encoder) to use a persistent input surface in place of input 2080 * buffers. This may only be called after {@link #configure} and before {@link #start}, in 2081 * lieu of {@link #createInputSurface}. 2082 * @param surface a persistent input surface created by {@link #createPersistentInputSurface} 2083 * @throws IllegalStateException if not in the Configured state or does not require an input 2084 * surface. 2085 * @throws IllegalArgumentException if the surface was not created by 2086 * {@link #createPersistentInputSurface}. 2087 */ setInputSurface(@onNull Surface surface)2088 public void setInputSurface(@NonNull Surface surface) { 2089 if (!(surface instanceof PersistentSurface)) { 2090 throw new IllegalArgumentException("not a PersistentSurface"); 2091 } 2092 native_setInputSurface(surface); 2093 } 2094 2095 @NonNull native_createPersistentInputSurface()2096 private static native final PersistentSurface native_createPersistentInputSurface(); native_releasePersistentInputSurface(@onNull Surface surface)2097 private static native final void native_releasePersistentInputSurface(@NonNull Surface surface); native_setInputSurface(@onNull Surface surface)2098 private native final void native_setInputSurface(@NonNull Surface surface); 2099 native_setCallback(@ullable Callback cb)2100 private native final void native_setCallback(@Nullable Callback cb); 2101 native_configure( @ullable String[] keys, @Nullable Object[] values, @Nullable Surface surface, @Nullable MediaCrypto crypto, @Nullable IHwBinder descramblerBinder, @ConfigureFlag int flags)2102 private native final void native_configure( 2103 @Nullable String[] keys, @Nullable Object[] values, 2104 @Nullable Surface surface, @Nullable MediaCrypto crypto, 2105 @Nullable IHwBinder descramblerBinder, @ConfigureFlag int flags); 2106 2107 /** 2108 * Requests a Surface to use as the input to an encoder, in place of input buffers. This 2109 * may only be called after {@link #configure} and before {@link #start}. 2110 * <p> 2111 * The application is responsible for calling release() on the Surface when 2112 * done. 2113 * <p> 2114 * The Surface must be rendered with a hardware-accelerated API, such as OpenGL ES. 2115 * {@link android.view.Surface#lockCanvas(android.graphics.Rect)} may fail or produce 2116 * unexpected results. 2117 * @throws IllegalStateException if not in the Configured state. 2118 */ 2119 @NonNull createInputSurface()2120 public native final Surface createInputSurface(); 2121 2122 /** 2123 * After successfully configuring the component, call {@code start}. 2124 * <p> 2125 * Call {@code start} also if the codec is configured in asynchronous mode, 2126 * and it has just been flushed, to resume requesting input buffers. 2127 * @throws IllegalStateException if not in the Configured state 2128 * or just after {@link #flush} for a codec that is configured 2129 * in asynchronous mode. 2130 * @throws MediaCodec.CodecException upon codec error. Note that some codec errors 2131 * for start may be attributed to future method calls. 2132 */ start()2133 public final void start() { 2134 native_start(); 2135 synchronized(mBufferLock) { 2136 cacheBuffers(true /* input */); 2137 cacheBuffers(false /* input */); 2138 } 2139 } native_start()2140 private native final void native_start(); 2141 2142 /** 2143 * Finish the decode/encode session, note that the codec instance 2144 * remains active and ready to be {@link #start}ed again. 2145 * To ensure that it is available to other client call {@link #release} 2146 * and don't just rely on garbage collection to eventually do this for you. 2147 * @throws IllegalStateException if in the Released state. 2148 */ stop()2149 public final void stop() { 2150 native_stop(); 2151 freeAllTrackedBuffers(); 2152 2153 synchronized (mListenerLock) { 2154 if (mCallbackHandler != null) { 2155 mCallbackHandler.removeMessages(EVENT_SET_CALLBACK); 2156 mCallbackHandler.removeMessages(EVENT_CALLBACK); 2157 } 2158 if (mOnFrameRenderedHandler != null) { 2159 mOnFrameRenderedHandler.removeMessages(EVENT_FRAME_RENDERED); 2160 } 2161 } 2162 } 2163 native_stop()2164 private native final void native_stop(); 2165 2166 /** 2167 * Flush both input and output ports of the component. 2168 * <p> 2169 * Upon return, all indices previously returned in calls to {@link #dequeueInputBuffer 2170 * dequeueInputBuffer} and {@link #dequeueOutputBuffer dequeueOutputBuffer} — or obtained 2171 * via {@link Callback#onInputBufferAvailable onInputBufferAvailable} or 2172 * {@link Callback#onOutputBufferAvailable onOutputBufferAvailable} callbacks — become 2173 * invalid, and all buffers are owned by the codec. 2174 * <p> 2175 * If the codec is configured in asynchronous mode, call {@link #start} 2176 * after {@code flush} has returned to resume codec operations. The codec 2177 * will not request input buffers until this has happened. 2178 * <strong>Note, however, that there may still be outstanding {@code onOutputBufferAvailable} 2179 * callbacks that were not handled prior to calling {@code flush}. 2180 * The indices returned via these callbacks also become invalid upon calling {@code flush} and 2181 * should be discarded.</strong> 2182 * <p> 2183 * If the codec is configured in synchronous mode, codec will resume 2184 * automatically if it is configured with an input surface. Otherwise, it 2185 * will resume when {@link #dequeueInputBuffer dequeueInputBuffer} is called. 2186 * 2187 * @throws IllegalStateException if not in the Executing state. 2188 * @throws MediaCodec.CodecException upon codec error. 2189 */ flush()2190 public final void flush() { 2191 synchronized(mBufferLock) { 2192 invalidateByteBuffers(mCachedInputBuffers); 2193 invalidateByteBuffers(mCachedOutputBuffers); 2194 mDequeuedInputBuffers.clear(); 2195 mDequeuedOutputBuffers.clear(); 2196 } 2197 native_flush(); 2198 } 2199 native_flush()2200 private native final void native_flush(); 2201 2202 /** 2203 * Thrown when an internal codec error occurs. 2204 */ 2205 public final static class CodecException extends IllegalStateException { 2206 @UnsupportedAppUsage CodecException(int errorCode, int actionCode, @Nullable String detailMessage)2207 CodecException(int errorCode, int actionCode, @Nullable String detailMessage) { 2208 super(detailMessage); 2209 mErrorCode = errorCode; 2210 mActionCode = actionCode; 2211 2212 // TODO get this from codec 2213 final String sign = errorCode < 0 ? "neg_" : ""; 2214 mDiagnosticInfo = 2215 "android.media.MediaCodec.error_" + sign + Math.abs(errorCode); 2216 } 2217 2218 /** 2219 * Returns true if the codec exception is a transient issue, 2220 * perhaps due to resource constraints, and that the method 2221 * (or encoding/decoding) may be retried at a later time. 2222 */ 2223 public boolean isTransient() { 2224 return mActionCode == ACTION_TRANSIENT; 2225 } 2226 2227 /** 2228 * Returns true if the codec cannot proceed further, 2229 * but can be recovered by stopping, configuring, 2230 * and starting again. 2231 */ 2232 public boolean isRecoverable() { 2233 return mActionCode == ACTION_RECOVERABLE; 2234 } 2235 2236 /** 2237 * Retrieve the error code associated with a CodecException 2238 */ 2239 public int getErrorCode() { 2240 return mErrorCode; 2241 } 2242 2243 /** 2244 * Retrieve a developer-readable diagnostic information string 2245 * associated with the exception. Do not show this to end-users, 2246 * since this string will not be localized or generally 2247 * comprehensible to end-users. 2248 */ 2249 public @NonNull String getDiagnosticInfo() { 2250 return mDiagnosticInfo; 2251 } 2252 2253 /** 2254 * This indicates required resource was not able to be allocated. 2255 */ 2256 public static final int ERROR_INSUFFICIENT_RESOURCE = 1100; 2257 2258 /** 2259 * This indicates the resource manager reclaimed the media resource used by the codec. 2260 * <p> 2261 * With this exception, the codec must be released, as it has moved to terminal state. 2262 */ 2263 public static final int ERROR_RECLAIMED = 1101; 2264 2265 /** @hide */ 2266 @IntDef({ 2267 ERROR_INSUFFICIENT_RESOURCE, 2268 ERROR_RECLAIMED, 2269 }) 2270 @Retention(RetentionPolicy.SOURCE) 2271 public @interface ReasonCode {} 2272 2273 /* Must be in sync with android_media_MediaCodec.cpp */ 2274 private final static int ACTION_TRANSIENT = 1; 2275 private final static int ACTION_RECOVERABLE = 2; 2276 2277 private final String mDiagnosticInfo; 2278 private final int mErrorCode; 2279 private final int mActionCode; 2280 } 2281 2282 /** 2283 * Thrown when a crypto error occurs while queueing a secure input buffer. 2284 */ 2285 public final static class CryptoException extends RuntimeException { 2286 public CryptoException(int errorCode, @Nullable String detailMessage) { 2287 super(detailMessage); 2288 mErrorCode = errorCode; 2289 } 2290 2291 /** 2292 * This indicates that the requested key was not found when trying to 2293 * perform a decrypt operation. The operation can be retried after adding 2294 * the correct decryption key. 2295 */ 2296 public static final int ERROR_NO_KEY = 1; 2297 2298 /** 2299 * This indicates that the key used for decryption is no longer 2300 * valid due to license term expiration. The operation can be retried 2301 * after updating the expired keys. 2302 */ 2303 public static final int ERROR_KEY_EXPIRED = 2; 2304 2305 /** 2306 * This indicates that a required crypto resource was not able to be 2307 * allocated while attempting the requested operation. The operation 2308 * can be retried if the app is able to release resources. 2309 */ 2310 public static final int ERROR_RESOURCE_BUSY = 3; 2311 2312 /** 2313 * This indicates that the output protection levels supported by the 2314 * device are not sufficient to meet the requirements set by the 2315 * content owner in the license policy. 2316 */ 2317 public static final int ERROR_INSUFFICIENT_OUTPUT_PROTECTION = 4; 2318 2319 /** 2320 * This indicates that decryption was attempted on a session that is 2321 * not opened, which could be due to a failure to open the session, 2322 * closing the session prematurely, or the session being reclaimed 2323 * by the resource manager. 2324 */ 2325 public static final int ERROR_SESSION_NOT_OPENED = 5; 2326 2327 /** 2328 * This indicates that an operation was attempted that could not be 2329 * supported by the crypto system of the device in its current 2330 * configuration. It may occur when the license policy requires 2331 * device security features that aren't supported by the device, 2332 * or due to an internal error in the crypto system that prevents 2333 * the specified security policy from being met. 2334 */ 2335 public static final int ERROR_UNSUPPORTED_OPERATION = 6; 2336 2337 /** 2338 * This indicates that the security level of the device is not 2339 * sufficient to meet the requirements set by the content owner 2340 * in the license policy. 2341 */ 2342 public static final int ERROR_INSUFFICIENT_SECURITY = 7; 2343 2344 /** 2345 * This indicates that the video frame being decrypted exceeds 2346 * the size of the device's protected output buffers. When 2347 * encountering this error the app should try playing content 2348 * of a lower resolution. 2349 */ 2350 public static final int ERROR_FRAME_TOO_LARGE = 8; 2351 2352 /** 2353 * This error indicates that session state has been 2354 * invalidated. It can occur on devices that are not capable 2355 * of retaining crypto session state across device 2356 * suspend/resume. The session must be closed and a new 2357 * session opened to resume operation. 2358 */ 2359 public static final int ERROR_LOST_STATE = 9; 2360 2361 /** @hide */ 2362 @IntDef({ 2363 ERROR_NO_KEY, 2364 ERROR_KEY_EXPIRED, 2365 ERROR_RESOURCE_BUSY, 2366 ERROR_INSUFFICIENT_OUTPUT_PROTECTION, 2367 ERROR_SESSION_NOT_OPENED, 2368 ERROR_UNSUPPORTED_OPERATION, 2369 ERROR_INSUFFICIENT_SECURITY, 2370 ERROR_FRAME_TOO_LARGE, 2371 ERROR_LOST_STATE 2372 }) 2373 @Retention(RetentionPolicy.SOURCE) 2374 public @interface CryptoErrorCode {} 2375 2376 /** 2377 * Retrieve the error code associated with a CryptoException 2378 */ 2379 @CryptoErrorCode 2380 public int getErrorCode() { 2381 return mErrorCode; 2382 } 2383 2384 private int mErrorCode; 2385 } 2386 2387 /** 2388 * After filling a range of the input buffer at the specified index 2389 * submit it to the component. Once an input buffer is queued to 2390 * the codec, it MUST NOT be used until it is later retrieved by 2391 * {@link #getInputBuffer} in response to a {@link #dequeueInputBuffer} 2392 * return value or a {@link Callback#onInputBufferAvailable} 2393 * callback. 2394 * <p> 2395 * Many decoders require the actual compressed data stream to be 2396 * preceded by "codec specific data", i.e. setup data used to initialize 2397 * the codec such as PPS/SPS in the case of AVC video or code tables 2398 * in the case of vorbis audio. 2399 * The class {@link android.media.MediaExtractor} provides codec 2400 * specific data as part of 2401 * the returned track format in entries named "csd-0", "csd-1" ... 2402 * <p> 2403 * These buffers can be submitted directly after {@link #start} or 2404 * {@link #flush} by specifying the flag {@link 2405 * #BUFFER_FLAG_CODEC_CONFIG}. However, if you configure the 2406 * codec with a {@link MediaFormat} containing these keys, they 2407 * will be automatically submitted by MediaCodec directly after 2408 * start. Therefore, the use of {@link 2409 * #BUFFER_FLAG_CODEC_CONFIG} flag is discouraged and is 2410 * recommended only for advanced users. 2411 * <p> 2412 * To indicate that this is the final piece of input data (or rather that 2413 * no more input data follows unless the decoder is subsequently flushed) 2414 * specify the flag {@link #BUFFER_FLAG_END_OF_STREAM}. 2415 * <p class=note> 2416 * <strong>Note:</strong> Prior to {@link android.os.Build.VERSION_CODES#M}, 2417 * {@code presentationTimeUs} was not propagated to the frame timestamp of (rendered) 2418 * Surface output buffers, and the resulting frame timestamp was undefined. 2419 * Use {@link #releaseOutputBuffer(int, long)} to ensure a specific frame timestamp is set. 2420 * Similarly, since frame timestamps can be used by the destination surface for rendering 2421 * synchronization, <strong>care must be taken to normalize presentationTimeUs so as to not be 2422 * mistaken for a system time. (See {@linkplain #releaseOutputBuffer(int, long) 2423 * SurfaceView specifics}).</strong> 2424 * 2425 * @param index The index of a client-owned input buffer previously returned 2426 * in a call to {@link #dequeueInputBuffer}. 2427 * @param offset The byte offset into the input buffer at which the data starts. 2428 * @param size The number of bytes of valid input data. 2429 * @param presentationTimeUs The presentation timestamp in microseconds for this 2430 * buffer. This is normally the media time at which this 2431 * buffer should be presented (rendered). When using an output 2432 * surface, this will be propagated as the {@link 2433 * SurfaceTexture#getTimestamp timestamp} for the frame (after 2434 * conversion to nanoseconds). 2435 * @param flags A bitmask of flags 2436 * {@link #BUFFER_FLAG_CODEC_CONFIG} and {@link #BUFFER_FLAG_END_OF_STREAM}. 2437 * While not prohibited, most codecs do not use the 2438 * {@link #BUFFER_FLAG_KEY_FRAME} flag for input buffers. 2439 * @throws IllegalStateException if not in the Executing state. 2440 * @throws MediaCodec.CodecException upon codec error. 2441 * @throws CryptoException if a crypto object has been specified in 2442 * {@link #configure} 2443 */ 2444 public final void queueInputBuffer( 2445 int index, 2446 int offset, int size, long presentationTimeUs, int flags) 2447 throws CryptoException { 2448 synchronized(mBufferLock) { 2449 invalidateByteBuffer(mCachedInputBuffers, index); 2450 mDequeuedInputBuffers.remove(index); 2451 } 2452 try { 2453 native_queueInputBuffer( 2454 index, offset, size, presentationTimeUs, flags); 2455 } catch (CryptoException | IllegalStateException e) { 2456 revalidateByteBuffer(mCachedInputBuffers, index); 2457 throw e; 2458 } 2459 } 2460 2461 private native final void native_queueInputBuffer( 2462 int index, 2463 int offset, int size, long presentationTimeUs, int flags) 2464 throws CryptoException; 2465 2466 public static final int CRYPTO_MODE_UNENCRYPTED = 0; 2467 public static final int CRYPTO_MODE_AES_CTR = 1; 2468 public static final int CRYPTO_MODE_AES_CBC = 2; 2469 2470 /** 2471 * Metadata describing the structure of an encrypted input sample. 2472 * <p> 2473 * A buffer's data is considered to be partitioned into "subSamples". Each subSample starts with 2474 * a run of plain, unencrypted bytes followed by a run of encrypted bytes. Either of these runs 2475 * may be empty. If pattern encryption applies, each of the encrypted runs is encrypted only 2476 * partly, according to a repeating pattern of "encrypt" and "skip" blocks. 2477 * {@link #numBytesOfClearData} can be null to indicate that all data is encrypted, and 2478 * {@link #numBytesOfEncryptedData} can be null to indicate that all data is clear. At least one 2479 * of {@link #numBytesOfClearData} and {@link #numBytesOfEncryptedData} must be non-null. 2480 * <p> 2481 * This information encapsulates per-sample metadata as outlined in ISO/IEC FDIS 23001-7:2016 2482 * "Common encryption in ISO base media file format files". 2483 * <p> 2484 * <h3>ISO-CENC Schemes</h3> 2485 * ISO/IEC FDIS 23001-7:2016 defines four possible schemes by which media may be encrypted, 2486 * corresponding to each possible combination of an AES mode with the presence or absence of 2487 * patterned encryption. 2488 * 2489 * <table style="width: 0%"> 2490 * <thead> 2491 * <tr> 2492 * <th> </th> 2493 * <th>AES-CTR</th> 2494 * <th>AES-CBC</th> 2495 * </tr> 2496 * </thead> 2497 * <tbody> 2498 * <tr> 2499 * <th>Without Patterns</th> 2500 * <td>cenc</td> 2501 * <td>cbc1</td> 2502 * </tr><tr> 2503 * <th>With Patterns</th> 2504 * <td>cens</td> 2505 * <td>cbcs</td> 2506 * </tr> 2507 * </tbody> 2508 * </table> 2509 * 2510 * For {@code CryptoInfo}, the scheme is selected implicitly by the combination of the 2511 * {@link #mode} field and the value set with {@link #setPattern}. For the pattern, setting the 2512 * pattern to all zeroes (that is, both {@code blocksToEncrypt} and {@code blocksToSkip} are 2513 * zero) is interpreted as turning patterns off completely. A scheme that does not use patterns 2514 * will be selected, either cenc or cbc1. Setting the pattern to any nonzero value will choose 2515 * one of the pattern-supporting schemes, cens or cbcs. The default pattern if 2516 * {@link #setPattern} is never called is all zeroes. 2517 * <p> 2518 * <h4>HLS SAMPLE-AES Audio</h4> 2519 * HLS SAMPLE-AES audio is encrypted in a manner compatible with the cbcs scheme, except that it 2520 * does not use patterned encryption. However, if {@link #setPattern} is used to set the pattern 2521 * to all zeroes, this will be interpreted as selecting the cbc1 scheme. The cbc1 scheme cannot 2522 * successfully decrypt HLS SAMPLE-AES audio because of differences in how the IVs are handled. 2523 * For this reason, it is recommended that a pattern of {@code 1} encrypted block and {@code 0} 2524 * skip blocks be used with HLS SAMPLE-AES audio. This will trigger decryption to use cbcs mode 2525 * while still decrypting every block. 2526 */ 2527 public final static class CryptoInfo { 2528 /** 2529 * The number of subSamples that make up the buffer's contents. 2530 */ 2531 public int numSubSamples; 2532 /** 2533 * The number of leading unencrypted bytes in each subSample. If null, all bytes are treated 2534 * as encrypted and {@link #numBytesOfEncryptedData} must be specified. 2535 */ 2536 public int[] numBytesOfClearData; 2537 /** 2538 * The number of trailing encrypted bytes in each subSample. If null, all bytes are treated 2539 * as clear and {@link #numBytesOfClearData} must be specified. 2540 */ 2541 public int[] numBytesOfEncryptedData; 2542 /** 2543 * A 16-byte key id 2544 */ 2545 public byte[] key; 2546 /** 2547 * A 16-byte initialization vector 2548 */ 2549 public byte[] iv; 2550 /** 2551 * The type of encryption that has been applied, 2552 * see {@link #CRYPTO_MODE_UNENCRYPTED}, {@link #CRYPTO_MODE_AES_CTR} 2553 * and {@link #CRYPTO_MODE_AES_CBC} 2554 */ 2555 public int mode; 2556 2557 /** 2558 * Metadata describing an encryption pattern for the protected bytes in a subsample. An 2559 * encryption pattern consists of a repeating sequence of crypto blocks comprised of a 2560 * number of encrypted blocks followed by a number of unencrypted, or skipped, blocks. 2561 */ 2562 public final static class Pattern { 2563 /** 2564 * Number of blocks to be encrypted in the pattern. If both this and 2565 * {@link #mSkipBlocks} are zero, pattern encryption is inoperative. 2566 */ 2567 private int mEncryptBlocks; 2568 2569 /** 2570 * Number of blocks to be skipped (left clear) in the pattern. If both this and 2571 * {@link #mEncryptBlocks} are zero, pattern encryption is inoperative. 2572 */ 2573 private int mSkipBlocks; 2574 2575 /** 2576 * Construct a sample encryption pattern given the number of blocks to encrypt and skip 2577 * in the pattern. If both parameters are zero, pattern encryption is inoperative. 2578 */ 2579 public Pattern(int blocksToEncrypt, int blocksToSkip) { 2580 set(blocksToEncrypt, blocksToSkip); 2581 } 2582 2583 /** 2584 * Set the number of blocks to encrypt and skip in a sample encryption pattern. If both 2585 * parameters are zero, pattern encryption is inoperative. 2586 */ 2587 public void set(int blocksToEncrypt, int blocksToSkip) { 2588 mEncryptBlocks = blocksToEncrypt; 2589 mSkipBlocks = blocksToSkip; 2590 } 2591 2592 /** 2593 * Return the number of blocks to skip in a sample encryption pattern. 2594 */ 2595 public int getSkipBlocks() { 2596 return mSkipBlocks; 2597 } 2598 2599 /** 2600 * Return the number of blocks to encrypt in a sample encryption pattern. 2601 */ 2602 public int getEncryptBlocks() { 2603 return mEncryptBlocks; 2604 } 2605 }; 2606 2607 private final Pattern zeroPattern = new Pattern(0, 0); 2608 2609 /** 2610 * The pattern applicable to the protected data in each subsample. 2611 */ 2612 private Pattern pattern; 2613 2614 /** 2615 * Set the subsample count, clear/encrypted sizes, key, IV and mode fields of 2616 * a {@link MediaCodec.CryptoInfo} instance. 2617 */ 2618 public void set( 2619 int newNumSubSamples, 2620 @NonNull int[] newNumBytesOfClearData, 2621 @NonNull int[] newNumBytesOfEncryptedData, 2622 @NonNull byte[] newKey, 2623 @NonNull byte[] newIV, 2624 int newMode) { 2625 numSubSamples = newNumSubSamples; 2626 numBytesOfClearData = newNumBytesOfClearData; 2627 numBytesOfEncryptedData = newNumBytesOfEncryptedData; 2628 key = newKey; 2629 iv = newIV; 2630 mode = newMode; 2631 pattern = zeroPattern; 2632 } 2633 2634 /** 2635 * Set the encryption pattern on a {@link MediaCodec.CryptoInfo} instance. 2636 * See {@link MediaCodec.CryptoInfo.Pattern}. 2637 */ 2638 public void setPattern(Pattern newPattern) { 2639 pattern = newPattern; 2640 } 2641 2642 private void setPattern(int blocksToEncrypt, int blocksToSkip) { 2643 pattern = new Pattern(blocksToEncrypt, blocksToSkip); 2644 } 2645 2646 @Override 2647 public String toString() { 2648 StringBuilder builder = new StringBuilder(); 2649 builder.append(numSubSamples + " subsamples, key ["); 2650 String hexdigits = "0123456789abcdef"; 2651 for (int i = 0; i < key.length; i++) { 2652 builder.append(hexdigits.charAt((key[i] & 0xf0) >> 4)); 2653 builder.append(hexdigits.charAt(key[i] & 0x0f)); 2654 } 2655 builder.append("], iv ["); 2656 for (int i = 0; i < key.length; i++) { 2657 builder.append(hexdigits.charAt((iv[i] & 0xf0) >> 4)); 2658 builder.append(hexdigits.charAt(iv[i] & 0x0f)); 2659 } 2660 builder.append("], clear "); Arrays.toString(numBytesOfClearData)2661 builder.append(Arrays.toString(numBytesOfClearData)); 2662 builder.append(", encrypted "); Arrays.toString(numBytesOfEncryptedData)2663 builder.append(Arrays.toString(numBytesOfEncryptedData)); 2664 return builder.toString(); 2665 } 2666 }; 2667 2668 /** 2669 * Similar to {@link #queueInputBuffer queueInputBuffer} but submits a buffer that is 2670 * potentially encrypted. 2671 * <strong>Check out further notes at {@link #queueInputBuffer queueInputBuffer}.</strong> 2672 * 2673 * @param index The index of a client-owned input buffer previously returned 2674 * in a call to {@link #dequeueInputBuffer}. 2675 * @param offset The byte offset into the input buffer at which the data starts. 2676 * @param info Metadata required to facilitate decryption, the object can be 2677 * reused immediately after this call returns. 2678 * @param presentationTimeUs The presentation timestamp in microseconds for this 2679 * buffer. This is normally the media time at which this 2680 * buffer should be presented (rendered). 2681 * @param flags A bitmask of flags 2682 * {@link #BUFFER_FLAG_CODEC_CONFIG} and {@link #BUFFER_FLAG_END_OF_STREAM}. 2683 * While not prohibited, most codecs do not use the 2684 * {@link #BUFFER_FLAG_KEY_FRAME} flag for input buffers. 2685 * @throws IllegalStateException if not in the Executing state. 2686 * @throws MediaCodec.CodecException upon codec error. 2687 * @throws CryptoException if an error occurs while attempting to decrypt the buffer. 2688 * An error code associated with the exception helps identify the 2689 * reason for the failure. 2690 */ queueSecureInputBuffer( int index, int offset, @NonNull CryptoInfo info, long presentationTimeUs, int flags)2691 public final void queueSecureInputBuffer( 2692 int index, 2693 int offset, 2694 @NonNull CryptoInfo info, 2695 long presentationTimeUs, 2696 int flags) throws CryptoException { 2697 synchronized(mBufferLock) { 2698 invalidateByteBuffer(mCachedInputBuffers, index); 2699 mDequeuedInputBuffers.remove(index); 2700 } 2701 try { 2702 native_queueSecureInputBuffer( 2703 index, offset, info, presentationTimeUs, flags); 2704 } catch (CryptoException | IllegalStateException e) { 2705 revalidateByteBuffer(mCachedInputBuffers, index); 2706 throw e; 2707 } 2708 } 2709 native_queueSecureInputBuffer( int index, int offset, @NonNull CryptoInfo info, long presentationTimeUs, int flags)2710 private native final void native_queueSecureInputBuffer( 2711 int index, 2712 int offset, 2713 @NonNull CryptoInfo info, 2714 long presentationTimeUs, 2715 int flags) throws CryptoException; 2716 2717 /** 2718 * Returns the index of an input buffer to be filled with valid data 2719 * or -1 if no such buffer is currently available. 2720 * This method will return immediately if timeoutUs == 0, wait indefinitely 2721 * for the availability of an input buffer if timeoutUs < 0 or wait up 2722 * to "timeoutUs" microseconds if timeoutUs > 0. 2723 * @param timeoutUs The timeout in microseconds, a negative timeout indicates "infinite". 2724 * @throws IllegalStateException if not in the Executing state, 2725 * or codec is configured in asynchronous mode. 2726 * @throws MediaCodec.CodecException upon codec error. 2727 */ dequeueInputBuffer(long timeoutUs)2728 public final int dequeueInputBuffer(long timeoutUs) { 2729 int res = native_dequeueInputBuffer(timeoutUs); 2730 if (res >= 0) { 2731 synchronized(mBufferLock) { 2732 validateInputByteBuffer(mCachedInputBuffers, res); 2733 } 2734 } 2735 return res; 2736 } 2737 native_dequeueInputBuffer(long timeoutUs)2738 private native final int native_dequeueInputBuffer(long timeoutUs); 2739 2740 /** 2741 * If a non-negative timeout had been specified in the call 2742 * to {@link #dequeueOutputBuffer}, indicates that the call timed out. 2743 */ 2744 public static final int INFO_TRY_AGAIN_LATER = -1; 2745 2746 /** 2747 * The output format has changed, subsequent data will follow the new 2748 * format. {@link #getOutputFormat()} returns the new format. Note, that 2749 * you can also use the new {@link #getOutputFormat(int)} method to 2750 * get the format for a specific output buffer. This frees you from 2751 * having to track output format changes. 2752 */ 2753 public static final int INFO_OUTPUT_FORMAT_CHANGED = -2; 2754 2755 /** 2756 * The output buffers have changed, the client must refer to the new 2757 * set of output buffers returned by {@link #getOutputBuffers} from 2758 * this point on. 2759 * 2760 * <p>Additionally, this event signals that the video scaling mode 2761 * may have been reset to the default.</p> 2762 * 2763 * @deprecated This return value can be ignored as {@link 2764 * #getOutputBuffers} has been deprecated. Client should 2765 * request a current buffer using on of the get-buffer or 2766 * get-image methods each time one has been dequeued. 2767 */ 2768 public static final int INFO_OUTPUT_BUFFERS_CHANGED = -3; 2769 2770 /** @hide */ 2771 @IntDef({ 2772 INFO_TRY_AGAIN_LATER, 2773 INFO_OUTPUT_FORMAT_CHANGED, 2774 INFO_OUTPUT_BUFFERS_CHANGED, 2775 }) 2776 @Retention(RetentionPolicy.SOURCE) 2777 public @interface OutputBufferInfo {} 2778 2779 /** 2780 * Dequeue an output buffer, block at most "timeoutUs" microseconds. 2781 * Returns the index of an output buffer that has been successfully 2782 * decoded or one of the INFO_* constants. 2783 * @param info Will be filled with buffer meta data. 2784 * @param timeoutUs The timeout in microseconds, a negative timeout indicates "infinite". 2785 * @throws IllegalStateException if not in the Executing state, 2786 * or codec is configured in asynchronous mode. 2787 * @throws MediaCodec.CodecException upon codec error. 2788 */ 2789 @OutputBufferInfo dequeueOutputBuffer( @onNull BufferInfo info, long timeoutUs)2790 public final int dequeueOutputBuffer( 2791 @NonNull BufferInfo info, long timeoutUs) { 2792 int res = native_dequeueOutputBuffer(info, timeoutUs); 2793 synchronized(mBufferLock) { 2794 if (res == INFO_OUTPUT_BUFFERS_CHANGED) { 2795 cacheBuffers(false /* input */); 2796 } else if (res >= 0) { 2797 validateOutputByteBuffer(mCachedOutputBuffers, res, info); 2798 if (mHasSurface) { 2799 mDequeuedOutputInfos.put(res, info.dup()); 2800 } 2801 } 2802 } 2803 return res; 2804 } 2805 native_dequeueOutputBuffer( @onNull BufferInfo info, long timeoutUs)2806 private native final int native_dequeueOutputBuffer( 2807 @NonNull BufferInfo info, long timeoutUs); 2808 2809 /** 2810 * If you are done with a buffer, use this call to return the buffer to the codec 2811 * or to render it on the output surface. If you configured the codec with an 2812 * output surface, setting {@code render} to {@code true} will first send the buffer 2813 * to that output surface. The surface will release the buffer back to the codec once 2814 * it is no longer used/displayed. 2815 * 2816 * Once an output buffer is released to the codec, it MUST NOT 2817 * be used until it is later retrieved by {@link #getOutputBuffer} in response 2818 * to a {@link #dequeueOutputBuffer} return value or a 2819 * {@link Callback#onOutputBufferAvailable} callback. 2820 * 2821 * @param index The index of a client-owned output buffer previously returned 2822 * from a call to {@link #dequeueOutputBuffer}. 2823 * @param render If a valid surface was specified when configuring the codec, 2824 * passing true renders this output buffer to the surface. 2825 * @throws IllegalStateException if not in the Executing state. 2826 * @throws MediaCodec.CodecException upon codec error. 2827 */ releaseOutputBuffer(int index, boolean render)2828 public final void releaseOutputBuffer(int index, boolean render) { 2829 BufferInfo info = null; 2830 synchronized(mBufferLock) { 2831 invalidateByteBuffer(mCachedOutputBuffers, index); 2832 mDequeuedOutputBuffers.remove(index); 2833 if (mHasSurface) { 2834 info = mDequeuedOutputInfos.remove(index); 2835 } 2836 } 2837 releaseOutputBuffer(index, render, false /* updatePTS */, 0 /* dummy */); 2838 } 2839 2840 /** 2841 * If you are done with a buffer, use this call to update its surface timestamp 2842 * and return it to the codec to render it on the output surface. If you 2843 * have not specified an output surface when configuring this video codec, 2844 * this call will simply return the buffer to the codec.<p> 2845 * 2846 * The timestamp may have special meaning depending on the destination surface. 2847 * 2848 * <table> 2849 * <tr><th>SurfaceView specifics</th></tr> 2850 * <tr><td> 2851 * If you render your buffer on a {@link android.view.SurfaceView}, 2852 * you can use the timestamp to render the buffer at a specific time (at the 2853 * VSYNC at or after the buffer timestamp). For this to work, the timestamp 2854 * needs to be <i>reasonably close</i> to the current {@link System#nanoTime}. 2855 * Currently, this is set as within one (1) second. A few notes: 2856 * 2857 * <ul> 2858 * <li>the buffer will not be returned to the codec until the timestamp 2859 * has passed and the buffer is no longer used by the {@link android.view.Surface}. 2860 * <li>buffers are processed sequentially, so you may block subsequent buffers to 2861 * be displayed on the {@link android.view.Surface}. This is important if you 2862 * want to react to user action, e.g. stop the video or seek. 2863 * <li>if multiple buffers are sent to the {@link android.view.Surface} to be 2864 * rendered at the same VSYNC, the last one will be shown, and the other ones 2865 * will be dropped. 2866 * <li>if the timestamp is <em>not</em> "reasonably close" to the current system 2867 * time, the {@link android.view.Surface} will ignore the timestamp, and 2868 * display the buffer at the earliest feasible time. In this mode it will not 2869 * drop frames. 2870 * <li>for best performance and quality, call this method when you are about 2871 * two VSYNCs' time before the desired render time. For 60Hz displays, this is 2872 * about 33 msec. 2873 * </ul> 2874 * </td></tr> 2875 * </table> 2876 * 2877 * Once an output buffer is released to the codec, it MUST NOT 2878 * be used until it is later retrieved by {@link #getOutputBuffer} in response 2879 * to a {@link #dequeueOutputBuffer} return value or a 2880 * {@link Callback#onOutputBufferAvailable} callback. 2881 * 2882 * @param index The index of a client-owned output buffer previously returned 2883 * from a call to {@link #dequeueOutputBuffer}. 2884 * @param renderTimestampNs The timestamp to associate with this buffer when 2885 * it is sent to the Surface. 2886 * @throws IllegalStateException if not in the Executing state. 2887 * @throws MediaCodec.CodecException upon codec error. 2888 */ releaseOutputBuffer(int index, long renderTimestampNs)2889 public final void releaseOutputBuffer(int index, long renderTimestampNs) { 2890 BufferInfo info = null; 2891 synchronized(mBufferLock) { 2892 invalidateByteBuffer(mCachedOutputBuffers, index); 2893 mDequeuedOutputBuffers.remove(index); 2894 if (mHasSurface) { 2895 info = mDequeuedOutputInfos.remove(index); 2896 } 2897 } 2898 releaseOutputBuffer( 2899 index, true /* render */, true /* updatePTS */, renderTimestampNs); 2900 } 2901 2902 @UnsupportedAppUsage releaseOutputBuffer( int index, boolean render, boolean updatePTS, long timeNs)2903 private native final void releaseOutputBuffer( 2904 int index, boolean render, boolean updatePTS, long timeNs); 2905 2906 /** 2907 * Signals end-of-stream on input. Equivalent to submitting an empty buffer with 2908 * {@link #BUFFER_FLAG_END_OF_STREAM} set. This may only be used with 2909 * encoders receiving input from a Surface created by {@link #createInputSurface}. 2910 * @throws IllegalStateException if not in the Executing state. 2911 * @throws MediaCodec.CodecException upon codec error. 2912 */ signalEndOfInputStream()2913 public native final void signalEndOfInputStream(); 2914 2915 /** 2916 * Call this after dequeueOutputBuffer signals a format change by returning 2917 * {@link #INFO_OUTPUT_FORMAT_CHANGED}. 2918 * You can also call this after {@link #configure} returns 2919 * successfully to get the output format initially configured 2920 * for the codec. Do this to determine what optional 2921 * configuration parameters were supported by the codec. 2922 * 2923 * @throws IllegalStateException if not in the Executing or 2924 * Configured state. 2925 * @throws MediaCodec.CodecException upon codec error. 2926 */ 2927 @NonNull getOutputFormat()2928 public final MediaFormat getOutputFormat() { 2929 return new MediaFormat(getFormatNative(false /* input */)); 2930 } 2931 2932 /** 2933 * Call this after {@link #configure} returns successfully to 2934 * get the input format accepted by the codec. Do this to 2935 * determine what optional configuration parameters were 2936 * supported by the codec. 2937 * 2938 * @throws IllegalStateException if not in the Executing or 2939 * Configured state. 2940 * @throws MediaCodec.CodecException upon codec error. 2941 */ 2942 @NonNull getInputFormat()2943 public final MediaFormat getInputFormat() { 2944 return new MediaFormat(getFormatNative(true /* input */)); 2945 } 2946 2947 /** 2948 * Returns the output format for a specific output buffer. 2949 * 2950 * @param index The index of a client-owned input buffer previously 2951 * returned from a call to {@link #dequeueInputBuffer}. 2952 * 2953 * @return the format for the output buffer, or null if the index 2954 * is not a dequeued output buffer. 2955 */ 2956 @NonNull getOutputFormat(int index)2957 public final MediaFormat getOutputFormat(int index) { 2958 return new MediaFormat(getOutputFormatNative(index)); 2959 } 2960 2961 @NonNull getFormatNative(boolean input)2962 private native final Map<String, Object> getFormatNative(boolean input); 2963 2964 @NonNull getOutputFormatNative(int index)2965 private native final Map<String, Object> getOutputFormatNative(int index); 2966 2967 // used to track dequeued buffers 2968 private static class BufferMap { 2969 // various returned representations of the codec buffer 2970 private static class CodecBuffer { 2971 private Image mImage; 2972 private ByteBuffer mByteBuffer; 2973 free()2974 public void free() { 2975 if (mByteBuffer != null) { 2976 // all of our ByteBuffers are direct 2977 java.nio.NioUtils.freeDirectBuffer(mByteBuffer); 2978 mByteBuffer = null; 2979 } 2980 if (mImage != null) { 2981 mImage.close(); 2982 mImage = null; 2983 } 2984 } 2985 setImage(@ullable Image image)2986 public void setImage(@Nullable Image image) { 2987 free(); 2988 mImage = image; 2989 } 2990 setByteBuffer(@ullable ByteBuffer buffer)2991 public void setByteBuffer(@Nullable ByteBuffer buffer) { 2992 free(); 2993 mByteBuffer = buffer; 2994 } 2995 } 2996 2997 private final Map<Integer, CodecBuffer> mMap = 2998 new HashMap<Integer, CodecBuffer>(); 2999 remove(int index)3000 public void remove(int index) { 3001 CodecBuffer buffer = mMap.get(index); 3002 if (buffer != null) { 3003 buffer.free(); 3004 mMap.remove(index); 3005 } 3006 } 3007 put(int index, @Nullable ByteBuffer newBuffer)3008 public void put(int index, @Nullable ByteBuffer newBuffer) { 3009 CodecBuffer buffer = mMap.get(index); 3010 if (buffer == null) { // likely 3011 buffer = new CodecBuffer(); 3012 mMap.put(index, buffer); 3013 } 3014 buffer.setByteBuffer(newBuffer); 3015 } 3016 put(int index, @Nullable Image newImage)3017 public void put(int index, @Nullable Image newImage) { 3018 CodecBuffer buffer = mMap.get(index); 3019 if (buffer == null) { // likely 3020 buffer = new CodecBuffer(); 3021 mMap.put(index, buffer); 3022 } 3023 buffer.setImage(newImage); 3024 } 3025 clear()3026 public void clear() { 3027 for (CodecBuffer buffer: mMap.values()) { 3028 buffer.free(); 3029 } 3030 mMap.clear(); 3031 } 3032 } 3033 3034 private ByteBuffer[] mCachedInputBuffers; 3035 private ByteBuffer[] mCachedOutputBuffers; 3036 private final BufferMap mDequeuedInputBuffers = new BufferMap(); 3037 private final BufferMap mDequeuedOutputBuffers = new BufferMap(); 3038 private final Map<Integer, BufferInfo> mDequeuedOutputInfos = 3039 new HashMap<Integer, BufferInfo>(); 3040 final private Object mBufferLock; 3041 invalidateByteBuffer( @ullable ByteBuffer[] buffers, int index)3042 private final void invalidateByteBuffer( 3043 @Nullable ByteBuffer[] buffers, int index) { 3044 if (buffers != null && index >= 0 && index < buffers.length) { 3045 ByteBuffer buffer = buffers[index]; 3046 if (buffer != null) { 3047 buffer.setAccessible(false); 3048 } 3049 } 3050 } 3051 validateInputByteBuffer( @ullable ByteBuffer[] buffers, int index)3052 private final void validateInputByteBuffer( 3053 @Nullable ByteBuffer[] buffers, int index) { 3054 if (buffers != null && index >= 0 && index < buffers.length) { 3055 ByteBuffer buffer = buffers[index]; 3056 if (buffer != null) { 3057 buffer.setAccessible(true); 3058 buffer.clear(); 3059 } 3060 } 3061 } 3062 revalidateByteBuffer( @ullable ByteBuffer[] buffers, int index)3063 private final void revalidateByteBuffer( 3064 @Nullable ByteBuffer[] buffers, int index) { 3065 synchronized(mBufferLock) { 3066 if (buffers != null && index >= 0 && index < buffers.length) { 3067 ByteBuffer buffer = buffers[index]; 3068 if (buffer != null) { 3069 buffer.setAccessible(true); 3070 } 3071 } 3072 } 3073 } 3074 validateOutputByteBuffer( @ullable ByteBuffer[] buffers, int index, @NonNull BufferInfo info)3075 private final void validateOutputByteBuffer( 3076 @Nullable ByteBuffer[] buffers, int index, @NonNull BufferInfo info) { 3077 if (buffers != null && index >= 0 && index < buffers.length) { 3078 ByteBuffer buffer = buffers[index]; 3079 if (buffer != null) { 3080 buffer.setAccessible(true); 3081 buffer.limit(info.offset + info.size).position(info.offset); 3082 } 3083 } 3084 } 3085 invalidateByteBuffers(@ullable ByteBuffer[] buffers)3086 private final void invalidateByteBuffers(@Nullable ByteBuffer[] buffers) { 3087 if (buffers != null) { 3088 for (ByteBuffer buffer: buffers) { 3089 if (buffer != null) { 3090 buffer.setAccessible(false); 3091 } 3092 } 3093 } 3094 } 3095 freeByteBuffer(@ullable ByteBuffer buffer)3096 private final void freeByteBuffer(@Nullable ByteBuffer buffer) { 3097 if (buffer != null /* && buffer.isDirect() */) { 3098 // all of our ByteBuffers are direct 3099 java.nio.NioUtils.freeDirectBuffer(buffer); 3100 } 3101 } 3102 freeByteBuffers(@ullable ByteBuffer[] buffers)3103 private final void freeByteBuffers(@Nullable ByteBuffer[] buffers) { 3104 if (buffers != null) { 3105 for (ByteBuffer buffer: buffers) { 3106 freeByteBuffer(buffer); 3107 } 3108 } 3109 } 3110 freeAllTrackedBuffers()3111 private final void freeAllTrackedBuffers() { 3112 synchronized(mBufferLock) { 3113 freeByteBuffers(mCachedInputBuffers); 3114 freeByteBuffers(mCachedOutputBuffers); 3115 mCachedInputBuffers = null; 3116 mCachedOutputBuffers = null; 3117 mDequeuedInputBuffers.clear(); 3118 mDequeuedOutputBuffers.clear(); 3119 } 3120 } 3121 cacheBuffers(boolean input)3122 private final void cacheBuffers(boolean input) { 3123 ByteBuffer[] buffers = null; 3124 try { 3125 buffers = getBuffers(input); 3126 invalidateByteBuffers(buffers); 3127 } catch (IllegalStateException e) { 3128 // we don't get buffers in async mode 3129 } 3130 if (input) { 3131 mCachedInputBuffers = buffers; 3132 } else { 3133 mCachedOutputBuffers = buffers; 3134 } 3135 } 3136 3137 /** 3138 * Retrieve the set of input buffers. Call this after start() 3139 * returns. After calling this method, any ByteBuffers 3140 * previously returned by an earlier call to this method MUST no 3141 * longer be used. 3142 * 3143 * @deprecated Use the new {@link #getInputBuffer} method instead 3144 * each time an input buffer is dequeued. 3145 * 3146 * <b>Note:</b> As of API 21, dequeued input buffers are 3147 * automatically {@link java.nio.Buffer#clear cleared}. 3148 * 3149 * <em>Do not use this method if using an input surface.</em> 3150 * 3151 * @throws IllegalStateException if not in the Executing state, 3152 * or codec is configured in asynchronous mode. 3153 * @throws MediaCodec.CodecException upon codec error. 3154 */ 3155 @NonNull getInputBuffers()3156 public ByteBuffer[] getInputBuffers() { 3157 if (mCachedInputBuffers == null) { 3158 throw new IllegalStateException(); 3159 } 3160 // FIXME: check codec status 3161 return mCachedInputBuffers; 3162 } 3163 3164 /** 3165 * Retrieve the set of output buffers. Call this after start() 3166 * returns and whenever dequeueOutputBuffer signals an output 3167 * buffer change by returning {@link 3168 * #INFO_OUTPUT_BUFFERS_CHANGED}. After calling this method, any 3169 * ByteBuffers previously returned by an earlier call to this 3170 * method MUST no longer be used. 3171 * 3172 * @deprecated Use the new {@link #getOutputBuffer} method instead 3173 * each time an output buffer is dequeued. This method is not 3174 * supported if codec is configured in asynchronous mode. 3175 * 3176 * <b>Note:</b> As of API 21, the position and limit of output 3177 * buffers that are dequeued will be set to the valid data 3178 * range. 3179 * 3180 * <em>Do not use this method if using an output surface.</em> 3181 * 3182 * @throws IllegalStateException if not in the Executing state, 3183 * or codec is configured in asynchronous mode. 3184 * @throws MediaCodec.CodecException upon codec error. 3185 */ 3186 @NonNull getOutputBuffers()3187 public ByteBuffer[] getOutputBuffers() { 3188 if (mCachedOutputBuffers == null) { 3189 throw new IllegalStateException(); 3190 } 3191 // FIXME: check codec status 3192 return mCachedOutputBuffers; 3193 } 3194 3195 /** 3196 * Returns a {@link java.nio.Buffer#clear cleared}, writable ByteBuffer 3197 * object for a dequeued input buffer index to contain the input data. 3198 * 3199 * After calling this method any ByteBuffer or Image object 3200 * previously returned for the same input index MUST no longer 3201 * be used. 3202 * 3203 * @param index The index of a client-owned input buffer previously 3204 * returned from a call to {@link #dequeueInputBuffer}, 3205 * or received via an onInputBufferAvailable callback. 3206 * 3207 * @return the input buffer, or null if the index is not a dequeued 3208 * input buffer, or if the codec is configured for surface input. 3209 * 3210 * @throws IllegalStateException if not in the Executing state. 3211 * @throws MediaCodec.CodecException upon codec error. 3212 */ 3213 @Nullable getInputBuffer(int index)3214 public ByteBuffer getInputBuffer(int index) { 3215 ByteBuffer newBuffer = getBuffer(true /* input */, index); 3216 synchronized(mBufferLock) { 3217 invalidateByteBuffer(mCachedInputBuffers, index); 3218 mDequeuedInputBuffers.put(index, newBuffer); 3219 } 3220 return newBuffer; 3221 } 3222 3223 /** 3224 * Returns a writable Image object for a dequeued input buffer 3225 * index to contain the raw input video frame. 3226 * 3227 * After calling this method any ByteBuffer or Image object 3228 * previously returned for the same input index MUST no longer 3229 * be used. 3230 * 3231 * @param index The index of a client-owned input buffer previously 3232 * returned from a call to {@link #dequeueInputBuffer}, 3233 * or received via an onInputBufferAvailable callback. 3234 * 3235 * @return the input image, or null if the index is not a 3236 * dequeued input buffer, or not a ByteBuffer that contains a 3237 * raw image. 3238 * 3239 * @throws IllegalStateException if not in the Executing state. 3240 * @throws MediaCodec.CodecException upon codec error. 3241 */ 3242 @Nullable getInputImage(int index)3243 public Image getInputImage(int index) { 3244 Image newImage = getImage(true /* input */, index); 3245 synchronized(mBufferLock) { 3246 invalidateByteBuffer(mCachedInputBuffers, index); 3247 mDequeuedInputBuffers.put(index, newImage); 3248 } 3249 return newImage; 3250 } 3251 3252 /** 3253 * Returns a read-only ByteBuffer for a dequeued output buffer 3254 * index. The position and limit of the returned buffer are set 3255 * to the valid output data. 3256 * 3257 * After calling this method, any ByteBuffer or Image object 3258 * previously returned for the same output index MUST no longer 3259 * be used. 3260 * 3261 * @param index The index of a client-owned output buffer previously 3262 * returned from a call to {@link #dequeueOutputBuffer}, 3263 * or received via an onOutputBufferAvailable callback. 3264 * 3265 * @return the output buffer, or null if the index is not a dequeued 3266 * output buffer, or the codec is configured with an output surface. 3267 * 3268 * @throws IllegalStateException if not in the Executing state. 3269 * @throws MediaCodec.CodecException upon codec error. 3270 */ 3271 @Nullable getOutputBuffer(int index)3272 public ByteBuffer getOutputBuffer(int index) { 3273 ByteBuffer newBuffer = getBuffer(false /* input */, index); 3274 synchronized(mBufferLock) { 3275 invalidateByteBuffer(mCachedOutputBuffers, index); 3276 mDequeuedOutputBuffers.put(index, newBuffer); 3277 } 3278 return newBuffer; 3279 } 3280 3281 /** 3282 * Returns a read-only Image object for a dequeued output buffer 3283 * index that contains the raw video frame. 3284 * 3285 * After calling this method, any ByteBuffer or Image object previously 3286 * returned for the same output index MUST no longer be used. 3287 * 3288 * @param index The index of a client-owned output buffer previously 3289 * returned from a call to {@link #dequeueOutputBuffer}, 3290 * or received via an onOutputBufferAvailable callback. 3291 * 3292 * @return the output image, or null if the index is not a 3293 * dequeued output buffer, not a raw video frame, or if the codec 3294 * was configured with an output surface. 3295 * 3296 * @throws IllegalStateException if not in the Executing state. 3297 * @throws MediaCodec.CodecException upon codec error. 3298 */ 3299 @Nullable getOutputImage(int index)3300 public Image getOutputImage(int index) { 3301 Image newImage = getImage(false /* input */, index); 3302 synchronized(mBufferLock) { 3303 invalidateByteBuffer(mCachedOutputBuffers, index); 3304 mDequeuedOutputBuffers.put(index, newImage); 3305 } 3306 return newImage; 3307 } 3308 3309 /** 3310 * The content is scaled to the surface dimensions 3311 */ 3312 public static final int VIDEO_SCALING_MODE_SCALE_TO_FIT = 1; 3313 3314 /** 3315 * The content is scaled, maintaining its aspect ratio, the whole 3316 * surface area is used, content may be cropped. 3317 * <p class=note> 3318 * This mode is only suitable for content with 1:1 pixel aspect ratio as you cannot 3319 * configure the pixel aspect ratio for a {@link Surface}. 3320 * <p class=note> 3321 * As of {@link android.os.Build.VERSION_CODES#N} release, this mode may not work if 3322 * the video is {@linkplain MediaFormat#KEY_ROTATION rotated} by 90 or 270 degrees. 3323 */ 3324 public static final int VIDEO_SCALING_MODE_SCALE_TO_FIT_WITH_CROPPING = 2; 3325 3326 /** @hide */ 3327 @IntDef({ 3328 VIDEO_SCALING_MODE_SCALE_TO_FIT, 3329 VIDEO_SCALING_MODE_SCALE_TO_FIT_WITH_CROPPING, 3330 }) 3331 @Retention(RetentionPolicy.SOURCE) 3332 public @interface VideoScalingMode {} 3333 3334 /** 3335 * If a surface has been specified in a previous call to {@link #configure} 3336 * specifies the scaling mode to use. The default is "scale to fit". 3337 * <p class=note> 3338 * The scaling mode may be reset to the <strong>default</strong> each time an 3339 * {@link #INFO_OUTPUT_BUFFERS_CHANGED} event is received from the codec; therefore, the client 3340 * must call this method after every buffer change event (and before the first output buffer is 3341 * released for rendering) to ensure consistent scaling mode. 3342 * <p class=note> 3343 * Since the {@link #INFO_OUTPUT_BUFFERS_CHANGED} event is deprecated, this can also be done 3344 * after each {@link #INFO_OUTPUT_FORMAT_CHANGED} event. 3345 * 3346 * @throws IllegalArgumentException if mode is not recognized. 3347 * @throws IllegalStateException if in the Released state. 3348 */ setVideoScalingMode(@ideoScalingMode int mode)3349 public native final void setVideoScalingMode(@VideoScalingMode int mode); 3350 3351 /** 3352 * Sets the audio presentation. 3353 * @param presentation see {@link AudioPresentation}. In particular, id should be set. 3354 */ setAudioPresentation(@onNull AudioPresentation presentation)3355 public void setAudioPresentation(@NonNull AudioPresentation presentation) { 3356 if (presentation == null) { 3357 throw new NullPointerException("audio presentation is null"); 3358 } 3359 native_setAudioPresentation(presentation.getPresentationId(), presentation.getProgramId()); 3360 } 3361 native_setAudioPresentation(int presentationId, int programId)3362 private native void native_setAudioPresentation(int presentationId, int programId); 3363 3364 /** 3365 * Retrieve the codec name. 3366 * 3367 * If the codec was created by createDecoderByType or createEncoderByType, what component is 3368 * chosen is not known beforehand. This method returns the name of the codec that was 3369 * selected by the platform. 3370 * 3371 * <strong>Note:</strong> Implementations may provide multiple aliases (codec 3372 * names) for the same underlying codec, any of which can be used to instantiate the same 3373 * underlying codec in {@link MediaCodec#createByCodecName}. This method returns the 3374 * name used to create the codec in this case. 3375 * 3376 * @throws IllegalStateException if in the Released state. 3377 */ 3378 @NonNull getName()3379 public final String getName() { 3380 // get canonical name to handle exception 3381 String canonicalName = getCanonicalName(); 3382 return mNameAtCreation != null ? mNameAtCreation : canonicalName; 3383 } 3384 3385 /** 3386 * Retrieve the underlying codec name. 3387 * 3388 * This method is similar to {@link #getName}, except that it returns the underlying component 3389 * name even if an alias was used to create this MediaCodec object by name, 3390 * 3391 * @throws IllegalStateException if in the Released state. 3392 */ 3393 @NonNull getCanonicalName()3394 public native final String getCanonicalName(); 3395 3396 /** 3397 * Return Metrics data about the current codec instance. 3398 * 3399 * @return a {@link PersistableBundle} containing the set of attributes and values 3400 * available for the media being handled by this instance of MediaCodec 3401 * The attributes are descibed in {@link MetricsConstants}. 3402 * 3403 * Additional vendor-specific fields may also be present in 3404 * the return value. 3405 */ getMetrics()3406 public PersistableBundle getMetrics() { 3407 PersistableBundle bundle = native_getMetrics(); 3408 return bundle; 3409 } 3410 native_getMetrics()3411 private native PersistableBundle native_getMetrics(); 3412 3413 /** 3414 * Change a video encoder's target bitrate on the fly. The value is an 3415 * Integer object containing the new bitrate in bps. 3416 * 3417 * @see #setParameters(Bundle) 3418 */ 3419 public static final String PARAMETER_KEY_VIDEO_BITRATE = "video-bitrate"; 3420 3421 /** 3422 * Temporarily suspend/resume encoding of input data. While suspended 3423 * input data is effectively discarded instead of being fed into the 3424 * encoder. This parameter really only makes sense to use with an encoder 3425 * in "surface-input" mode, as the client code has no control over the 3426 * input-side of the encoder in that case. 3427 * The value is an Integer object containing the value 1 to suspend 3428 * or the value 0 to resume. 3429 * 3430 * @see #setParameters(Bundle) 3431 */ 3432 public static final String PARAMETER_KEY_SUSPEND = "drop-input-frames"; 3433 3434 /** 3435 * When {@link #PARAMETER_KEY_SUSPEND} is present, the client can also 3436 * optionally use this key to specify the timestamp (in micro-second) 3437 * at which the suspend/resume operation takes effect. 3438 * 3439 * Note that the specified timestamp must be greater than or equal to the 3440 * timestamp of any previously queued suspend/resume operations. 3441 * 3442 * The value is a long int, indicating the timestamp to suspend/resume. 3443 * 3444 * @see #setParameters(Bundle) 3445 */ 3446 public static final String PARAMETER_KEY_SUSPEND_TIME = "drop-start-time-us"; 3447 3448 /** 3449 * Specify an offset (in micro-second) to be added on top of the timestamps 3450 * onward. A typical use case is to apply an adjust to the timestamps after 3451 * a period of pause by the user. 3452 * 3453 * This parameter can only be used on an encoder in "surface-input" mode. 3454 * 3455 * The value is a long int, indicating the timestamp offset to be applied. 3456 * 3457 * @see #setParameters(Bundle) 3458 */ 3459 public static final String PARAMETER_KEY_OFFSET_TIME = "time-offset-us"; 3460 3461 /** 3462 * Request that the encoder produce a sync frame "soon". 3463 * Provide an Integer with the value 0. 3464 * 3465 * @see #setParameters(Bundle) 3466 */ 3467 public static final String PARAMETER_KEY_REQUEST_SYNC_FRAME = "request-sync"; 3468 3469 /** 3470 * Set the HDR10+ metadata on the next queued input frame. 3471 * 3472 * Provide a byte array of data that's conforming to the 3473 * user_data_registered_itu_t_t35() syntax of SEI message for ST 2094-40. 3474 *<p> 3475 * For decoders: 3476 *<p> 3477 * When a decoder is configured for one of the HDR10+ profiles that uses 3478 * out-of-band metadata (such as {@link 3479 * MediaCodecInfo.CodecProfileLevel#VP9Profile2HDR10Plus} or {@link 3480 * MediaCodecInfo.CodecProfileLevel#VP9Profile3HDR10Plus}), this 3481 * parameter sets the HDR10+ metadata on the next input buffer queued 3482 * to the decoder. A decoder supporting these profiles must propagate 3483 * the metadata to the format of the output buffer corresponding to this 3484 * particular input buffer (under key {@link MediaFormat#KEY_HDR10_PLUS_INFO}). 3485 * The metadata should be applied to that output buffer and the buffers 3486 * following it (in display order), until the next output buffer (in 3487 * display order) upon which an HDR10+ metadata is set. 3488 *<p> 3489 * This parameter shouldn't be set if the decoder is not configured for 3490 * an HDR10+ profile that uses out-of-band metadata. In particular, 3491 * it shouldn't be set for HDR10+ profiles that uses in-band metadata 3492 * where the metadata is embedded in the input buffers, for example 3493 * {@link MediaCodecInfo.CodecProfileLevel#HEVCProfileMain10HDR10Plus}. 3494 *<p> 3495 * For encoders: 3496 *<p> 3497 * When an encoder is configured for one of the HDR10+ profiles and the 3498 * operates in byte buffer input mode (instead of surface input mode), 3499 * this parameter sets the HDR10+ metadata on the next input buffer queued 3500 * to the encoder. For the HDR10+ profiles that uses out-of-band metadata 3501 * (such as {@link MediaCodecInfo.CodecProfileLevel#VP9Profile2HDR10Plus}, 3502 * or {@link MediaCodecInfo.CodecProfileLevel#VP9Profile3HDR10Plus}), 3503 * the metadata must be propagated to the format of the output buffer 3504 * corresponding to this particular input buffer (under key {@link 3505 * MediaFormat#KEY_HDR10_PLUS_INFO}). For the HDR10+ profiles that uses 3506 * in-band metadata (such as {@link 3507 * MediaCodecInfo.CodecProfileLevel#HEVCProfileMain10HDR10Plus}), the 3508 * metadata info must be embedded in the corresponding output buffer itself. 3509 *<p> 3510 * This parameter shouldn't be set if the encoder is not configured for 3511 * an HDR10+ profile, or if it's operating in surface input mode. 3512 *<p> 3513 * 3514 * @see MediaFormat#KEY_HDR10_PLUS_INFO 3515 */ 3516 public static final String PARAMETER_KEY_HDR10_PLUS_INFO = MediaFormat.KEY_HDR10_PLUS_INFO; 3517 3518 /** 3519 * Communicate additional parameter changes to the component instance. 3520 * <b>Note:</b> Some of these parameter changes may silently fail to apply. 3521 * 3522 * @param params The bundle of parameters to set. 3523 * @throws IllegalStateException if in the Released state. 3524 */ setParameters(@ullable Bundle params)3525 public final void setParameters(@Nullable Bundle params) { 3526 if (params == null) { 3527 return; 3528 } 3529 3530 String[] keys = new String[params.size()]; 3531 Object[] values = new Object[params.size()]; 3532 3533 int i = 0; 3534 for (final String key: params.keySet()) { 3535 keys[i] = key; 3536 Object value = params.get(key); 3537 3538 // Bundle's byte array is a byte[], JNI layer only takes ByteBuffer 3539 if (value instanceof byte[]) { 3540 values[i] = ByteBuffer.wrap((byte[])value); 3541 } else { 3542 values[i] = value; 3543 } 3544 ++i; 3545 } 3546 3547 setParameters(keys, values); 3548 } 3549 3550 /** 3551 * Sets an asynchronous callback for actionable MediaCodec events. 3552 * 3553 * If the client intends to use the component in asynchronous mode, 3554 * a valid callback should be provided before {@link #configure} is called. 3555 * 3556 * When asynchronous callback is enabled, the client should not call 3557 * {@link #getInputBuffers}, {@link #getOutputBuffers}, 3558 * {@link #dequeueInputBuffer(long)} or {@link #dequeueOutputBuffer(BufferInfo, long)}. 3559 * <p> 3560 * Also, {@link #flush} behaves differently in asynchronous mode. After calling 3561 * {@code flush}, you must call {@link #start} to "resume" receiving input buffers, 3562 * even if an input surface was created. 3563 * 3564 * @param cb The callback that will run. Use {@code null} to clear a previously 3565 * set callback (before {@link #configure configure} is called and run 3566 * in synchronous mode). 3567 * @param handler Callbacks will happen on the handler's thread. If {@code null}, 3568 * callbacks are done on the default thread (the caller's thread or the 3569 * main thread.) 3570 */ setCallback(@ullable Callback cb, @Nullable Handler handler)3571 public void setCallback(@Nullable /* MediaCodec. */ Callback cb, @Nullable Handler handler) { 3572 if (cb != null) { 3573 synchronized (mListenerLock) { 3574 EventHandler newHandler = getEventHandlerOn(handler, mCallbackHandler); 3575 // NOTE: there are no callbacks on the handler at this time, but check anyways 3576 // even if we were to extend this to be callable dynamically, it must 3577 // be called when codec is flushed, so no messages are pending. 3578 if (newHandler != mCallbackHandler) { 3579 mCallbackHandler.removeMessages(EVENT_SET_CALLBACK); 3580 mCallbackHandler.removeMessages(EVENT_CALLBACK); 3581 mCallbackHandler = newHandler; 3582 } 3583 } 3584 } else if (mCallbackHandler != null) { 3585 mCallbackHandler.removeMessages(EVENT_SET_CALLBACK); 3586 mCallbackHandler.removeMessages(EVENT_CALLBACK); 3587 } 3588 3589 if (mCallbackHandler != null) { 3590 // set java callback on main handler 3591 Message msg = mCallbackHandler.obtainMessage(EVENT_SET_CALLBACK, 0, 0, cb); 3592 mCallbackHandler.sendMessage(msg); 3593 3594 // set native handler here, don't post to handler because 3595 // it may cause the callback to be delayed and set in a wrong state. 3596 // Note that native codec may start sending events to the callback 3597 // handler after this returns. 3598 native_setCallback(cb); 3599 } 3600 } 3601 3602 /** 3603 * Sets an asynchronous callback for actionable MediaCodec events on the default 3604 * looper. 3605 * <p> 3606 * Same as {@link #setCallback(Callback, Handler)} with handler set to null. 3607 * @param cb The callback that will run. Use {@code null} to clear a previously 3608 * set callback (before {@link #configure configure} is called and run 3609 * in synchronous mode). 3610 * @see #setCallback(Callback, Handler) 3611 */ setCallback(@ullable Callback cb)3612 public void setCallback(@Nullable /* MediaCodec. */ Callback cb) { 3613 setCallback(cb, null /* handler */); 3614 } 3615 3616 /** 3617 * Listener to be called when an output frame has rendered on the output surface 3618 * 3619 * @see MediaCodec#setOnFrameRenderedListener 3620 */ 3621 public interface OnFrameRenderedListener { 3622 3623 /** 3624 * Called when an output frame has rendered on the output surface. 3625 * <p> 3626 * <strong>Note:</strong> This callback is for informational purposes only: to get precise 3627 * render timing samples, and can be significantly delayed and batched. Some frames may have 3628 * been rendered even if there was no callback generated. 3629 * 3630 * @param codec the MediaCodec instance 3631 * @param presentationTimeUs the presentation time (media time) of the frame rendered. 3632 * This is usually the same as specified in {@link #queueInputBuffer}; however, 3633 * some codecs may alter the media time by applying some time-based transformation, 3634 * such as frame rate conversion. In that case, presentation time corresponds 3635 * to the actual output frame rendered. 3636 * @param nanoTime The system time when the frame was rendered. 3637 * 3638 * @see System#nanoTime 3639 */ onFrameRendered( @onNull MediaCodec codec, long presentationTimeUs, long nanoTime)3640 public void onFrameRendered( 3641 @NonNull MediaCodec codec, long presentationTimeUs, long nanoTime); 3642 } 3643 3644 /** 3645 * Registers a callback to be invoked when an output frame is rendered on the output surface. 3646 * <p> 3647 * This method can be called in any codec state, but will only have an effect in the 3648 * Executing state for codecs that render buffers to the output surface. 3649 * <p> 3650 * <strong>Note:</strong> This callback is for informational purposes only: to get precise 3651 * render timing samples, and can be significantly delayed and batched. Some frames may have 3652 * been rendered even if there was no callback generated. 3653 * 3654 * @param listener the callback that will be run 3655 * @param handler the callback will be run on the handler's thread. If {@code null}, 3656 * the callback will be run on the default thread, which is the looper 3657 * from which the codec was created, or a new thread if there was none. 3658 */ setOnFrameRenderedListener( @ullable OnFrameRenderedListener listener, @Nullable Handler handler)3659 public void setOnFrameRenderedListener( 3660 @Nullable OnFrameRenderedListener listener, @Nullable Handler handler) { 3661 synchronized (mListenerLock) { 3662 mOnFrameRenderedListener = listener; 3663 if (listener != null) { 3664 EventHandler newHandler = getEventHandlerOn(handler, mOnFrameRenderedHandler); 3665 if (newHandler != mOnFrameRenderedHandler) { 3666 mOnFrameRenderedHandler.removeMessages(EVENT_FRAME_RENDERED); 3667 } 3668 mOnFrameRenderedHandler = newHandler; 3669 } else if (mOnFrameRenderedHandler != null) { 3670 mOnFrameRenderedHandler.removeMessages(EVENT_FRAME_RENDERED); 3671 } 3672 native_enableOnFrameRenderedListener(listener != null); 3673 } 3674 } 3675 native_enableOnFrameRenderedListener(boolean enable)3676 private native void native_enableOnFrameRenderedListener(boolean enable); 3677 getEventHandlerOn( @ullable Handler handler, @NonNull EventHandler lastHandler)3678 private EventHandler getEventHandlerOn( 3679 @Nullable Handler handler, @NonNull EventHandler lastHandler) { 3680 if (handler == null) { 3681 return mEventHandler; 3682 } else { 3683 Looper looper = handler.getLooper(); 3684 if (lastHandler.getLooper() == looper) { 3685 return lastHandler; 3686 } else { 3687 return new EventHandler(this, looper); 3688 } 3689 } 3690 } 3691 3692 /** 3693 * MediaCodec callback interface. Used to notify the user asynchronously 3694 * of various MediaCodec events. 3695 */ 3696 public static abstract class Callback { 3697 /** 3698 * Called when an input buffer becomes available. 3699 * 3700 * @param codec The MediaCodec object. 3701 * @param index The index of the available input buffer. 3702 */ onInputBufferAvailable(@onNull MediaCodec codec, int index)3703 public abstract void onInputBufferAvailable(@NonNull MediaCodec codec, int index); 3704 3705 /** 3706 * Called when an output buffer becomes available. 3707 * 3708 * @param codec The MediaCodec object. 3709 * @param index The index of the available output buffer. 3710 * @param info Info regarding the available output buffer {@link MediaCodec.BufferInfo}. 3711 */ onOutputBufferAvailable( @onNull MediaCodec codec, int index, @NonNull BufferInfo info)3712 public abstract void onOutputBufferAvailable( 3713 @NonNull MediaCodec codec, int index, @NonNull BufferInfo info); 3714 3715 /** 3716 * Called when the MediaCodec encountered an error 3717 * 3718 * @param codec The MediaCodec object. 3719 * @param e The {@link MediaCodec.CodecException} object describing the error. 3720 */ onError(@onNull MediaCodec codec, @NonNull CodecException e)3721 public abstract void onError(@NonNull MediaCodec codec, @NonNull CodecException e); 3722 3723 /** 3724 * Called when the output format has changed 3725 * 3726 * @param codec The MediaCodec object. 3727 * @param format The new output format. 3728 */ onOutputFormatChanged( @onNull MediaCodec codec, @NonNull MediaFormat format)3729 public abstract void onOutputFormatChanged( 3730 @NonNull MediaCodec codec, @NonNull MediaFormat format); 3731 } 3732 postEventFromNative( int what, int arg1, int arg2, @Nullable Object obj)3733 private void postEventFromNative( 3734 int what, int arg1, int arg2, @Nullable Object obj) { 3735 synchronized (mListenerLock) { 3736 EventHandler handler = mEventHandler; 3737 if (what == EVENT_CALLBACK) { 3738 handler = mCallbackHandler; 3739 } else if (what == EVENT_FRAME_RENDERED) { 3740 handler = mOnFrameRenderedHandler; 3741 } 3742 if (handler != null) { 3743 Message msg = handler.obtainMessage(what, arg1, arg2, obj); 3744 handler.sendMessage(msg); 3745 } 3746 } 3747 } 3748 3749 @UnsupportedAppUsage setParameters(@onNull String[] keys, @NonNull Object[] values)3750 private native final void setParameters(@NonNull String[] keys, @NonNull Object[] values); 3751 3752 /** 3753 * Get the codec info. If the codec was created by createDecoderByType 3754 * or createEncoderByType, what component is chosen is not known beforehand, 3755 * and thus the caller does not have the MediaCodecInfo. 3756 * @throws IllegalStateException if in the Released state. 3757 */ 3758 @NonNull getCodecInfo()3759 public MediaCodecInfo getCodecInfo() { 3760 // Get the codec name first. If the codec is already released, 3761 // IllegalStateException will be thrown here. 3762 String name = getName(); 3763 synchronized (mCodecInfoLock) { 3764 if (mCodecInfo == null) { 3765 // Get the codec info for this codec itself first. Only initialize 3766 // the full codec list if this somehow fails because it can be slow. 3767 mCodecInfo = getOwnCodecInfo(); 3768 if (mCodecInfo == null) { 3769 mCodecInfo = MediaCodecList.getInfoFor(name); 3770 } 3771 } 3772 return mCodecInfo; 3773 } 3774 } 3775 3776 @NonNull getOwnCodecInfo()3777 private native final MediaCodecInfo getOwnCodecInfo(); 3778 3779 @NonNull 3780 @UnsupportedAppUsage getBuffers(boolean input)3781 private native final ByteBuffer[] getBuffers(boolean input); 3782 3783 @Nullable getBuffer(boolean input, int index)3784 private native final ByteBuffer getBuffer(boolean input, int index); 3785 3786 @Nullable getImage(boolean input, int index)3787 private native final Image getImage(boolean input, int index); 3788 native_init()3789 private static native final void native_init(); 3790 native_setup( @onNull String name, boolean nameIsType, boolean encoder)3791 private native final void native_setup( 3792 @NonNull String name, boolean nameIsType, boolean encoder); 3793 native_finalize()3794 private native final void native_finalize(); 3795 3796 static { 3797 System.loadLibrary("media_jni"); native_init()3798 native_init(); 3799 } 3800 3801 @UnsupportedAppUsage(maxTargetSdk = Build.VERSION_CODES.P, trackingBug = 115609023) 3802 private long mNativeContext = 0; 3803 private final Lock mNativeContextLock = new ReentrantLock(); 3804 lockAndGetContext()3805 private final long lockAndGetContext() { 3806 mNativeContextLock.lock(); 3807 return mNativeContext; 3808 } 3809 setAndUnlockContext(long context)3810 private final void setAndUnlockContext(long context) { 3811 mNativeContext = context; 3812 mNativeContextLock.unlock(); 3813 } 3814 3815 /** @hide */ 3816 public static class MediaImage extends Image { 3817 private final boolean mIsReadOnly; 3818 private final int mWidth; 3819 private final int mHeight; 3820 private final int mFormat; 3821 private long mTimestamp; 3822 private final Plane[] mPlanes; 3823 private final ByteBuffer mBuffer; 3824 private final ByteBuffer mInfo; 3825 private final int mXOffset; 3826 private final int mYOffset; 3827 3828 private final static int TYPE_YUV = 1; 3829 3830 private final int mTransform = 0; //Default no transform 3831 private final int mScalingMode = 0; //Default frozen scaling mode 3832 3833 @Override getFormat()3834 public int getFormat() { 3835 throwISEIfImageIsInvalid(); 3836 return mFormat; 3837 } 3838 3839 @Override getHeight()3840 public int getHeight() { 3841 throwISEIfImageIsInvalid(); 3842 return mHeight; 3843 } 3844 3845 @Override getWidth()3846 public int getWidth() { 3847 throwISEIfImageIsInvalid(); 3848 return mWidth; 3849 } 3850 3851 @Override getTransform()3852 public int getTransform() { 3853 throwISEIfImageIsInvalid(); 3854 return mTransform; 3855 } 3856 3857 @Override getScalingMode()3858 public int getScalingMode() { 3859 throwISEIfImageIsInvalid(); 3860 return mScalingMode; 3861 } 3862 3863 @Override getTimestamp()3864 public long getTimestamp() { 3865 throwISEIfImageIsInvalid(); 3866 return mTimestamp; 3867 } 3868 3869 @Override 3870 @NonNull getPlanes()3871 public Plane[] getPlanes() { 3872 throwISEIfImageIsInvalid(); 3873 return Arrays.copyOf(mPlanes, mPlanes.length); 3874 } 3875 3876 @Override close()3877 public void close() { 3878 if (mIsImageValid) { 3879 java.nio.NioUtils.freeDirectBuffer(mBuffer); 3880 mIsImageValid = false; 3881 } 3882 } 3883 3884 /** 3885 * Set the crop rectangle associated with this frame. 3886 * <p> 3887 * The crop rectangle specifies the region of valid pixels in the image, 3888 * using coordinates in the largest-resolution plane. 3889 */ 3890 @Override setCropRect(@ullable Rect cropRect)3891 public void setCropRect(@Nullable Rect cropRect) { 3892 if (mIsReadOnly) { 3893 throw new ReadOnlyBufferException(); 3894 } 3895 super.setCropRect(cropRect); 3896 } 3897 3898 MediaImage( @onNull ByteBuffer buffer, @NonNull ByteBuffer info, boolean readOnly, long timestamp, int xOffset, int yOffset, @Nullable Rect cropRect)3899 public MediaImage( 3900 @NonNull ByteBuffer buffer, @NonNull ByteBuffer info, boolean readOnly, 3901 long timestamp, int xOffset, int yOffset, @Nullable Rect cropRect) { 3902 mFormat = ImageFormat.YUV_420_888; 3903 mTimestamp = timestamp; 3904 mIsImageValid = true; 3905 mIsReadOnly = buffer.isReadOnly(); 3906 mBuffer = buffer.duplicate(); 3907 3908 // save offsets and info 3909 mXOffset = xOffset; 3910 mYOffset = yOffset; 3911 mInfo = info; 3912 3913 // read media-info. See MediaImage2 3914 if (info.remaining() == 104) { 3915 int type = info.getInt(); 3916 if (type != TYPE_YUV) { 3917 throw new UnsupportedOperationException("unsupported type: " + type); 3918 } 3919 int numPlanes = info.getInt(); 3920 if (numPlanes != 3) { 3921 throw new RuntimeException("unexpected number of planes: " + numPlanes); 3922 } 3923 mWidth = info.getInt(); 3924 mHeight = info.getInt(); 3925 if (mWidth < 1 || mHeight < 1) { 3926 throw new UnsupportedOperationException( 3927 "unsupported size: " + mWidth + "x" + mHeight); 3928 } 3929 int bitDepth = info.getInt(); 3930 if (bitDepth != 8) { 3931 throw new UnsupportedOperationException("unsupported bit depth: " + bitDepth); 3932 } 3933 int bitDepthAllocated = info.getInt(); 3934 if (bitDepthAllocated != 8) { 3935 throw new UnsupportedOperationException( 3936 "unsupported allocated bit depth: " + bitDepthAllocated); 3937 } 3938 mPlanes = new MediaPlane[numPlanes]; 3939 for (int ix = 0; ix < numPlanes; ix++) { 3940 int planeOffset = info.getInt(); 3941 int colInc = info.getInt(); 3942 int rowInc = info.getInt(); 3943 int horiz = info.getInt(); 3944 int vert = info.getInt(); 3945 if (horiz != vert || horiz != (ix == 0 ? 1 : 2)) { 3946 throw new UnsupportedOperationException("unexpected subsampling: " 3947 + horiz + "x" + vert + " on plane " + ix); 3948 } 3949 if (colInc < 1 || rowInc < 1) { 3950 throw new UnsupportedOperationException("unexpected strides: " 3951 + colInc + " pixel, " + rowInc + " row on plane " + ix); 3952 } 3953 3954 buffer.clear(); 3955 buffer.position(mBuffer.position() + planeOffset 3956 + (xOffset / horiz) * colInc + (yOffset / vert) * rowInc); 3957 buffer.limit(buffer.position() + Utils.divUp(bitDepth, 8) 3958 + (mHeight / vert - 1) * rowInc + (mWidth / horiz - 1) * colInc); 3959 mPlanes[ix] = new MediaPlane(buffer.slice(), rowInc, colInc); 3960 } 3961 } else { 3962 throw new UnsupportedOperationException( 3963 "unsupported info length: " + info.remaining()); 3964 } 3965 3966 if (cropRect == null) { 3967 cropRect = new Rect(0, 0, mWidth, mHeight); 3968 } 3969 cropRect.offset(-xOffset, -yOffset); 3970 super.setCropRect(cropRect); 3971 } 3972 3973 private class MediaPlane extends Plane { MediaPlane(@onNull ByteBuffer buffer, int rowInc, int colInc)3974 public MediaPlane(@NonNull ByteBuffer buffer, int rowInc, int colInc) { 3975 mData = buffer; 3976 mRowInc = rowInc; 3977 mColInc = colInc; 3978 } 3979 3980 @Override getRowStride()3981 public int getRowStride() { 3982 throwISEIfImageIsInvalid(); 3983 return mRowInc; 3984 } 3985 3986 @Override getPixelStride()3987 public int getPixelStride() { 3988 throwISEIfImageIsInvalid(); 3989 return mColInc; 3990 } 3991 3992 @Override 3993 @NonNull getBuffer()3994 public ByteBuffer getBuffer() { 3995 throwISEIfImageIsInvalid(); 3996 return mData; 3997 } 3998 3999 private final int mRowInc; 4000 private final int mColInc; 4001 private final ByteBuffer mData; 4002 } 4003 } 4004 4005 public final static class MetricsConstants 4006 { MetricsConstants()4007 private MetricsConstants() {} 4008 4009 /** 4010 * Key to extract the codec being used 4011 * from the {@link MediaCodec#getMetrics} return value. 4012 * The value is a String. 4013 */ 4014 public static final String CODEC = "android.media.mediacodec.codec"; 4015 4016 /** 4017 * Key to extract the MIME type 4018 * from the {@link MediaCodec#getMetrics} return value. 4019 * The value is a String. 4020 */ 4021 public static final String MIME_TYPE = "android.media.mediacodec.mime"; 4022 4023 /** 4024 * Key to extract what the codec mode 4025 * from the {@link MediaCodec#getMetrics} return value. 4026 * The value is a String. Values will be one of the constants 4027 * {@link #MODE_AUDIO} or {@link #MODE_VIDEO}. 4028 */ 4029 public static final String MODE = "android.media.mediacodec.mode"; 4030 4031 /** 4032 * The value returned for the key {@link #MODE} when the 4033 * codec is a audio codec. 4034 */ 4035 public static final String MODE_AUDIO = "audio"; 4036 4037 /** 4038 * The value returned for the key {@link #MODE} when the 4039 * codec is a video codec. 4040 */ 4041 public static final String MODE_VIDEO = "video"; 4042 4043 /** 4044 * Key to extract the flag indicating whether the codec is running 4045 * as an encoder or decoder from the {@link MediaCodec#getMetrics} return value. 4046 * The value is an integer. 4047 * A 0 indicates decoder; 1 indicates encoder. 4048 */ 4049 public static final String ENCODER = "android.media.mediacodec.encoder"; 4050 4051 /** 4052 * Key to extract the flag indicating whether the codec is running 4053 * in secure (DRM) mode from the {@link MediaCodec#getMetrics} return value. 4054 * The value is an integer. 4055 */ 4056 public static final String SECURE = "android.media.mediacodec.secure"; 4057 4058 /** 4059 * Key to extract the width (in pixels) of the video track 4060 * from the {@link MediaCodec#getMetrics} return value. 4061 * The value is an integer. 4062 */ 4063 public static final String WIDTH = "android.media.mediacodec.width"; 4064 4065 /** 4066 * Key to extract the height (in pixels) of the video track 4067 * from the {@link MediaCodec#getMetrics} return value. 4068 * The value is an integer. 4069 */ 4070 public static final String HEIGHT = "android.media.mediacodec.height"; 4071 4072 /** 4073 * Key to extract the rotation (in degrees) to properly orient the video 4074 * from the {@link MediaCodec#getMetrics} return. 4075 * The value is a integer. 4076 */ 4077 public static final String ROTATION = "android.media.mediacodec.rotation"; 4078 4079 } 4080 } 4081
