1 /* 2 * Copyright (C) 2015 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package android.media; 18 19 import android.annotation.IntDef; 20 import android.annotation.NonNull; 21 import android.annotation.Nullable; 22 import android.media.AudioTrack; 23 import android.media.PlaybackParams; 24 import android.os.Handler; 25 import android.os.Looper; 26 import android.os.Message; 27 import android.util.Log; 28 import android.view.Surface; 29 30 import java.lang.annotation.Retention; 31 import java.lang.annotation.RetentionPolicy; 32 import java.nio.ByteBuffer; 33 import java.util.concurrent.TimeUnit; 34 import java.util.LinkedList; 35 import java.util.List; 36 37 /** 38 * MediaSync class can be used to synchronously play audio and video streams. 39 * It can be used to play audio-only or video-only stream, too. 40 * 41 * <p>MediaSync is generally used like this: 42 * <pre> 43 * MediaSync sync = new MediaSync(); 44 * sync.setSurface(surface); 45 * Surface inputSurface = sync.createInputSurface(); 46 * ... 47 * // MediaCodec videoDecoder = ...; 48 * videoDecoder.configure(format, inputSurface, ...); 49 * ... 50 * sync.setAudioTrack(audioTrack); 51 * sync.setCallback(new MediaSync.Callback() { 52 * {@literal @Override} 53 * public void onAudioBufferConsumed(MediaSync sync, ByteBuffer audioBuffer, int bufferId) { 54 * ... 55 * } 56 * }, null); 57 * // This needs to be done since sync is paused on creation. 58 * sync.setPlaybackParams(new PlaybackParams().setSpeed(1.f)); 59 * 60 * for (;;) { 61 * ... 62 * // send video frames to surface for rendering, e.g., call 63 * // videoDecoder.releaseOutputBuffer(videoOutputBufferIx, videoPresentationTimeNs); 64 * // More details are available as below. 65 * ... 66 * sync.queueAudio(audioByteBuffer, bufferId, audioPresentationTimeUs); // non-blocking. 67 * // The audioByteBuffer and bufferId will be returned via callback. 68 * // More details are available as below. 69 * ... 70 * ... 71 * } 72 * sync.setPlaybackParams(new PlaybackParams().setSpeed(0.f)); 73 * sync.release(); 74 * sync = null; 75 * 76 * // The following code snippet illustrates how video/audio raw frames are created by 77 * // MediaCodec's, how they are fed to MediaSync and how they are returned by MediaSync. 78 * // This is the callback from MediaCodec. 79 * onOutputBufferAvailable(MediaCodec codec, int bufferId, BufferInfo info) { 80 * // ... 81 * if (codec == videoDecoder) { 82 * // surface timestamp must contain media presentation time in nanoseconds. 83 * codec.releaseOutputBuffer(bufferId, 1000 * info.presentationTime); 84 * } else { 85 * ByteBuffer audioByteBuffer = codec.getOutputBuffer(bufferId); 86 * sync.queueAudio(audioByteBuffer, bufferId, info.presentationTime); 87 * } 88 * // ... 89 * } 90 * 91 * // This is the callback from MediaSync. 92 * onAudioBufferConsumed(MediaSync sync, ByteBuffer buffer, int bufferId) { 93 * // ... 94 * audioDecoder.releaseBuffer(bufferId, false); 95 * // ... 96 * } 97 * 98 * </pre> 99 * 100 * The client needs to configure corresponding sink by setting the Surface and/or AudioTrack 101 * based on the stream type it will play. 102 * <p> 103 * For video, the client needs to call {@link #createInputSurface} to obtain a surface on 104 * which it will render video frames. 105 * <p> 106 * For audio, the client needs to set up audio track correctly, e.g., using {@link 107 * AudioTrack#MODE_STREAM}. The audio buffers are sent to MediaSync directly via {@link 108 * #queueAudio}, and are returned to the client via {@link Callback#onAudioBufferConsumed} 109 * asynchronously. The client should not modify an audio buffer till it's returned. 110 * <p> 111 * The client can optionally pre-fill audio/video buffers by setting playback rate to 0.0, 112 * and then feed audio/video buffers to corresponding components. This can reduce possible 113 * initial underrun. 114 * <p> 115 */ 116 public final class MediaSync { 117 /** 118 * MediaSync callback interface. Used to notify the user asynchronously 119 * of various MediaSync events. 120 */ 121 public static abstract class Callback { 122 /** 123 * Called when returning an audio buffer which has been consumed. 124 * 125 * @param sync The MediaSync object. 126 * @param audioBuffer The returned audio buffer. 127 * @param bufferId The ID associated with audioBuffer as passed into 128 * {@link MediaSync#queueAudio}. 129 */ onAudioBufferConsumed( @onNull MediaSync sync, @NonNull ByteBuffer audioBuffer, int bufferId)130 public abstract void onAudioBufferConsumed( 131 @NonNull MediaSync sync, @NonNull ByteBuffer audioBuffer, int bufferId); 132 } 133 134 /** Audio track failed. 135 * @see android.media.MediaSync.OnErrorListener 136 */ 137 public static final int MEDIASYNC_ERROR_AUDIOTRACK_FAIL = 1; 138 139 /** The surface failed to handle video buffers. 140 * @see android.media.MediaSync.OnErrorListener 141 */ 142 public static final int MEDIASYNC_ERROR_SURFACE_FAIL = 2; 143 144 /** 145 * Interface definition of a callback to be invoked when there 146 * has been an error during an asynchronous operation (other errors 147 * will throw exceptions at method call time). 148 */ 149 public interface OnErrorListener { 150 /** 151 * Called to indicate an error. 152 * 153 * @param sync The MediaSync the error pertains to 154 * @param what The type of error that has occurred: 155 * <ul> 156 * <li>{@link #MEDIASYNC_ERROR_AUDIOTRACK_FAIL} 157 * <li>{@link #MEDIASYNC_ERROR_SURFACE_FAIL} 158 * </ul> 159 * @param extra an extra code, specific to the error. Typically 160 * implementation dependent. 161 */ onError(@onNull MediaSync sync, int what, int extra)162 void onError(@NonNull MediaSync sync, int what, int extra); 163 } 164 165 private static final String TAG = "MediaSync"; 166 167 private static final int EVENT_CALLBACK = 1; 168 private static final int EVENT_SET_CALLBACK = 2; 169 170 private static final int CB_RETURN_AUDIO_BUFFER = 1; 171 172 private static class AudioBuffer { 173 public ByteBuffer mByteBuffer; 174 public int mBufferIndex; 175 long mPresentationTimeUs; 176 AudioBuffer(@onNull ByteBuffer byteBuffer, int bufferId, long presentationTimeUs)177 public AudioBuffer(@NonNull ByteBuffer byteBuffer, int bufferId, 178 long presentationTimeUs) { 179 mByteBuffer = byteBuffer; 180 mBufferIndex = bufferId; 181 mPresentationTimeUs = presentationTimeUs; 182 } 183 } 184 185 private final Object mCallbackLock = new Object(); 186 private Handler mCallbackHandler = null; 187 private MediaSync.Callback mCallback = null; 188 189 private final Object mOnErrorListenerLock = new Object(); 190 private Handler mOnErrorListenerHandler = null; 191 private MediaSync.OnErrorListener mOnErrorListener = null; 192 193 private Thread mAudioThread = null; 194 // Created on mAudioThread when mAudioThread is started. When used on user thread, they should 195 // be guarded by checking mAudioThread. 196 private Handler mAudioHandler = null; 197 private Looper mAudioLooper = null; 198 199 private final Object mAudioLock = new Object(); 200 private AudioTrack mAudioTrack = null; 201 private List<AudioBuffer> mAudioBuffers = new LinkedList<AudioBuffer>(); 202 // this is only used for paused/running decisions, so it is not affected by clock drift 203 private float mPlaybackRate = 0.0f; 204 205 private long mNativeContext; 206 207 /** 208 * Class constructor. On creation, MediaSync is paused, i.e., playback rate is 0.0f. 209 */ MediaSync()210 public MediaSync() { 211 native_setup(); 212 } 213 native_setup()214 private native final void native_setup(); 215 216 @Override finalize()217 protected void finalize() { 218 native_finalize(); 219 } 220 native_finalize()221 private native final void native_finalize(); 222 223 /** 224 * Make sure you call this when you're done to free up any opened 225 * component instance instead of relying on the garbage collector 226 * to do this for you at some point in the future. 227 */ release()228 public final void release() { 229 returnAudioBuffers(); 230 if (mAudioThread != null) { 231 if (mAudioLooper != null) { 232 mAudioLooper.quit(); 233 } 234 } 235 setCallback(null, null); 236 native_release(); 237 } 238 native_release()239 private native final void native_release(); 240 241 /** 242 * Sets an asynchronous callback for actionable MediaSync events. 243 * <p> 244 * This method can be called multiple times to update a previously set callback. If the 245 * handler is changed, undelivered notifications scheduled for the old handler may be dropped. 246 * <p> 247 * <b>Do not call this inside callback.</b> 248 * 249 * @param cb The callback that will run. Use {@code null} to stop receiving callbacks. 250 * @param handler The Handler that will run the callback. Use {@code null} to use MediaSync's 251 * internal handler if it exists. 252 */ setCallback(@ullable Callback cb, @Nullable Handler handler)253 public void setCallback(@Nullable /* MediaSync. */ Callback cb, @Nullable Handler handler) { 254 synchronized(mCallbackLock) { 255 if (handler != null) { 256 mCallbackHandler = handler; 257 } else { 258 Looper looper; 259 if ((looper = Looper.myLooper()) == null) { 260 looper = Looper.getMainLooper(); 261 } 262 if (looper == null) { 263 mCallbackHandler = null; 264 } else { 265 mCallbackHandler = new Handler(looper); 266 } 267 } 268 269 mCallback = cb; 270 } 271 } 272 273 /** 274 * Sets an asynchronous callback for error events. 275 * <p> 276 * This method can be called multiple times to update a previously set listener. If the 277 * handler is changed, undelivered notifications scheduled for the old handler may be dropped. 278 * <p> 279 * <b>Do not call this inside callback.</b> 280 * 281 * @param listener The callback that will run. Use {@code null} to stop receiving callbacks. 282 * @param handler The Handler that will run the callback. Use {@code null} to use MediaSync's 283 * internal handler if it exists. 284 */ setOnErrorListener(@ullable OnErrorListener listener, @Nullable Handler handler)285 public void setOnErrorListener(@Nullable /* MediaSync. */ OnErrorListener listener, 286 @Nullable Handler handler) { 287 synchronized(mOnErrorListenerLock) { 288 if (handler != null) { 289 mOnErrorListenerHandler = handler; 290 } else { 291 Looper looper; 292 if ((looper = Looper.myLooper()) == null) { 293 looper = Looper.getMainLooper(); 294 } 295 if (looper == null) { 296 mOnErrorListenerHandler = null; 297 } else { 298 mOnErrorListenerHandler = new Handler(looper); 299 } 300 } 301 302 mOnErrorListener = listener; 303 } 304 } 305 306 /** 307 * Sets the output surface for MediaSync. 308 * <p> 309 * Currently, this is only supported in the Initialized state. 310 * 311 * @param surface Specify a surface on which to render the video data. 312 * @throws IllegalArgumentException if the surface has been released, is invalid, 313 * or can not be connected. 314 * @throws IllegalStateException if setting the surface is not supported, e.g. 315 * not in the Initialized state, or another surface has already been set. 316 */ setSurface(@ullable Surface surface)317 public void setSurface(@Nullable Surface surface) { 318 native_setSurface(surface); 319 } 320 native_setSurface(@ullable Surface surface)321 private native final void native_setSurface(@Nullable Surface surface); 322 323 /** 324 * Sets the audio track for MediaSync. 325 * <p> 326 * Currently, this is only supported in the Initialized state. 327 * 328 * @param audioTrack Specify an AudioTrack through which to render the audio data. 329 * @throws IllegalArgumentException if the audioTrack has been released, or is invalid. 330 * @throws IllegalStateException if setting the audio track is not supported, e.g. 331 * not in the Initialized state, or another audio track has already been set. 332 */ setAudioTrack(@ullable AudioTrack audioTrack)333 public void setAudioTrack(@Nullable AudioTrack audioTrack) { 334 native_setAudioTrack(audioTrack); 335 mAudioTrack = audioTrack; 336 if (audioTrack != null && mAudioThread == null) { 337 createAudioThread(); 338 } 339 } 340 native_setAudioTrack(@ullable AudioTrack audioTrack)341 private native final void native_setAudioTrack(@Nullable AudioTrack audioTrack); 342 343 /** 344 * Requests a Surface to use as the input. This may only be called after 345 * {@link #setSurface}. 346 * <p> 347 * The application is responsible for calling release() on the Surface when 348 * done. 349 * @throws IllegalStateException if not set, or another input surface has 350 * already been created. 351 */ 352 @NonNull createInputSurface()353 public native final Surface createInputSurface(); 354 355 /** 356 * Sets playback rate using {@link PlaybackParams}. 357 * <p> 358 * When using MediaSync with {@link AudioTrack}, set playback params using this 359 * call instead of calling it directly on the track, so that the sync is aware of 360 * the params change. 361 * <p> 362 * This call also works if there is no audio track. 363 * 364 * @param params the playback params to use. {@link PlaybackParams#getSpeed 365 * Speed} is the ratio between desired playback rate and normal one. 1.0 means 366 * normal playback speed. 0.0 means pause. Value larger than 1.0 means faster playback, 367 * while value between 0.0 and 1.0 for slower playback. <b>Note:</b> the normal rate 368 * does not change as a result of this call. To restore the original rate at any time, 369 * use speed of 1.0. 370 * 371 * @throws IllegalStateException if the internal sync engine or the audio track has not 372 * been initialized. 373 * @throws IllegalArgumentException if the params are not supported. 374 */ setPlaybackParams(@onNull PlaybackParams params)375 public void setPlaybackParams(@NonNull PlaybackParams params) { 376 synchronized(mAudioLock) { 377 mPlaybackRate = native_setPlaybackParams(params);; 378 } 379 if (mPlaybackRate != 0.0 && mAudioThread != null) { 380 postRenderAudio(0); 381 } 382 } 383 384 /** 385 * Gets the playback rate using {@link PlaybackParams}. 386 * 387 * @return the playback rate being used. 388 * 389 * @throws IllegalStateException if the internal sync engine or the audio track has not 390 * been initialized. 391 */ 392 @NonNull getPlaybackParams()393 public native PlaybackParams getPlaybackParams(); 394 native_setPlaybackParams(@onNull PlaybackParams params)395 private native float native_setPlaybackParams(@NonNull PlaybackParams params); 396 397 /** 398 * Sets A/V sync mode. 399 * 400 * @param params the A/V sync params to apply 401 * 402 * @throws IllegalStateException if the internal player engine has not been 403 * initialized. 404 * @throws IllegalArgumentException if params are not supported. 405 */ setSyncParams(@onNull SyncParams params)406 public void setSyncParams(@NonNull SyncParams params) { 407 synchronized(mAudioLock) { 408 mPlaybackRate = native_setSyncParams(params);; 409 } 410 if (mPlaybackRate != 0.0 && mAudioThread != null) { 411 postRenderAudio(0); 412 } 413 } 414 native_setSyncParams(@onNull SyncParams params)415 private native float native_setSyncParams(@NonNull SyncParams params); 416 417 /** 418 * Gets the A/V sync mode. 419 * 420 * @return the A/V sync params 421 * 422 * @throws IllegalStateException if the internal player engine has not been 423 * initialized. 424 */ 425 @NonNull getSyncParams()426 public native SyncParams getSyncParams(); 427 428 /** 429 * Flushes all buffers from the sync object. 430 * <p> 431 * All pending unprocessed audio and video buffers are discarded. If an audio track was 432 * configured, it is flushed and stopped. If a video output surface was configured, the 433 * last frame queued to it is left on the frame. Queue a blank video frame to clear the 434 * surface, 435 * <p> 436 * No callbacks are received for the flushed buffers. 437 * 438 * @throws IllegalStateException if the internal player engine has not been 439 * initialized. 440 */ flush()441 public void flush() { 442 synchronized(mAudioLock) { 443 mAudioBuffers.clear(); 444 mCallbackHandler.removeCallbacksAndMessages(null); 445 } 446 if (mAudioTrack != null) { 447 mAudioTrack.pause(); 448 mAudioTrack.flush(); 449 // Call stop() to signal to the AudioSink to completely fill the 450 // internal buffer before resuming playback. 451 mAudioTrack.stop(); 452 } 453 native_flush(); 454 } 455 native_flush()456 private native final void native_flush(); 457 458 /** 459 * Get current playback position. 460 * <p> 461 * The MediaTimestamp represents how the media time correlates to the system time in 462 * a linear fashion using an anchor and a clock rate. During regular playback, the media 463 * time moves fairly constantly (though the anchor frame may be rebased to a current 464 * system time, the linear correlation stays steady). Therefore, this method does not 465 * need to be called often. 466 * <p> 467 * To help users get current playback position, this method always anchors the timestamp 468 * to the current {@link System#nanoTime system time}, so 469 * {@link MediaTimestamp#getAnchorMediaTimeUs} can be used as current playback position. 470 * 471 * @return a MediaTimestamp object if a timestamp is available, or {@code null} if no timestamp 472 * is available, e.g. because the media player has not been initialized. 473 * 474 * @see MediaTimestamp 475 */ 476 @Nullable getTimestamp()477 public MediaTimestamp getTimestamp() 478 { 479 try { 480 // TODO: create the timestamp in native 481 MediaTimestamp timestamp = new MediaTimestamp(); 482 if (native_getTimestamp(timestamp)) { 483 return timestamp; 484 } else { 485 return null; 486 } 487 } catch (IllegalStateException e) { 488 return null; 489 } 490 } 491 native_getTimestamp(@onNull MediaTimestamp timestamp)492 private native final boolean native_getTimestamp(@NonNull MediaTimestamp timestamp); 493 494 /** 495 * Queues the audio data asynchronously for playback (AudioTrack must be in streaming mode). 496 * If the audio track was flushed as a result of {@link #flush}, it will be restarted. 497 * @param audioData the buffer that holds the data to play. This buffer will be returned 498 * to the client via registered callback. 499 * @param bufferId an integer used to identify audioData. It will be returned to 500 * the client along with audioData. This helps applications to keep track of audioData, 501 * e.g., it can be used to store the output buffer index used by the audio codec. 502 * @param presentationTimeUs the presentation timestamp in microseconds for the first frame 503 * in the buffer. 504 * @throws IllegalStateException if audio track is not set or internal configureation 505 * has not been done correctly. 506 */ queueAudio( @onNull ByteBuffer audioData, int bufferId, long presentationTimeUs)507 public void queueAudio( 508 @NonNull ByteBuffer audioData, int bufferId, long presentationTimeUs) { 509 if (mAudioTrack == null || mAudioThread == null) { 510 throw new IllegalStateException( 511 "AudioTrack is NOT set or audio thread is not created"); 512 } 513 514 synchronized(mAudioLock) { 515 mAudioBuffers.add(new AudioBuffer(audioData, bufferId, presentationTimeUs)); 516 } 517 518 if (mPlaybackRate != 0.0) { 519 postRenderAudio(0); 520 } 521 } 522 523 // When called on user thread, make sure to check mAudioThread != null. postRenderAudio(long delayMillis)524 private void postRenderAudio(long delayMillis) { 525 mAudioHandler.postDelayed(new Runnable() { 526 public void run() { 527 synchronized(mAudioLock) { 528 if (mPlaybackRate == 0.0) { 529 return; 530 } 531 532 if (mAudioBuffers.isEmpty()) { 533 return; 534 } 535 536 AudioBuffer audioBuffer = mAudioBuffers.get(0); 537 int size = audioBuffer.mByteBuffer.remaining(); 538 // restart audio track after flush 539 if (size > 0 && mAudioTrack.getPlayState() != AudioTrack.PLAYSTATE_PLAYING) { 540 try { 541 mAudioTrack.play(); 542 } catch (IllegalStateException e) { 543 Log.w(TAG, "could not start audio track"); 544 } 545 } 546 int sizeWritten = mAudioTrack.write( 547 audioBuffer.mByteBuffer, 548 size, 549 AudioTrack.WRITE_NON_BLOCKING); 550 if (sizeWritten > 0) { 551 if (audioBuffer.mPresentationTimeUs != -1) { 552 native_updateQueuedAudioData( 553 size, audioBuffer.mPresentationTimeUs); 554 audioBuffer.mPresentationTimeUs = -1; 555 } 556 557 if (sizeWritten == size) { 558 postReturnByteBuffer(audioBuffer); 559 mAudioBuffers.remove(0); 560 if (!mAudioBuffers.isEmpty()) { 561 postRenderAudio(0); 562 } 563 return; 564 } 565 } 566 long pendingTimeMs = TimeUnit.MICROSECONDS.toMillis( 567 native_getPlayTimeForPendingAudioFrames()); 568 postRenderAudio(pendingTimeMs / 2); 569 } 570 } 571 }, delayMillis); 572 } 573 native_updateQueuedAudioData( int sizeInBytes, long presentationTimeUs)574 private native final void native_updateQueuedAudioData( 575 int sizeInBytes, long presentationTimeUs); 576 native_getPlayTimeForPendingAudioFrames()577 private native final long native_getPlayTimeForPendingAudioFrames(); 578 postReturnByteBuffer(@onNull final AudioBuffer audioBuffer)579 private final void postReturnByteBuffer(@NonNull final AudioBuffer audioBuffer) { 580 synchronized(mCallbackLock) { 581 if (mCallbackHandler != null) { 582 final MediaSync sync = this; 583 mCallbackHandler.post(new Runnable() { 584 public void run() { 585 Callback callback; 586 synchronized(mCallbackLock) { 587 callback = mCallback; 588 if (mCallbackHandler == null 589 || mCallbackHandler.getLooper().getThread() 590 != Thread.currentThread()) { 591 // callback handler has been changed. 592 return; 593 } 594 } 595 if (callback != null) { 596 callback.onAudioBufferConsumed(sync, audioBuffer.mByteBuffer, 597 audioBuffer.mBufferIndex); 598 } 599 } 600 }); 601 } 602 } 603 } 604 returnAudioBuffers()605 private final void returnAudioBuffers() { 606 synchronized(mAudioLock) { 607 for (AudioBuffer audioBuffer: mAudioBuffers) { 608 postReturnByteBuffer(audioBuffer); 609 } 610 mAudioBuffers.clear(); 611 } 612 } 613 createAudioThread()614 private void createAudioThread() { 615 mAudioThread = new Thread() { 616 @Override 617 public void run() { 618 Looper.prepare(); 619 synchronized(mAudioLock) { 620 mAudioLooper = Looper.myLooper(); 621 mAudioHandler = new Handler(); 622 mAudioLock.notify(); 623 } 624 Looper.loop(); 625 } 626 }; 627 mAudioThread.start(); 628 629 synchronized(mAudioLock) { 630 try { 631 mAudioLock.wait(); 632 } catch(InterruptedException e) { 633 } 634 } 635 } 636 637 static { 638 System.loadLibrary("media_jni"); native_init()639 native_init(); 640 } 641 native_init()642 private static native final void native_init(); 643 } 644