1 /* 2 * Copyright (C) 2014 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package android.media.cts; 17 18 import android.media.AudioTrack; 19 import android.media.MediaCodec; 20 import android.media.MediaExtractor; 21 import android.media.MediaFormat; 22 import android.os.Handler; 23 import android.os.Looper; 24 import android.util.Log; 25 import android.view.Surface; 26 import java.nio.ByteBuffer; 27 import java.util.LinkedList; 28 29 /** 30 * Class for directly managing both audio and video playback by 31 * using {@link MediaCodec} and {@link AudioTrack}. 32 */ 33 public class CodecState { 34 private static final String TAG = CodecState.class.getSimpleName(); 35 36 private boolean mSawInputEOS; 37 private volatile boolean mSawOutputEOS; 38 private boolean mLimitQueueDepth; 39 private boolean mTunneled; 40 private boolean mIsAudio; 41 private int mAudioSessionId; 42 private ByteBuffer[] mCodecInputBuffers; 43 private ByteBuffer[] mCodecOutputBuffers; 44 private int mTrackIndex; 45 private LinkedList<Integer> mAvailableInputBufferIndices; 46 private LinkedList<Integer> mAvailableOutputBufferIndices; 47 private LinkedList<MediaCodec.BufferInfo> mAvailableOutputBufferInfos; 48 private volatile long mPresentationTimeUs; 49 private long mSampleBaseTimeUs; 50 private MediaCodec mCodec; 51 private MediaTimeProvider mMediaTimeProvider; 52 private MediaExtractor mExtractor; 53 private MediaFormat mFormat; 54 private MediaFormat mOutputFormat; 55 private NonBlockingAudioTrack mAudioTrack; 56 private volatile OnFrameRenderedListener mOnFrameRenderedListener; 57 58 /** 59 * Manages audio and video playback using MediaCodec and AudioTrack. 60 */ CodecState( MediaTimeProvider mediaTimeProvider, MediaExtractor extractor, int trackIndex, MediaFormat format, MediaCodec codec, boolean limitQueueDepth, boolean tunneled, int audioSessionId)61 public CodecState( 62 MediaTimeProvider mediaTimeProvider, 63 MediaExtractor extractor, 64 int trackIndex, 65 MediaFormat format, 66 MediaCodec codec, 67 boolean limitQueueDepth, 68 boolean tunneled, 69 int audioSessionId) { 70 mMediaTimeProvider = mediaTimeProvider; 71 mExtractor = extractor; 72 mTrackIndex = trackIndex; 73 mFormat = format; 74 mSawInputEOS = mSawOutputEOS = false; 75 mLimitQueueDepth = limitQueueDepth; 76 mTunneled = tunneled; 77 mAudioSessionId = audioSessionId; 78 mSampleBaseTimeUs = -1; 79 80 mCodec = codec; 81 82 mAvailableInputBufferIndices = new LinkedList<Integer>(); 83 mAvailableOutputBufferIndices = new LinkedList<Integer>(); 84 mAvailableOutputBufferInfos = new LinkedList<MediaCodec.BufferInfo>(); 85 86 mPresentationTimeUs = 0; 87 88 String mime = mFormat.getString(MediaFormat.KEY_MIME); 89 Log.d(TAG, "CodecState::CodecState " + mime); 90 mIsAudio = mime.startsWith("audio/"); 91 92 if (mTunneled && !mIsAudio) { 93 mOnFrameRenderedListener = new OnFrameRenderedListener(); 94 codec.setOnFrameRenderedListener(mOnFrameRenderedListener, 95 new Handler(Looper.getMainLooper())); 96 } 97 } 98 release()99 public void release() { 100 mCodec.stop(); 101 mCodecInputBuffers = null; 102 mCodecOutputBuffers = null; 103 mOutputFormat = null; 104 105 mAvailableInputBufferIndices.clear(); 106 mAvailableOutputBufferIndices.clear(); 107 mAvailableOutputBufferInfos.clear(); 108 109 mAvailableInputBufferIndices = null; 110 mAvailableOutputBufferIndices = null; 111 mAvailableOutputBufferInfos = null; 112 113 if (mOnFrameRenderedListener != null) { 114 mCodec.setOnFrameRenderedListener(null, null); 115 mOnFrameRenderedListener = null; 116 } 117 118 mCodec.release(); 119 mCodec = null; 120 121 if (mAudioTrack != null) { 122 mAudioTrack.release(); 123 mAudioTrack = null; 124 } 125 } 126 start()127 public void start() { 128 mCodec.start(); 129 mCodecInputBuffers = mCodec.getInputBuffers(); 130 if (!mTunneled || mIsAudio) { 131 mCodecOutputBuffers = mCodec.getOutputBuffers(); 132 } 133 134 if (mAudioTrack != null) { 135 mAudioTrack.play(); 136 } 137 } 138 pause()139 public void pause() { 140 if (mAudioTrack != null) { 141 mAudioTrack.pause(); 142 } 143 } 144 getCurrentPositionUs()145 public long getCurrentPositionUs() { 146 return mPresentationTimeUs; 147 } 148 flush()149 public void flush() { 150 mAvailableInputBufferIndices.clear(); 151 if (!mTunneled || mIsAudio) { 152 mAvailableOutputBufferIndices.clear(); 153 mAvailableOutputBufferInfos.clear(); 154 } 155 156 mSawInputEOS = false; 157 mSawOutputEOS = false; 158 159 if (mAudioTrack != null 160 && mAudioTrack.getPlayState() != AudioTrack.PLAYSTATE_PLAYING) { 161 mAudioTrack.flush(); 162 } 163 164 mCodec.flush(); 165 } 166 isEnded()167 public boolean isEnded() { 168 return mSawInputEOS && mSawOutputEOS; 169 } 170 171 /** 172 * doSomeWork() is the worker function that does all buffer handling and decoding works. 173 * It first reads data from {@link MediaExtractor} and pushes it into {@link MediaCodec}; 174 * it then dequeues buffer from {@link MediaCodec}, consumes it and pushes back to its own 175 * buffer queue for next round reading data from {@link MediaExtractor}. 176 */ doSomeWork()177 public void doSomeWork() { 178 int indexInput = mCodec.dequeueInputBuffer(0 /* timeoutUs */); 179 180 if (indexInput != MediaCodec.INFO_TRY_AGAIN_LATER) { 181 mAvailableInputBufferIndices.add(indexInput); 182 } 183 184 while (feedInputBuffer()) { 185 } 186 187 if (mIsAudio || !mTunneled) { 188 MediaCodec.BufferInfo info = new MediaCodec.BufferInfo(); 189 int indexOutput = mCodec.dequeueOutputBuffer(info, 0 /* timeoutUs */); 190 191 if (indexOutput == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) { 192 mOutputFormat = mCodec.getOutputFormat(); 193 onOutputFormatChanged(); 194 } else if (indexOutput == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) { 195 mCodecOutputBuffers = mCodec.getOutputBuffers(); 196 } else if (indexOutput != MediaCodec.INFO_TRY_AGAIN_LATER) { 197 mAvailableOutputBufferIndices.add(indexOutput); 198 mAvailableOutputBufferInfos.add(info); 199 } 200 201 while (drainOutputBuffer()) { 202 } 203 } 204 } 205 206 /** Returns true if more input data could be fed. */ feedInputBuffer()207 private boolean feedInputBuffer() throws MediaCodec.CryptoException, IllegalStateException { 208 if (mSawInputEOS || mAvailableInputBufferIndices.isEmpty()) { 209 return false; 210 } 211 212 // stalls read if audio queue is larger than 2MB full so we will not occupy too much heap 213 if (mLimitQueueDepth && mAudioTrack != null && 214 mAudioTrack.getNumBytesQueued() > 2 * 1024 * 1024) { 215 return false; 216 } 217 218 int index = mAvailableInputBufferIndices.peekFirst().intValue(); 219 220 ByteBuffer codecData = mCodecInputBuffers[index]; 221 222 int trackIndex = mExtractor.getSampleTrackIndex(); 223 224 if (trackIndex == mTrackIndex) { 225 int sampleSize = 226 mExtractor.readSampleData(codecData, 0 /* offset */); 227 228 long sampleTime = mExtractor.getSampleTime(); 229 230 int sampleFlags = mExtractor.getSampleFlags(); 231 232 if (sampleSize <= 0) { 233 Log.d(TAG, "sampleSize: " + sampleSize + " trackIndex:" + trackIndex + 234 " sampleTime:" + sampleTime + " sampleFlags:" + sampleFlags); 235 mSawInputEOS = true; 236 return false; 237 } 238 239 if (mTunneled && !mIsAudio) { 240 if (mSampleBaseTimeUs == -1) { 241 mSampleBaseTimeUs = sampleTime; 242 } 243 sampleTime -= mSampleBaseTimeUs; 244 } 245 246 if ((sampleFlags & MediaExtractor.SAMPLE_FLAG_ENCRYPTED) != 0) { 247 MediaCodec.CryptoInfo info = new MediaCodec.CryptoInfo(); 248 mExtractor.getSampleCryptoInfo(info); 249 250 mCodec.queueSecureInputBuffer( 251 index, 0 /* offset */, info, sampleTime, 0 /* flags */); 252 } else { 253 mCodec.queueInputBuffer( 254 index, 0 /* offset */, sampleSize, sampleTime, 0 /* flags */); 255 } 256 257 mAvailableInputBufferIndices.removeFirst(); 258 mExtractor.advance(); 259 260 return true; 261 } else if (trackIndex < 0) { 262 Log.d(TAG, "saw input EOS on track " + mTrackIndex); 263 264 mSawInputEOS = true; 265 266 mCodec.queueInputBuffer( 267 index, 0 /* offset */, 0 /* sampleSize */, 268 0 /* sampleTime */, MediaCodec.BUFFER_FLAG_END_OF_STREAM); 269 270 mAvailableInputBufferIndices.removeFirst(); 271 } 272 273 return false; 274 } 275 onOutputFormatChanged()276 private void onOutputFormatChanged() { 277 String mime = mOutputFormat.getString(MediaFormat.KEY_MIME); 278 // b/9250789 279 Log.d(TAG, "CodecState::onOutputFormatChanged " + mime); 280 281 mIsAudio = false; 282 if (mime.startsWith("audio/")) { 283 mIsAudio = true; 284 int sampleRate = 285 mOutputFormat.getInteger(MediaFormat.KEY_SAMPLE_RATE); 286 287 int channelCount = 288 mOutputFormat.getInteger(MediaFormat.KEY_CHANNEL_COUNT); 289 290 Log.d(TAG, "CodecState::onOutputFormatChanged Audio" + 291 " sampleRate:" + sampleRate + " channels:" + channelCount); 292 // We do a check here after we receive data from MediaExtractor and before 293 // we pass them down to AudioTrack. If MediaExtractor works properly, this 294 // check is not necessary, however, in our tests, we found that there 295 // are a few cases where ch=0 and samplerate=0 were returned by MediaExtractor. 296 if (channelCount < 1 || channelCount > 8 || 297 sampleRate < 8000 || sampleRate > 128000) { 298 return; 299 } 300 mAudioTrack = new NonBlockingAudioTrack(sampleRate, channelCount, 301 mTunneled, mAudioSessionId); 302 mAudioTrack.play(); 303 } 304 305 if (mime.startsWith("video/")) { 306 int width = mOutputFormat.getInteger(MediaFormat.KEY_WIDTH); 307 int height = mOutputFormat.getInteger(MediaFormat.KEY_HEIGHT); 308 Log.d(TAG, "CodecState::onOutputFormatChanged Video" + 309 " width:" + width + " height:" + height); 310 } 311 } 312 313 /** Returns true if more output data could be drained. */ drainOutputBuffer()314 private boolean drainOutputBuffer() { 315 if (mSawOutputEOS || mAvailableOutputBufferIndices.isEmpty()) { 316 return false; 317 } 318 319 int index = mAvailableOutputBufferIndices.peekFirst().intValue(); 320 MediaCodec.BufferInfo info = mAvailableOutputBufferInfos.peekFirst(); 321 322 if ((info.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) { 323 Log.d(TAG, "saw output EOS on track " + mTrackIndex); 324 325 mSawOutputEOS = true; 326 327 // Do not stop audio track here. Video presentation may not finish 328 // yet, stopping the auido track now would result in getAudioTimeUs 329 // returning 0 and prevent video samples from being presented. 330 // We stop the audio track before the playback thread exits. 331 return false; 332 } 333 334 long realTimeUs = 335 mMediaTimeProvider.getRealTimeUsForMediaTime(info.presentationTimeUs); 336 337 long nowUs = mMediaTimeProvider.getNowUs(); 338 339 long lateUs = nowUs - realTimeUs; 340 341 if (mAudioTrack != null) { 342 ByteBuffer buffer = mCodecOutputBuffers[index]; 343 byte[] audioArray = new byte[info.size]; 344 buffer.get(audioArray); 345 buffer.clear(); 346 347 mAudioTrack.write(ByteBuffer.wrap(audioArray), info.size, 348 info.presentationTimeUs*1000); 349 350 mCodec.releaseOutputBuffer(index, false /* render */); 351 352 mPresentationTimeUs = info.presentationTimeUs; 353 354 mAvailableOutputBufferIndices.removeFirst(); 355 mAvailableOutputBufferInfos.removeFirst(); 356 return true; 357 } else { 358 // video 359 boolean render; 360 361 if (lateUs < -45000) { 362 // too early; 363 return false; 364 } else if (lateUs > 30000) { 365 Log.d(TAG, "video late by " + lateUs + " us."); 366 render = false; 367 } else { 368 render = true; 369 mPresentationTimeUs = info.presentationTimeUs; 370 } 371 372 mCodec.releaseOutputBuffer(index, render); 373 374 mAvailableOutputBufferIndices.removeFirst(); 375 mAvailableOutputBufferInfos.removeFirst(); 376 return true; 377 } 378 } 379 380 /** Callback called by the renderer in tunneling mode. */ 381 private class OnFrameRenderedListener implements MediaCodec.OnFrameRenderedListener { 382 private static final long TUNNELING_EOS_PRESENTATION_TIME_US = Long.MAX_VALUE; 383 384 @Override onFrameRendered(MediaCodec codec, long presentationTimeUs, long nanoTime)385 public void onFrameRendered(MediaCodec codec, long presentationTimeUs, long nanoTime) { 386 if (this != mOnFrameRenderedListener) { 387 return; // stale event 388 } 389 if (presentationTimeUs == TUNNELING_EOS_PRESENTATION_TIME_US) { 390 mSawOutputEOS = true; 391 } else { 392 mPresentationTimeUs = presentationTimeUs; 393 } 394 } 395 } 396 getAudioTimeUs()397 public long getAudioTimeUs() { 398 if (mAudioTrack == null) { 399 return 0; 400 } 401 402 return mAudioTrack.getAudioTimeUs(); 403 } 404 process()405 public void process() { 406 if (mAudioTrack != null) { 407 mAudioTrack.process(); 408 } 409 } 410 stop()411 public void stop() { 412 if (mAudioTrack != null) { 413 mAudioTrack.stop(); 414 } 415 } 416 setOutputSurface(Surface surface)417 public void setOutputSurface(Surface surface) { 418 if (mAudioTrack != null) { 419 throw new UnsupportedOperationException("Cannot set surface on audio codec"); 420 } 421 mCodec.setOutputSurface(surface); 422 } 423 } 424