1 /*
2 * Copyright (C) 2010 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 //#define LOG_NDEBUG 0
18 #define LOG_TAG "NuPlayerRenderer"
19 #include <utils/Log.h>
20
21 #include "AWakeLock.h"
22 #include "NuPlayerRenderer.h"
23 #include <algorithm>
24 #include <cutils/properties.h>
25 #include <media/stagefright/foundation/ADebug.h>
26 #include <media/stagefright/foundation/AMessage.h>
27 #include <media/stagefright/foundation/AUtils.h>
28 #include <media/stagefright/MediaClock.h>
29 #include <media/stagefright/MediaCodecConstants.h>
30 #include <media/stagefright/MediaDefs.h>
31 #include <media/stagefright/MediaErrors.h>
32 #include <media/stagefright/MetaData.h>
33 #include <media/stagefright/Utils.h>
34 #include <media/stagefright/VideoFrameScheduler.h>
35 #include <media/MediaCodecBuffer.h>
36
37 #include <inttypes.h>
38
39 namespace android {
40
41 /*
42 * Example of common configuration settings in shell script form
43
44 #Turn offload audio off (use PCM for Play Music) -- AudioPolicyManager
45 adb shell setprop audio.offload.disable 1
46
47 #Allow offload audio with video (requires offloading to be enabled) -- AudioPolicyManager
48 adb shell setprop audio.offload.video 1
49
50 #Use audio callbacks for PCM data
51 adb shell setprop media.stagefright.audio.cbk 1
52
53 #Use deep buffer for PCM data with video (it is generally enabled for audio-only)
54 adb shell setprop media.stagefright.audio.deep 1
55
56 #Set size of buffers for pcm audio sink in msec (example: 1000 msec)
57 adb shell setprop media.stagefright.audio.sink 1000
58
59 * These configurations take effect for the next track played (not the current track).
60 */
61
getUseAudioCallbackSetting()62 static inline bool getUseAudioCallbackSetting() {
63 return property_get_bool("media.stagefright.audio.cbk", false /* default_value */);
64 }
65
getAudioSinkPcmMsSetting()66 static inline int32_t getAudioSinkPcmMsSetting() {
67 return property_get_int32(
68 "media.stagefright.audio.sink", 500 /* default_value */);
69 }
70
71 // Maximum time in paused state when offloading audio decompression. When elapsed, the AudioSink
72 // is closed to allow the audio DSP to power down.
73 static const int64_t kOffloadPauseMaxUs = 10000000LL;
74
75 // Maximum allowed delay from AudioSink, 1.5 seconds.
76 static const int64_t kMaxAllowedAudioSinkDelayUs = 1500000LL;
77
78 static const int64_t kMinimumAudioClockUpdatePeriodUs = 20 /* msec */ * 1000;
79
80 // Default video frame display duration when only video exists.
81 // Used to set max media time in MediaClock.
82 static const int64_t kDefaultVideoFrameIntervalUs = 100000LL;
83
84 // static
85 const NuPlayer::Renderer::PcmInfo NuPlayer::Renderer::AUDIO_PCMINFO_INITIALIZER = {
86 AUDIO_CHANNEL_NONE,
87 AUDIO_OUTPUT_FLAG_NONE,
88 AUDIO_FORMAT_INVALID,
89 0, // mNumChannels
90 0 // mSampleRate
91 };
92
93 // static
94 const int64_t NuPlayer::Renderer::kMinPositionUpdateDelayUs = 100000ll;
95
audioFormatFromEncoding(int32_t pcmEncoding)96 static audio_format_t constexpr audioFormatFromEncoding(int32_t pcmEncoding) {
97 switch (pcmEncoding) {
98 case kAudioEncodingPcmFloat:
99 return AUDIO_FORMAT_PCM_FLOAT;
100 case kAudioEncodingPcm16bit:
101 return AUDIO_FORMAT_PCM_16_BIT;
102 case kAudioEncodingPcm8bit:
103 return AUDIO_FORMAT_PCM_8_BIT; // TODO: do we want to support this?
104 default:
105 ALOGE("%s: Invalid encoding: %d", __func__, pcmEncoding);
106 return AUDIO_FORMAT_INVALID;
107 }
108 }
109
Renderer(const sp<MediaPlayerBase::AudioSink> & sink,const sp<MediaClock> & mediaClock,const sp<AMessage> & notify,uint32_t flags)110 NuPlayer::Renderer::Renderer(
111 const sp<MediaPlayerBase::AudioSink> &sink,
112 const sp<MediaClock> &mediaClock,
113 const sp<AMessage> ¬ify,
114 uint32_t flags)
115 : mAudioSink(sink),
116 mUseVirtualAudioSink(false),
117 mNotify(notify),
118 mFlags(flags),
119 mNumFramesWritten(0),
120 mDrainAudioQueuePending(false),
121 mDrainVideoQueuePending(false),
122 mAudioQueueGeneration(0),
123 mVideoQueueGeneration(0),
124 mAudioDrainGeneration(0),
125 mVideoDrainGeneration(0),
126 mAudioEOSGeneration(0),
127 mMediaClock(mediaClock),
128 mPlaybackSettings(AUDIO_PLAYBACK_RATE_DEFAULT),
129 mAudioFirstAnchorTimeMediaUs(-1),
130 mAnchorTimeMediaUs(-1),
131 mAnchorNumFramesWritten(-1),
132 mVideoLateByUs(0LL),
133 mNextVideoTimeMediaUs(-1),
134 mHasAudio(false),
135 mHasVideo(false),
136 mNotifyCompleteAudio(false),
137 mNotifyCompleteVideo(false),
138 mSyncQueues(false),
139 mPaused(false),
140 mPauseDrainAudioAllowedUs(0),
141 mVideoSampleReceived(false),
142 mVideoRenderingStarted(false),
143 mVideoRenderingStartGeneration(0),
144 mAudioRenderingStartGeneration(0),
145 mRenderingDataDelivered(false),
146 mNextAudioClockUpdateTimeUs(-1),
147 mLastAudioMediaTimeUs(-1),
148 mAudioOffloadPauseTimeoutGeneration(0),
149 mAudioTornDown(false),
150 mCurrentOffloadInfo(AUDIO_INFO_INITIALIZER),
151 mCurrentPcmInfo(AUDIO_PCMINFO_INITIALIZER),
152 mTotalBuffersQueued(0),
153 mLastAudioBufferDrained(0),
154 mUseAudioCallback(false),
155 mWakeLock(new AWakeLock()) {
156 CHECK(mediaClock != NULL);
157 mPlaybackRate = mPlaybackSettings.mSpeed;
158 mMediaClock->setPlaybackRate(mPlaybackRate);
159 }
160
~Renderer()161 NuPlayer::Renderer::~Renderer() {
162 if (offloadingAudio()) {
163 mAudioSink->stop();
164 mAudioSink->flush();
165 mAudioSink->close();
166 }
167
168 // Try to avoid racing condition in case callback is still on.
169 Mutex::Autolock autoLock(mLock);
170 if (mUseAudioCallback) {
171 flushQueue(&mAudioQueue);
172 flushQueue(&mVideoQueue);
173 }
174 mWakeLock.clear();
175 mVideoScheduler.clear();
176 mNotify.clear();
177 mAudioSink.clear();
178 }
179
queueBuffer(bool audio,const sp<MediaCodecBuffer> & buffer,const sp<AMessage> & notifyConsumed)180 void NuPlayer::Renderer::queueBuffer(
181 bool audio,
182 const sp<MediaCodecBuffer> &buffer,
183 const sp<AMessage> ¬ifyConsumed) {
184 sp<AMessage> msg = new AMessage(kWhatQueueBuffer, this);
185 msg->setInt32("queueGeneration", getQueueGeneration(audio));
186 msg->setInt32("audio", static_cast<int32_t>(audio));
187 msg->setObject("buffer", buffer);
188 msg->setMessage("notifyConsumed", notifyConsumed);
189 msg->post();
190 }
191
queueEOS(bool audio,status_t finalResult)192 void NuPlayer::Renderer::queueEOS(bool audio, status_t finalResult) {
193 CHECK_NE(finalResult, (status_t)OK);
194
195 sp<AMessage> msg = new AMessage(kWhatQueueEOS, this);
196 msg->setInt32("queueGeneration", getQueueGeneration(audio));
197 msg->setInt32("audio", static_cast<int32_t>(audio));
198 msg->setInt32("finalResult", finalResult);
199 msg->post();
200 }
201
setPlaybackSettings(const AudioPlaybackRate & rate)202 status_t NuPlayer::Renderer::setPlaybackSettings(const AudioPlaybackRate &rate) {
203 sp<AMessage> msg = new AMessage(kWhatConfigPlayback, this);
204 writeToAMessage(msg, rate);
205 sp<AMessage> response;
206 status_t err = msg->postAndAwaitResponse(&response);
207 if (err == OK && response != NULL) {
208 CHECK(response->findInt32("err", &err));
209 }
210 return err;
211 }
212
onConfigPlayback(const AudioPlaybackRate & rate)213 status_t NuPlayer::Renderer::onConfigPlayback(const AudioPlaybackRate &rate /* sanitized */) {
214 if (rate.mSpeed == 0.f) {
215 onPause();
216 // don't call audiosink's setPlaybackRate if pausing, as pitch does not
217 // have to correspond to the any non-0 speed (e.g old speed). Keep
218 // settings nonetheless, using the old speed, in case audiosink changes.
219 AudioPlaybackRate newRate = rate;
220 newRate.mSpeed = mPlaybackSettings.mSpeed;
221 mPlaybackSettings = newRate;
222 return OK;
223 }
224
225 if (mAudioSink != NULL && mAudioSink->ready()) {
226 status_t err = mAudioSink->setPlaybackRate(rate);
227 if (err != OK) {
228 return err;
229 }
230 }
231 mPlaybackSettings = rate;
232 mPlaybackRate = rate.mSpeed;
233 mMediaClock->setPlaybackRate(mPlaybackRate);
234 return OK;
235 }
236
getPlaybackSettings(AudioPlaybackRate * rate)237 status_t NuPlayer::Renderer::getPlaybackSettings(AudioPlaybackRate *rate /* nonnull */) {
238 sp<AMessage> msg = new AMessage(kWhatGetPlaybackSettings, this);
239 sp<AMessage> response;
240 status_t err = msg->postAndAwaitResponse(&response);
241 if (err == OK && response != NULL) {
242 CHECK(response->findInt32("err", &err));
243 if (err == OK) {
244 readFromAMessage(response, rate);
245 }
246 }
247 return err;
248 }
249
onGetPlaybackSettings(AudioPlaybackRate * rate)250 status_t NuPlayer::Renderer::onGetPlaybackSettings(AudioPlaybackRate *rate /* nonnull */) {
251 if (mAudioSink != NULL && mAudioSink->ready()) {
252 status_t err = mAudioSink->getPlaybackRate(rate);
253 if (err == OK) {
254 if (!isAudioPlaybackRateEqual(*rate, mPlaybackSettings)) {
255 ALOGW("correcting mismatch in internal/external playback rate");
256 }
257 // get playback settings used by audiosink, as it may be
258 // slightly off due to audiosink not taking small changes.
259 mPlaybackSettings = *rate;
260 if (mPaused) {
261 rate->mSpeed = 0.f;
262 }
263 }
264 return err;
265 }
266 *rate = mPlaybackSettings;
267 return OK;
268 }
269
setSyncSettings(const AVSyncSettings & sync,float videoFpsHint)270 status_t NuPlayer::Renderer::setSyncSettings(const AVSyncSettings &sync, float videoFpsHint) {
271 sp<AMessage> msg = new AMessage(kWhatConfigSync, this);
272 writeToAMessage(msg, sync, videoFpsHint);
273 sp<AMessage> response;
274 status_t err = msg->postAndAwaitResponse(&response);
275 if (err == OK && response != NULL) {
276 CHECK(response->findInt32("err", &err));
277 }
278 return err;
279 }
280
onConfigSync(const AVSyncSettings & sync,float videoFpsHint __unused)281 status_t NuPlayer::Renderer::onConfigSync(const AVSyncSettings &sync, float videoFpsHint __unused) {
282 if (sync.mSource != AVSYNC_SOURCE_DEFAULT) {
283 return BAD_VALUE;
284 }
285 // TODO: support sync sources
286 return INVALID_OPERATION;
287 }
288
getSyncSettings(AVSyncSettings * sync,float * videoFps)289 status_t NuPlayer::Renderer::getSyncSettings(AVSyncSettings *sync, float *videoFps) {
290 sp<AMessage> msg = new AMessage(kWhatGetSyncSettings, this);
291 sp<AMessage> response;
292 status_t err = msg->postAndAwaitResponse(&response);
293 if (err == OK && response != NULL) {
294 CHECK(response->findInt32("err", &err));
295 if (err == OK) {
296 readFromAMessage(response, sync, videoFps);
297 }
298 }
299 return err;
300 }
301
onGetSyncSettings(AVSyncSettings * sync,float * videoFps)302 status_t NuPlayer::Renderer::onGetSyncSettings(
303 AVSyncSettings *sync /* nonnull */, float *videoFps /* nonnull */) {
304 *sync = mSyncSettings;
305 *videoFps = -1.f;
306 return OK;
307 }
308
flush(bool audio,bool notifyComplete)309 void NuPlayer::Renderer::flush(bool audio, bool notifyComplete) {
310 {
311 Mutex::Autolock autoLock(mLock);
312 if (audio) {
313 mNotifyCompleteAudio |= notifyComplete;
314 clearAudioFirstAnchorTime_l();
315 ++mAudioQueueGeneration;
316 ++mAudioDrainGeneration;
317 } else {
318 mNotifyCompleteVideo |= notifyComplete;
319 ++mVideoQueueGeneration;
320 ++mVideoDrainGeneration;
321 mNextVideoTimeMediaUs = -1;
322 }
323
324 mMediaClock->clearAnchor();
325 mVideoLateByUs = 0;
326 mSyncQueues = false;
327 }
328
329 sp<AMessage> msg = new AMessage(kWhatFlush, this);
330 msg->setInt32("audio", static_cast<int32_t>(audio));
331 msg->post();
332 }
333
signalTimeDiscontinuity()334 void NuPlayer::Renderer::signalTimeDiscontinuity() {
335 }
336
signalDisableOffloadAudio()337 void NuPlayer::Renderer::signalDisableOffloadAudio() {
338 (new AMessage(kWhatDisableOffloadAudio, this))->post();
339 }
340
signalEnableOffloadAudio()341 void NuPlayer::Renderer::signalEnableOffloadAudio() {
342 (new AMessage(kWhatEnableOffloadAudio, this))->post();
343 }
344
pause()345 void NuPlayer::Renderer::pause() {
346 (new AMessage(kWhatPause, this))->post();
347 }
348
resume()349 void NuPlayer::Renderer::resume() {
350 (new AMessage(kWhatResume, this))->post();
351 }
352
setVideoFrameRate(float fps)353 void NuPlayer::Renderer::setVideoFrameRate(float fps) {
354 sp<AMessage> msg = new AMessage(kWhatSetVideoFrameRate, this);
355 msg->setFloat("frame-rate", fps);
356 msg->post();
357 }
358
359 // Called on any threads without mLock acquired.
getCurrentPosition(int64_t * mediaUs)360 status_t NuPlayer::Renderer::getCurrentPosition(int64_t *mediaUs) {
361 status_t result = mMediaClock->getMediaTime(ALooper::GetNowUs(), mediaUs);
362 if (result == OK) {
363 return result;
364 }
365
366 // MediaClock has not started yet. Try to start it if possible.
367 {
368 Mutex::Autolock autoLock(mLock);
369 if (mAudioFirstAnchorTimeMediaUs == -1) {
370 return result;
371 }
372
373 AudioTimestamp ts;
374 status_t res = mAudioSink->getTimestamp(ts);
375 if (res != OK) {
376 return result;
377 }
378
379 // AudioSink has rendered some frames.
380 int64_t nowUs = ALooper::GetNowUs();
381 int64_t playedOutDurationUs = mAudioSink->getPlayedOutDurationUs(nowUs);
382 if (playedOutDurationUs == 0) {
383 *mediaUs = mAudioFirstAnchorTimeMediaUs;
384 return OK;
385 }
386 int64_t nowMediaUs = playedOutDurationUs + mAudioFirstAnchorTimeMediaUs;
387 mMediaClock->updateAnchor(nowMediaUs, nowUs, -1);
388 }
389
390 return mMediaClock->getMediaTime(ALooper::GetNowUs(), mediaUs);
391 }
392
clearAudioFirstAnchorTime_l()393 void NuPlayer::Renderer::clearAudioFirstAnchorTime_l() {
394 mAudioFirstAnchorTimeMediaUs = -1;
395 mMediaClock->setStartingTimeMedia(-1);
396 }
397
setAudioFirstAnchorTimeIfNeeded_l(int64_t mediaUs)398 void NuPlayer::Renderer::setAudioFirstAnchorTimeIfNeeded_l(int64_t mediaUs) {
399 if (mAudioFirstAnchorTimeMediaUs == -1) {
400 mAudioFirstAnchorTimeMediaUs = mediaUs;
401 mMediaClock->setStartingTimeMedia(mediaUs);
402 }
403 }
404
405 // Called on renderer looper.
clearAnchorTime()406 void NuPlayer::Renderer::clearAnchorTime() {
407 mMediaClock->clearAnchor();
408 mAnchorTimeMediaUs = -1;
409 mAnchorNumFramesWritten = -1;
410 }
411
setVideoLateByUs(int64_t lateUs)412 void NuPlayer::Renderer::setVideoLateByUs(int64_t lateUs) {
413 Mutex::Autolock autoLock(mLock);
414 mVideoLateByUs = lateUs;
415 }
416
getVideoLateByUs()417 int64_t NuPlayer::Renderer::getVideoLateByUs() {
418 Mutex::Autolock autoLock(mLock);
419 return mVideoLateByUs;
420 }
421
openAudioSink(const sp<AMessage> & format,bool offloadOnly,bool hasVideo,uint32_t flags,bool * isOffloaded,bool isStreaming)422 status_t NuPlayer::Renderer::openAudioSink(
423 const sp<AMessage> &format,
424 bool offloadOnly,
425 bool hasVideo,
426 uint32_t flags,
427 bool *isOffloaded,
428 bool isStreaming) {
429 sp<AMessage> msg = new AMessage(kWhatOpenAudioSink, this);
430 msg->setMessage("format", format);
431 msg->setInt32("offload-only", offloadOnly);
432 msg->setInt32("has-video", hasVideo);
433 msg->setInt32("flags", flags);
434 msg->setInt32("isStreaming", isStreaming);
435
436 sp<AMessage> response;
437 status_t postStatus = msg->postAndAwaitResponse(&response);
438
439 int32_t err;
440 if (postStatus != OK || response.get() == nullptr || !response->findInt32("err", &err)) {
441 err = INVALID_OPERATION;
442 } else if (err == OK && isOffloaded != NULL) {
443 int32_t offload;
444 CHECK(response->findInt32("offload", &offload));
445 *isOffloaded = (offload != 0);
446 }
447 return err;
448 }
449
closeAudioSink()450 void NuPlayer::Renderer::closeAudioSink() {
451 sp<AMessage> msg = new AMessage(kWhatCloseAudioSink, this);
452
453 sp<AMessage> response;
454 msg->postAndAwaitResponse(&response);
455 }
456
changeAudioFormat(const sp<AMessage> & format,bool offloadOnly,bool hasVideo,uint32_t flags,bool isStreaming,const sp<AMessage> & notify)457 void NuPlayer::Renderer::changeAudioFormat(
458 const sp<AMessage> &format,
459 bool offloadOnly,
460 bool hasVideo,
461 uint32_t flags,
462 bool isStreaming,
463 const sp<AMessage> ¬ify) {
464 sp<AMessage> meta = new AMessage;
465 meta->setMessage("format", format);
466 meta->setInt32("offload-only", offloadOnly);
467 meta->setInt32("has-video", hasVideo);
468 meta->setInt32("flags", flags);
469 meta->setInt32("isStreaming", isStreaming);
470
471 sp<AMessage> msg = new AMessage(kWhatChangeAudioFormat, this);
472 msg->setInt32("queueGeneration", getQueueGeneration(true /* audio */));
473 msg->setMessage("notify", notify);
474 msg->setMessage("meta", meta);
475 msg->post();
476 }
477
onMessageReceived(const sp<AMessage> & msg)478 void NuPlayer::Renderer::onMessageReceived(const sp<AMessage> &msg) {
479 switch (msg->what()) {
480 case kWhatOpenAudioSink:
481 {
482 sp<AMessage> format;
483 CHECK(msg->findMessage("format", &format));
484
485 int32_t offloadOnly;
486 CHECK(msg->findInt32("offload-only", &offloadOnly));
487
488 int32_t hasVideo;
489 CHECK(msg->findInt32("has-video", &hasVideo));
490
491 uint32_t flags;
492 CHECK(msg->findInt32("flags", (int32_t *)&flags));
493
494 uint32_t isStreaming;
495 CHECK(msg->findInt32("isStreaming", (int32_t *)&isStreaming));
496
497 status_t err = onOpenAudioSink(format, offloadOnly, hasVideo, flags, isStreaming);
498
499 sp<AMessage> response = new AMessage;
500 response->setInt32("err", err);
501 response->setInt32("offload", offloadingAudio());
502
503 sp<AReplyToken> replyID;
504 CHECK(msg->senderAwaitsResponse(&replyID));
505 response->postReply(replyID);
506
507 break;
508 }
509
510 case kWhatCloseAudioSink:
511 {
512 sp<AReplyToken> replyID;
513 CHECK(msg->senderAwaitsResponse(&replyID));
514
515 onCloseAudioSink();
516
517 sp<AMessage> response = new AMessage;
518 response->postReply(replyID);
519 break;
520 }
521
522 case kWhatStopAudioSink:
523 {
524 mAudioSink->stop();
525 break;
526 }
527
528 case kWhatChangeAudioFormat:
529 {
530 int32_t queueGeneration;
531 CHECK(msg->findInt32("queueGeneration", &queueGeneration));
532
533 sp<AMessage> notify;
534 CHECK(msg->findMessage("notify", ¬ify));
535
536 if (offloadingAudio()) {
537 ALOGW("changeAudioFormat should NOT be called in offload mode");
538 notify->setInt32("err", INVALID_OPERATION);
539 notify->post();
540 break;
541 }
542
543 sp<AMessage> meta;
544 CHECK(msg->findMessage("meta", &meta));
545
546 if (queueGeneration != getQueueGeneration(true /* audio */)
547 || mAudioQueue.empty()) {
548 onChangeAudioFormat(meta, notify);
549 break;
550 }
551
552 QueueEntry entry;
553 entry.mNotifyConsumed = notify;
554 entry.mMeta = meta;
555
556 Mutex::Autolock autoLock(mLock);
557 mAudioQueue.push_back(entry);
558 postDrainAudioQueue_l();
559
560 break;
561 }
562
563 case kWhatDrainAudioQueue:
564 {
565 mDrainAudioQueuePending = false;
566
567 int32_t generation;
568 CHECK(msg->findInt32("drainGeneration", &generation));
569 if (generation != getDrainGeneration(true /* audio */)) {
570 break;
571 }
572
573 if (onDrainAudioQueue()) {
574 uint32_t numFramesPlayed;
575 CHECK_EQ(mAudioSink->getPosition(&numFramesPlayed),
576 (status_t)OK);
577
578 // Handle AudioTrack race when start is immediately called after flush.
579 uint32_t numFramesPendingPlayout =
580 (mNumFramesWritten > numFramesPlayed ?
581 mNumFramesWritten - numFramesPlayed : 0);
582
583 // This is how long the audio sink will have data to
584 // play back.
585 int64_t delayUs =
586 mAudioSink->msecsPerFrame()
587 * numFramesPendingPlayout * 1000LL;
588 if (mPlaybackRate > 1.0f) {
589 delayUs /= mPlaybackRate;
590 }
591
592 // Let's give it more data after about half that time
593 // has elapsed.
594 delayUs /= 2;
595 // check the buffer size to estimate maximum delay permitted.
596 const int64_t maxDrainDelayUs = std::max(
597 mAudioSink->getBufferDurationInUs(), (int64_t)500000 /* half second */);
598 ALOGD_IF(delayUs > maxDrainDelayUs, "postDrainAudioQueue long delay: %lld > %lld",
599 (long long)delayUs, (long long)maxDrainDelayUs);
600 Mutex::Autolock autoLock(mLock);
601 postDrainAudioQueue_l(delayUs);
602 }
603 break;
604 }
605
606 case kWhatDrainVideoQueue:
607 {
608 int32_t generation;
609 CHECK(msg->findInt32("drainGeneration", &generation));
610 if (generation != getDrainGeneration(false /* audio */)) {
611 break;
612 }
613
614 mDrainVideoQueuePending = false;
615
616 onDrainVideoQueue();
617
618 postDrainVideoQueue();
619 break;
620 }
621
622 case kWhatPostDrainVideoQueue:
623 {
624 int32_t generation;
625 CHECK(msg->findInt32("drainGeneration", &generation));
626 if (generation != getDrainGeneration(false /* audio */)) {
627 break;
628 }
629
630 mDrainVideoQueuePending = false;
631 postDrainVideoQueue();
632 break;
633 }
634
635 case kWhatQueueBuffer:
636 {
637 onQueueBuffer(msg);
638 break;
639 }
640
641 case kWhatQueueEOS:
642 {
643 onQueueEOS(msg);
644 break;
645 }
646
647 case kWhatEOS:
648 {
649 int32_t generation;
650 CHECK(msg->findInt32("audioEOSGeneration", &generation));
651 if (generation != mAudioEOSGeneration) {
652 break;
653 }
654 status_t finalResult;
655 CHECK(msg->findInt32("finalResult", &finalResult));
656 notifyEOS(true /* audio */, finalResult);
657 break;
658 }
659
660 case kWhatConfigPlayback:
661 {
662 sp<AReplyToken> replyID;
663 CHECK(msg->senderAwaitsResponse(&replyID));
664 AudioPlaybackRate rate;
665 readFromAMessage(msg, &rate);
666 status_t err = onConfigPlayback(rate);
667 sp<AMessage> response = new AMessage;
668 response->setInt32("err", err);
669 response->postReply(replyID);
670 break;
671 }
672
673 case kWhatGetPlaybackSettings:
674 {
675 sp<AReplyToken> replyID;
676 CHECK(msg->senderAwaitsResponse(&replyID));
677 AudioPlaybackRate rate = AUDIO_PLAYBACK_RATE_DEFAULT;
678 status_t err = onGetPlaybackSettings(&rate);
679 sp<AMessage> response = new AMessage;
680 if (err == OK) {
681 writeToAMessage(response, rate);
682 }
683 response->setInt32("err", err);
684 response->postReply(replyID);
685 break;
686 }
687
688 case kWhatConfigSync:
689 {
690 sp<AReplyToken> replyID;
691 CHECK(msg->senderAwaitsResponse(&replyID));
692 AVSyncSettings sync;
693 float videoFpsHint;
694 readFromAMessage(msg, &sync, &videoFpsHint);
695 status_t err = onConfigSync(sync, videoFpsHint);
696 sp<AMessage> response = new AMessage;
697 response->setInt32("err", err);
698 response->postReply(replyID);
699 break;
700 }
701
702 case kWhatGetSyncSettings:
703 {
704 sp<AReplyToken> replyID;
705 CHECK(msg->senderAwaitsResponse(&replyID));
706
707 ALOGV("kWhatGetSyncSettings");
708 AVSyncSettings sync;
709 float videoFps = -1.f;
710 status_t err = onGetSyncSettings(&sync, &videoFps);
711 sp<AMessage> response = new AMessage;
712 if (err == OK) {
713 writeToAMessage(response, sync, videoFps);
714 }
715 response->setInt32("err", err);
716 response->postReply(replyID);
717 break;
718 }
719
720 case kWhatFlush:
721 {
722 onFlush(msg);
723 break;
724 }
725
726 case kWhatDisableOffloadAudio:
727 {
728 onDisableOffloadAudio();
729 break;
730 }
731
732 case kWhatEnableOffloadAudio:
733 {
734 onEnableOffloadAudio();
735 break;
736 }
737
738 case kWhatPause:
739 {
740 onPause();
741 break;
742 }
743
744 case kWhatResume:
745 {
746 onResume();
747 break;
748 }
749
750 case kWhatSetVideoFrameRate:
751 {
752 float fps;
753 CHECK(msg->findFloat("frame-rate", &fps));
754 onSetVideoFrameRate(fps);
755 break;
756 }
757
758 case kWhatAudioTearDown:
759 {
760 int32_t reason;
761 CHECK(msg->findInt32("reason", &reason));
762
763 onAudioTearDown((AudioTearDownReason)reason);
764 break;
765 }
766
767 case kWhatAudioOffloadPauseTimeout:
768 {
769 int32_t generation;
770 CHECK(msg->findInt32("drainGeneration", &generation));
771 if (generation != mAudioOffloadPauseTimeoutGeneration) {
772 break;
773 }
774 ALOGV("Audio Offload tear down due to pause timeout.");
775 onAudioTearDown(kDueToTimeout);
776 mWakeLock->release();
777 break;
778 }
779
780 default:
781 TRESPASS();
782 break;
783 }
784 }
785
postDrainAudioQueue_l(int64_t delayUs)786 void NuPlayer::Renderer::postDrainAudioQueue_l(int64_t delayUs) {
787 if (mDrainAudioQueuePending || mSyncQueues || mUseAudioCallback) {
788 return;
789 }
790
791 if (mAudioQueue.empty()) {
792 return;
793 }
794
795 // FIXME: if paused, wait until AudioTrack stop() is complete before delivering data.
796 if (mPaused) {
797 const int64_t diffUs = mPauseDrainAudioAllowedUs - ALooper::GetNowUs();
798 if (diffUs > delayUs) {
799 delayUs = diffUs;
800 }
801 }
802
803 mDrainAudioQueuePending = true;
804 sp<AMessage> msg = new AMessage(kWhatDrainAudioQueue, this);
805 msg->setInt32("drainGeneration", mAudioDrainGeneration);
806 msg->post(delayUs);
807 }
808
prepareForMediaRenderingStart_l()809 void NuPlayer::Renderer::prepareForMediaRenderingStart_l() {
810 mAudioRenderingStartGeneration = mAudioDrainGeneration;
811 mVideoRenderingStartGeneration = mVideoDrainGeneration;
812 mRenderingDataDelivered = false;
813 }
814
notifyIfMediaRenderingStarted_l()815 void NuPlayer::Renderer::notifyIfMediaRenderingStarted_l() {
816 if (mVideoRenderingStartGeneration == mVideoDrainGeneration &&
817 mAudioRenderingStartGeneration == mAudioDrainGeneration) {
818 mRenderingDataDelivered = true;
819 if (mPaused) {
820 return;
821 }
822 mVideoRenderingStartGeneration = -1;
823 mAudioRenderingStartGeneration = -1;
824
825 sp<AMessage> notify = mNotify->dup();
826 notify->setInt32("what", kWhatMediaRenderingStart);
827 notify->post();
828 }
829 }
830
831 // static
AudioSinkCallback(MediaPlayerBase::AudioSink *,void * buffer,size_t size,void * cookie,MediaPlayerBase::AudioSink::cb_event_t event)832 size_t NuPlayer::Renderer::AudioSinkCallback(
833 MediaPlayerBase::AudioSink * /* audioSink */,
834 void *buffer,
835 size_t size,
836 void *cookie,
837 MediaPlayerBase::AudioSink::cb_event_t event) {
838 NuPlayer::Renderer *me = (NuPlayer::Renderer *)cookie;
839
840 switch (event) {
841 case MediaPlayerBase::AudioSink::CB_EVENT_FILL_BUFFER:
842 {
843 return me->fillAudioBuffer(buffer, size);
844 break;
845 }
846
847 case MediaPlayerBase::AudioSink::CB_EVENT_STREAM_END:
848 {
849 ALOGV("AudioSink::CB_EVENT_STREAM_END");
850 me->notifyEOSCallback();
851 break;
852 }
853
854 case MediaPlayerBase::AudioSink::CB_EVENT_TEAR_DOWN:
855 {
856 ALOGV("AudioSink::CB_EVENT_TEAR_DOWN");
857 me->notifyAudioTearDown(kDueToError);
858 break;
859 }
860 }
861
862 return 0;
863 }
864
notifyEOSCallback()865 void NuPlayer::Renderer::notifyEOSCallback() {
866 Mutex::Autolock autoLock(mLock);
867
868 if (!mUseAudioCallback) {
869 return;
870 }
871
872 notifyEOS_l(true /* audio */, ERROR_END_OF_STREAM);
873 }
874
fillAudioBuffer(void * buffer,size_t size)875 size_t NuPlayer::Renderer::fillAudioBuffer(void *buffer, size_t size) {
876 Mutex::Autolock autoLock(mLock);
877
878 if (!mUseAudioCallback) {
879 return 0;
880 }
881
882 bool hasEOS = false;
883
884 size_t sizeCopied = 0;
885 bool firstEntry = true;
886 QueueEntry *entry; // will be valid after while loop if hasEOS is set.
887 while (sizeCopied < size && !mAudioQueue.empty()) {
888 entry = &*mAudioQueue.begin();
889
890 if (entry->mBuffer == NULL) { // EOS
891 hasEOS = true;
892 mAudioQueue.erase(mAudioQueue.begin());
893 break;
894 }
895
896 if (firstEntry && entry->mOffset == 0) {
897 firstEntry = false;
898 int64_t mediaTimeUs;
899 CHECK(entry->mBuffer->meta()->findInt64("timeUs", &mediaTimeUs));
900 ALOGV("fillAudioBuffer: rendering audio at media time %.2f secs", mediaTimeUs / 1E6);
901 setAudioFirstAnchorTimeIfNeeded_l(mediaTimeUs);
902 }
903
904 size_t copy = entry->mBuffer->size() - entry->mOffset;
905 size_t sizeRemaining = size - sizeCopied;
906 if (copy > sizeRemaining) {
907 copy = sizeRemaining;
908 }
909
910 memcpy((char *)buffer + sizeCopied,
911 entry->mBuffer->data() + entry->mOffset,
912 copy);
913
914 entry->mOffset += copy;
915 if (entry->mOffset == entry->mBuffer->size()) {
916 entry->mNotifyConsumed->post();
917 mAudioQueue.erase(mAudioQueue.begin());
918 entry = NULL;
919 }
920 sizeCopied += copy;
921
922 notifyIfMediaRenderingStarted_l();
923 }
924
925 if (mAudioFirstAnchorTimeMediaUs >= 0) {
926 int64_t nowUs = ALooper::GetNowUs();
927 int64_t nowMediaUs =
928 mAudioFirstAnchorTimeMediaUs + mAudioSink->getPlayedOutDurationUs(nowUs);
929 // we don't know how much data we are queueing for offloaded tracks.
930 mMediaClock->updateAnchor(nowMediaUs, nowUs, INT64_MAX);
931 }
932
933 // for non-offloaded audio, we need to compute the frames written because
934 // there is no EVENT_STREAM_END notification. The frames written gives
935 // an estimate on the pending played out duration.
936 if (!offloadingAudio()) {
937 mNumFramesWritten += sizeCopied / mAudioSink->frameSize();
938 }
939
940 if (hasEOS) {
941 (new AMessage(kWhatStopAudioSink, this))->post();
942 // As there is currently no EVENT_STREAM_END callback notification for
943 // non-offloaded audio tracks, we need to post the EOS ourselves.
944 if (!offloadingAudio()) {
945 int64_t postEOSDelayUs = 0;
946 if (mAudioSink->needsTrailingPadding()) {
947 postEOSDelayUs = getPendingAudioPlayoutDurationUs(ALooper::GetNowUs());
948 }
949 ALOGV("fillAudioBuffer: notifyEOS_l "
950 "mNumFramesWritten:%u finalResult:%d postEOSDelay:%lld",
951 mNumFramesWritten, entry->mFinalResult, (long long)postEOSDelayUs);
952 notifyEOS_l(true /* audio */, entry->mFinalResult, postEOSDelayUs);
953 }
954 }
955 return sizeCopied;
956 }
957
drainAudioQueueUntilLastEOS()958 void NuPlayer::Renderer::drainAudioQueueUntilLastEOS() {
959 List<QueueEntry>::iterator it = mAudioQueue.begin(), itEOS = it;
960 bool foundEOS = false;
961 while (it != mAudioQueue.end()) {
962 int32_t eos;
963 QueueEntry *entry = &*it++;
964 if ((entry->mBuffer == nullptr && entry->mNotifyConsumed == nullptr)
965 || (entry->mNotifyConsumed->findInt32("eos", &eos) && eos != 0)) {
966 itEOS = it;
967 foundEOS = true;
968 }
969 }
970
971 if (foundEOS) {
972 // post all replies before EOS and drop the samples
973 for (it = mAudioQueue.begin(); it != itEOS; it++) {
974 if (it->mBuffer == nullptr) {
975 if (it->mNotifyConsumed == nullptr) {
976 // delay doesn't matter as we don't even have an AudioTrack
977 notifyEOS(true /* audio */, it->mFinalResult);
978 } else {
979 // TAG for re-opening audio sink.
980 onChangeAudioFormat(it->mMeta, it->mNotifyConsumed);
981 }
982 } else {
983 it->mNotifyConsumed->post();
984 }
985 }
986 mAudioQueue.erase(mAudioQueue.begin(), itEOS);
987 }
988 }
989
onDrainAudioQueue()990 bool NuPlayer::Renderer::onDrainAudioQueue() {
991 // do not drain audio during teardown as queued buffers may be invalid.
992 if (mAudioTornDown) {
993 return false;
994 }
995 // TODO: This call to getPosition checks if AudioTrack has been created
996 // in AudioSink before draining audio. If AudioTrack doesn't exist, then
997 // CHECKs on getPosition will fail.
998 // We still need to figure out why AudioTrack is not created when
999 // this function is called. One possible reason could be leftover
1000 // audio. Another possible place is to check whether decoder
1001 // has received INFO_FORMAT_CHANGED as the first buffer since
1002 // AudioSink is opened there, and possible interactions with flush
1003 // immediately after start. Investigate error message
1004 // "vorbis_dsp_synthesis returned -135", along with RTSP.
1005 uint32_t numFramesPlayed;
1006 if (mAudioSink->getPosition(&numFramesPlayed) != OK) {
1007 // When getPosition fails, renderer will not reschedule the draining
1008 // unless new samples are queued.
1009 // If we have pending EOS (or "eos" marker for discontinuities), we need
1010 // to post these now as NuPlayerDecoder might be waiting for it.
1011 drainAudioQueueUntilLastEOS();
1012
1013 ALOGW("onDrainAudioQueue(): audio sink is not ready");
1014 return false;
1015 }
1016
1017 #if 0
1018 ssize_t numFramesAvailableToWrite =
1019 mAudioSink->frameCount() - (mNumFramesWritten - numFramesPlayed);
1020
1021 if (numFramesAvailableToWrite == mAudioSink->frameCount()) {
1022 ALOGI("audio sink underrun");
1023 } else {
1024 ALOGV("audio queue has %d frames left to play",
1025 mAudioSink->frameCount() - numFramesAvailableToWrite);
1026 }
1027 #endif
1028
1029 uint32_t prevFramesWritten = mNumFramesWritten;
1030 while (!mAudioQueue.empty()) {
1031 QueueEntry *entry = &*mAudioQueue.begin();
1032
1033 if (entry->mBuffer == NULL) {
1034 if (entry->mNotifyConsumed != nullptr) {
1035 // TAG for re-open audio sink.
1036 onChangeAudioFormat(entry->mMeta, entry->mNotifyConsumed);
1037 mAudioQueue.erase(mAudioQueue.begin());
1038 continue;
1039 }
1040
1041 // EOS
1042 if (mPaused) {
1043 // Do not notify EOS when paused.
1044 // This is needed to avoid switch to next clip while in pause.
1045 ALOGV("onDrainAudioQueue(): Do not notify EOS when paused");
1046 return false;
1047 }
1048
1049 int64_t postEOSDelayUs = 0;
1050 if (mAudioSink->needsTrailingPadding()) {
1051 postEOSDelayUs = getPendingAudioPlayoutDurationUs(ALooper::GetNowUs());
1052 }
1053 notifyEOS(true /* audio */, entry->mFinalResult, postEOSDelayUs);
1054 mLastAudioMediaTimeUs = getDurationUsIfPlayedAtSampleRate(mNumFramesWritten);
1055
1056 mAudioQueue.erase(mAudioQueue.begin());
1057 entry = NULL;
1058 if (mAudioSink->needsTrailingPadding()) {
1059 // If we're not in gapless playback (i.e. through setNextPlayer), we
1060 // need to stop the track here, because that will play out the last
1061 // little bit at the end of the file. Otherwise short files won't play.
1062 mAudioSink->stop();
1063 mNumFramesWritten = 0;
1064 }
1065 return false;
1066 }
1067
1068 mLastAudioBufferDrained = entry->mBufferOrdinal;
1069
1070 // ignore 0-sized buffer which could be EOS marker with no data
1071 if (entry->mOffset == 0 && entry->mBuffer->size() > 0) {
1072 int64_t mediaTimeUs;
1073 CHECK(entry->mBuffer->meta()->findInt64("timeUs", &mediaTimeUs));
1074 ALOGV("onDrainAudioQueue: rendering audio at media time %.2f secs",
1075 mediaTimeUs / 1E6);
1076 onNewAudioMediaTime(mediaTimeUs);
1077 }
1078
1079 size_t copy = entry->mBuffer->size() - entry->mOffset;
1080
1081 ssize_t written = mAudioSink->write(entry->mBuffer->data() + entry->mOffset,
1082 copy, false /* blocking */);
1083 if (written < 0) {
1084 // An error in AudioSink write. Perhaps the AudioSink was not properly opened.
1085 if (written == WOULD_BLOCK) {
1086 ALOGV("AudioSink write would block when writing %zu bytes", copy);
1087 } else {
1088 ALOGE("AudioSink write error(%zd) when writing %zu bytes", written, copy);
1089 // This can only happen when AudioSink was opened with doNotReconnect flag set to
1090 // true, in which case the NuPlayer will handle the reconnect.
1091 notifyAudioTearDown(kDueToError);
1092 }
1093 break;
1094 }
1095
1096 entry->mOffset += written;
1097 size_t remainder = entry->mBuffer->size() - entry->mOffset;
1098 if ((ssize_t)remainder < mAudioSink->frameSize()) {
1099 if (remainder > 0) {
1100 ALOGW("Corrupted audio buffer has fractional frames, discarding %zu bytes.",
1101 remainder);
1102 entry->mOffset += remainder;
1103 copy -= remainder;
1104 }
1105
1106 entry->mNotifyConsumed->post();
1107 mAudioQueue.erase(mAudioQueue.begin());
1108
1109 entry = NULL;
1110 }
1111
1112 size_t copiedFrames = written / mAudioSink->frameSize();
1113 mNumFramesWritten += copiedFrames;
1114
1115 {
1116 Mutex::Autolock autoLock(mLock);
1117 int64_t maxTimeMedia;
1118 maxTimeMedia =
1119 mAnchorTimeMediaUs +
1120 (int64_t)(max((long long)mNumFramesWritten - mAnchorNumFramesWritten, 0LL)
1121 * 1000LL * mAudioSink->msecsPerFrame());
1122 mMediaClock->updateMaxTimeMedia(maxTimeMedia);
1123
1124 notifyIfMediaRenderingStarted_l();
1125 }
1126
1127 if (written != (ssize_t)copy) {
1128 // A short count was received from AudioSink::write()
1129 //
1130 // AudioSink write is called in non-blocking mode.
1131 // It may return with a short count when:
1132 //
1133 // 1) Size to be copied is not a multiple of the frame size. Fractional frames are
1134 // discarded.
1135 // 2) The data to be copied exceeds the available buffer in AudioSink.
1136 // 3) An error occurs and data has been partially copied to the buffer in AudioSink.
1137 // 4) AudioSink is an AudioCache for data retrieval, and the AudioCache is exceeded.
1138
1139 // (Case 1)
1140 // Must be a multiple of the frame size. If it is not a multiple of a frame size, it
1141 // needs to fail, as we should not carry over fractional frames between calls.
1142 CHECK_EQ(copy % mAudioSink->frameSize(), 0u);
1143
1144 // (Case 2, 3, 4)
1145 // Return early to the caller.
1146 // Beware of calling immediately again as this may busy-loop if you are not careful.
1147 ALOGV("AudioSink write short frame count %zd < %zu", written, copy);
1148 break;
1149 }
1150 }
1151
1152 // calculate whether we need to reschedule another write.
1153 bool reschedule = !mAudioQueue.empty()
1154 && (!mPaused
1155 || prevFramesWritten != mNumFramesWritten); // permit pause to fill buffers
1156 //ALOGD("reschedule:%d empty:%d mPaused:%d prevFramesWritten:%u mNumFramesWritten:%u",
1157 // reschedule, mAudioQueue.empty(), mPaused, prevFramesWritten, mNumFramesWritten);
1158 return reschedule;
1159 }
1160
getDurationUsIfPlayedAtSampleRate(uint32_t numFrames)1161 int64_t NuPlayer::Renderer::getDurationUsIfPlayedAtSampleRate(uint32_t numFrames) {
1162 int32_t sampleRate = offloadingAudio() ?
1163 mCurrentOffloadInfo.sample_rate : mCurrentPcmInfo.mSampleRate;
1164 if (sampleRate == 0) {
1165 ALOGE("sampleRate is 0 in %s mode", offloadingAudio() ? "offload" : "non-offload");
1166 return 0;
1167 }
1168
1169 return (int64_t)(numFrames * 1000000LL / sampleRate);
1170 }
1171
1172 // Calculate duration of pending samples if played at normal rate (i.e., 1.0).
getPendingAudioPlayoutDurationUs(int64_t nowUs)1173 int64_t NuPlayer::Renderer::getPendingAudioPlayoutDurationUs(int64_t nowUs) {
1174 int64_t writtenAudioDurationUs = getDurationUsIfPlayedAtSampleRate(mNumFramesWritten);
1175 if (mUseVirtualAudioSink) {
1176 int64_t nowUs = ALooper::GetNowUs();
1177 int64_t mediaUs;
1178 if (mMediaClock->getMediaTime(nowUs, &mediaUs) != OK) {
1179 return 0LL;
1180 } else {
1181 return writtenAudioDurationUs - (mediaUs - mAudioFirstAnchorTimeMediaUs);
1182 }
1183 }
1184
1185 const int64_t audioSinkPlayedUs = mAudioSink->getPlayedOutDurationUs(nowUs);
1186 int64_t pendingUs = writtenAudioDurationUs - audioSinkPlayedUs;
1187 if (pendingUs < 0) {
1188 // This shouldn't happen unless the timestamp is stale.
1189 ALOGW("%s: pendingUs %lld < 0, clamping to zero, potential resume after pause "
1190 "writtenAudioDurationUs: %lld, audioSinkPlayedUs: %lld",
1191 __func__, (long long)pendingUs,
1192 (long long)writtenAudioDurationUs, (long long)audioSinkPlayedUs);
1193 pendingUs = 0;
1194 }
1195 return pendingUs;
1196 }
1197
getRealTimeUs(int64_t mediaTimeUs,int64_t nowUs)1198 int64_t NuPlayer::Renderer::getRealTimeUs(int64_t mediaTimeUs, int64_t nowUs) {
1199 int64_t realUs;
1200 if (mMediaClock->getRealTimeFor(mediaTimeUs, &realUs) != OK) {
1201 // If failed to get current position, e.g. due to audio clock is
1202 // not ready, then just play out video immediately without delay.
1203 return nowUs;
1204 }
1205 return realUs;
1206 }
1207
onNewAudioMediaTime(int64_t mediaTimeUs)1208 void NuPlayer::Renderer::onNewAudioMediaTime(int64_t mediaTimeUs) {
1209 Mutex::Autolock autoLock(mLock);
1210 // TRICKY: vorbis decoder generates multiple frames with the same
1211 // timestamp, so only update on the first frame with a given timestamp
1212 if (mediaTimeUs == mAnchorTimeMediaUs) {
1213 return;
1214 }
1215 setAudioFirstAnchorTimeIfNeeded_l(mediaTimeUs);
1216
1217 // mNextAudioClockUpdateTimeUs is -1 if we're waiting for audio sink to start
1218 if (mNextAudioClockUpdateTimeUs == -1) {
1219 AudioTimestamp ts;
1220 if (mAudioSink->getTimestamp(ts) == OK && ts.mPosition > 0) {
1221 mNextAudioClockUpdateTimeUs = 0; // start our clock updates
1222 }
1223 }
1224 int64_t nowUs = ALooper::GetNowUs();
1225 if (mNextAudioClockUpdateTimeUs >= 0) {
1226 if (nowUs >= mNextAudioClockUpdateTimeUs) {
1227 int64_t nowMediaUs = mediaTimeUs - getPendingAudioPlayoutDurationUs(nowUs);
1228 mMediaClock->updateAnchor(nowMediaUs, nowUs, mediaTimeUs);
1229 mUseVirtualAudioSink = false;
1230 mNextAudioClockUpdateTimeUs = nowUs + kMinimumAudioClockUpdatePeriodUs;
1231 }
1232 } else {
1233 int64_t unused;
1234 if ((mMediaClock->getMediaTime(nowUs, &unused) != OK)
1235 && (getDurationUsIfPlayedAtSampleRate(mNumFramesWritten)
1236 > kMaxAllowedAudioSinkDelayUs)) {
1237 // Enough data has been sent to AudioSink, but AudioSink has not rendered
1238 // any data yet. Something is wrong with AudioSink, e.g., the device is not
1239 // connected to audio out.
1240 // Switch to system clock. This essentially creates a virtual AudioSink with
1241 // initial latenty of getDurationUsIfPlayedAtSampleRate(mNumFramesWritten).
1242 // This virtual AudioSink renders audio data starting from the very first sample
1243 // and it's paced by system clock.
1244 ALOGW("AudioSink stuck. ARE YOU CONNECTED TO AUDIO OUT? Switching to system clock.");
1245 mMediaClock->updateAnchor(mAudioFirstAnchorTimeMediaUs, nowUs, mediaTimeUs);
1246 mUseVirtualAudioSink = true;
1247 }
1248 }
1249 mAnchorNumFramesWritten = mNumFramesWritten;
1250 mAnchorTimeMediaUs = mediaTimeUs;
1251 }
1252
1253 // Called without mLock acquired.
postDrainVideoQueue()1254 void NuPlayer::Renderer::postDrainVideoQueue() {
1255 if (mDrainVideoQueuePending
1256 || getSyncQueues()
1257 || (mPaused && mVideoSampleReceived)) {
1258 return;
1259 }
1260
1261 if (mVideoQueue.empty()) {
1262 return;
1263 }
1264
1265 QueueEntry &entry = *mVideoQueue.begin();
1266
1267 sp<AMessage> msg = new AMessage(kWhatDrainVideoQueue, this);
1268 msg->setInt32("drainGeneration", getDrainGeneration(false /* audio */));
1269
1270 if (entry.mBuffer == NULL) {
1271 // EOS doesn't carry a timestamp.
1272 msg->post();
1273 mDrainVideoQueuePending = true;
1274 return;
1275 }
1276
1277 int64_t nowUs = ALooper::GetNowUs();
1278 if (mFlags & FLAG_REAL_TIME) {
1279 int64_t realTimeUs;
1280 CHECK(entry.mBuffer->meta()->findInt64("timeUs", &realTimeUs));
1281
1282 realTimeUs = mVideoScheduler->schedule(realTimeUs * 1000) / 1000;
1283
1284 int64_t twoVsyncsUs = 2 * (mVideoScheduler->getVsyncPeriod() / 1000);
1285
1286 int64_t delayUs = realTimeUs - nowUs;
1287
1288 ALOGW_IF(delayUs > 500000, "unusually high delayUs: %lld", (long long)delayUs);
1289 // post 2 display refreshes before rendering is due
1290 msg->post(delayUs > twoVsyncsUs ? delayUs - twoVsyncsUs : 0);
1291
1292 mDrainVideoQueuePending = true;
1293 return;
1294 }
1295
1296 int64_t mediaTimeUs;
1297 CHECK(entry.mBuffer->meta()->findInt64("timeUs", &mediaTimeUs));
1298
1299 {
1300 Mutex::Autolock autoLock(mLock);
1301 if (mAnchorTimeMediaUs < 0) {
1302 mMediaClock->updateAnchor(mediaTimeUs, nowUs, mediaTimeUs);
1303 mAnchorTimeMediaUs = mediaTimeUs;
1304 }
1305 }
1306 mNextVideoTimeMediaUs = mediaTimeUs;
1307 if (!mHasAudio) {
1308 // smooth out videos >= 10fps
1309 mMediaClock->updateMaxTimeMedia(mediaTimeUs + kDefaultVideoFrameIntervalUs);
1310 }
1311
1312 if (!mVideoSampleReceived || mediaTimeUs < mAudioFirstAnchorTimeMediaUs) {
1313 msg->post();
1314 } else {
1315 int64_t twoVsyncsUs = 2 * (mVideoScheduler->getVsyncPeriod() / 1000);
1316
1317 // post 2 display refreshes before rendering is due
1318 mMediaClock->addTimer(msg, mediaTimeUs, -twoVsyncsUs);
1319 }
1320
1321 mDrainVideoQueuePending = true;
1322 }
1323
onDrainVideoQueue()1324 void NuPlayer::Renderer::onDrainVideoQueue() {
1325 if (mVideoQueue.empty()) {
1326 return;
1327 }
1328
1329 QueueEntry *entry = &*mVideoQueue.begin();
1330
1331 if (entry->mBuffer == NULL) {
1332 // EOS
1333
1334 notifyEOS(false /* audio */, entry->mFinalResult);
1335
1336 mVideoQueue.erase(mVideoQueue.begin());
1337 entry = NULL;
1338
1339 setVideoLateByUs(0);
1340 return;
1341 }
1342
1343 int64_t nowUs = ALooper::GetNowUs();
1344 int64_t realTimeUs;
1345 int64_t mediaTimeUs = -1;
1346 if (mFlags & FLAG_REAL_TIME) {
1347 CHECK(entry->mBuffer->meta()->findInt64("timeUs", &realTimeUs));
1348 } else {
1349 CHECK(entry->mBuffer->meta()->findInt64("timeUs", &mediaTimeUs));
1350
1351 realTimeUs = getRealTimeUs(mediaTimeUs, nowUs);
1352 }
1353 realTimeUs = mVideoScheduler->schedule(realTimeUs * 1000) / 1000;
1354
1355 bool tooLate = false;
1356
1357 if (!mPaused) {
1358 setVideoLateByUs(nowUs - realTimeUs);
1359 tooLate = (mVideoLateByUs > 40000);
1360
1361 if (tooLate) {
1362 ALOGV("video late by %lld us (%.2f secs)",
1363 (long long)mVideoLateByUs, mVideoLateByUs / 1E6);
1364 } else {
1365 int64_t mediaUs = 0;
1366 mMediaClock->getMediaTime(realTimeUs, &mediaUs);
1367 ALOGV("rendering video at media time %.2f secs",
1368 (mFlags & FLAG_REAL_TIME ? realTimeUs :
1369 mediaUs) / 1E6);
1370
1371 if (!(mFlags & FLAG_REAL_TIME)
1372 && mLastAudioMediaTimeUs != -1
1373 && mediaTimeUs > mLastAudioMediaTimeUs) {
1374 // If audio ends before video, video continues to drive media clock.
1375 // Also smooth out videos >= 10fps.
1376 mMediaClock->updateMaxTimeMedia(mediaTimeUs + kDefaultVideoFrameIntervalUs);
1377 }
1378 }
1379 } else {
1380 setVideoLateByUs(0);
1381 if (!mVideoSampleReceived && !mHasAudio) {
1382 // This will ensure that the first frame after a flush won't be used as anchor
1383 // when renderer is in paused state, because resume can happen any time after seek.
1384 clearAnchorTime();
1385 }
1386 }
1387
1388 // Always render the first video frame while keeping stats on A/V sync.
1389 if (!mVideoSampleReceived) {
1390 realTimeUs = nowUs;
1391 tooLate = false;
1392 }
1393
1394 entry->mNotifyConsumed->setInt64("timestampNs", realTimeUs * 1000LL);
1395 entry->mNotifyConsumed->setInt32("render", !tooLate);
1396 entry->mNotifyConsumed->post();
1397 mVideoQueue.erase(mVideoQueue.begin());
1398 entry = NULL;
1399
1400 mVideoSampleReceived = true;
1401
1402 if (!mPaused) {
1403 if (!mVideoRenderingStarted) {
1404 mVideoRenderingStarted = true;
1405 notifyVideoRenderingStart();
1406 }
1407 Mutex::Autolock autoLock(mLock);
1408 notifyIfMediaRenderingStarted_l();
1409 }
1410 }
1411
notifyVideoRenderingStart()1412 void NuPlayer::Renderer::notifyVideoRenderingStart() {
1413 sp<AMessage> notify = mNotify->dup();
1414 notify->setInt32("what", kWhatVideoRenderingStart);
1415 notify->post();
1416 }
1417
notifyEOS(bool audio,status_t finalResult,int64_t delayUs)1418 void NuPlayer::Renderer::notifyEOS(bool audio, status_t finalResult, int64_t delayUs) {
1419 Mutex::Autolock autoLock(mLock);
1420 notifyEOS_l(audio, finalResult, delayUs);
1421 }
1422
notifyEOS_l(bool audio,status_t finalResult,int64_t delayUs)1423 void NuPlayer::Renderer::notifyEOS_l(bool audio, status_t finalResult, int64_t delayUs) {
1424 if (audio && delayUs > 0) {
1425 sp<AMessage> msg = new AMessage(kWhatEOS, this);
1426 msg->setInt32("audioEOSGeneration", mAudioEOSGeneration);
1427 msg->setInt32("finalResult", finalResult);
1428 msg->post(delayUs);
1429 return;
1430 }
1431 sp<AMessage> notify = mNotify->dup();
1432 notify->setInt32("what", kWhatEOS);
1433 notify->setInt32("audio", static_cast<int32_t>(audio));
1434 notify->setInt32("finalResult", finalResult);
1435 notify->post(delayUs);
1436
1437 if (audio) {
1438 // Video might outlive audio. Clear anchor to enable video only case.
1439 mAnchorTimeMediaUs = -1;
1440 mHasAudio = false;
1441 if (mNextVideoTimeMediaUs >= 0) {
1442 int64_t mediaUs = 0;
1443 int64_t nowUs = ALooper::GetNowUs();
1444 status_t result = mMediaClock->getMediaTime(nowUs, &mediaUs);
1445 if (result == OK) {
1446 if (mNextVideoTimeMediaUs > mediaUs) {
1447 mMediaClock->updateMaxTimeMedia(mNextVideoTimeMediaUs);
1448 }
1449 } else {
1450 mMediaClock->updateAnchor(
1451 mNextVideoTimeMediaUs, nowUs,
1452 mNextVideoTimeMediaUs + kDefaultVideoFrameIntervalUs);
1453 }
1454 }
1455 }
1456 }
1457
notifyAudioTearDown(AudioTearDownReason reason)1458 void NuPlayer::Renderer::notifyAudioTearDown(AudioTearDownReason reason) {
1459 sp<AMessage> msg = new AMessage(kWhatAudioTearDown, this);
1460 msg->setInt32("reason", reason);
1461 msg->post();
1462 }
1463
onQueueBuffer(const sp<AMessage> & msg)1464 void NuPlayer::Renderer::onQueueBuffer(const sp<AMessage> &msg) {
1465 int32_t audio;
1466 CHECK(msg->findInt32("audio", &audio));
1467
1468 if (dropBufferIfStale(audio, msg)) {
1469 return;
1470 }
1471
1472 if (audio) {
1473 mHasAudio = true;
1474 } else {
1475 mHasVideo = true;
1476 }
1477
1478 if (mHasVideo) {
1479 if (mVideoScheduler == NULL) {
1480 mVideoScheduler = new VideoFrameScheduler();
1481 mVideoScheduler->init();
1482 }
1483 }
1484
1485 sp<RefBase> obj;
1486 CHECK(msg->findObject("buffer", &obj));
1487 sp<MediaCodecBuffer> buffer = static_cast<MediaCodecBuffer *>(obj.get());
1488
1489 sp<AMessage> notifyConsumed;
1490 CHECK(msg->findMessage("notifyConsumed", ¬ifyConsumed));
1491
1492 QueueEntry entry;
1493 entry.mBuffer = buffer;
1494 entry.mNotifyConsumed = notifyConsumed;
1495 entry.mOffset = 0;
1496 entry.mFinalResult = OK;
1497 entry.mBufferOrdinal = ++mTotalBuffersQueued;
1498
1499 if (audio) {
1500 Mutex::Autolock autoLock(mLock);
1501 mAudioQueue.push_back(entry);
1502 postDrainAudioQueue_l();
1503 } else {
1504 mVideoQueue.push_back(entry);
1505 postDrainVideoQueue();
1506 }
1507
1508 Mutex::Autolock autoLock(mLock);
1509 if (!mSyncQueues || mAudioQueue.empty() || mVideoQueue.empty()) {
1510 return;
1511 }
1512
1513 sp<MediaCodecBuffer> firstAudioBuffer = (*mAudioQueue.begin()).mBuffer;
1514 sp<MediaCodecBuffer> firstVideoBuffer = (*mVideoQueue.begin()).mBuffer;
1515
1516 if (firstAudioBuffer == NULL || firstVideoBuffer == NULL) {
1517 // EOS signalled on either queue.
1518 syncQueuesDone_l();
1519 return;
1520 }
1521
1522 int64_t firstAudioTimeUs;
1523 int64_t firstVideoTimeUs;
1524 CHECK(firstAudioBuffer->meta()
1525 ->findInt64("timeUs", &firstAudioTimeUs));
1526 CHECK(firstVideoBuffer->meta()
1527 ->findInt64("timeUs", &firstVideoTimeUs));
1528
1529 int64_t diff = firstVideoTimeUs - firstAudioTimeUs;
1530
1531 ALOGV("queueDiff = %.2f secs", diff / 1E6);
1532
1533 if (diff > 100000LL) {
1534 // Audio data starts More than 0.1 secs before video.
1535 // Drop some audio.
1536
1537 (*mAudioQueue.begin()).mNotifyConsumed->post();
1538 mAudioQueue.erase(mAudioQueue.begin());
1539 return;
1540 }
1541
1542 syncQueuesDone_l();
1543 }
1544
syncQueuesDone_l()1545 void NuPlayer::Renderer::syncQueuesDone_l() {
1546 if (!mSyncQueues) {
1547 return;
1548 }
1549
1550 mSyncQueues = false;
1551
1552 if (!mAudioQueue.empty()) {
1553 postDrainAudioQueue_l();
1554 }
1555
1556 if (!mVideoQueue.empty()) {
1557 mLock.unlock();
1558 postDrainVideoQueue();
1559 mLock.lock();
1560 }
1561 }
1562
onQueueEOS(const sp<AMessage> & msg)1563 void NuPlayer::Renderer::onQueueEOS(const sp<AMessage> &msg) {
1564 int32_t audio;
1565 CHECK(msg->findInt32("audio", &audio));
1566
1567 if (dropBufferIfStale(audio, msg)) {
1568 return;
1569 }
1570
1571 int32_t finalResult;
1572 CHECK(msg->findInt32("finalResult", &finalResult));
1573
1574 QueueEntry entry;
1575 entry.mOffset = 0;
1576 entry.mFinalResult = finalResult;
1577
1578 if (audio) {
1579 Mutex::Autolock autoLock(mLock);
1580 if (mAudioQueue.empty() && mSyncQueues) {
1581 syncQueuesDone_l();
1582 }
1583 mAudioQueue.push_back(entry);
1584 postDrainAudioQueue_l();
1585 } else {
1586 if (mVideoQueue.empty() && getSyncQueues()) {
1587 Mutex::Autolock autoLock(mLock);
1588 syncQueuesDone_l();
1589 }
1590 mVideoQueue.push_back(entry);
1591 postDrainVideoQueue();
1592 }
1593 }
1594
onFlush(const sp<AMessage> & msg)1595 void NuPlayer::Renderer::onFlush(const sp<AMessage> &msg) {
1596 int32_t audio, notifyComplete;
1597 CHECK(msg->findInt32("audio", &audio));
1598
1599 {
1600 Mutex::Autolock autoLock(mLock);
1601 if (audio) {
1602 notifyComplete = mNotifyCompleteAudio;
1603 mNotifyCompleteAudio = false;
1604 mLastAudioMediaTimeUs = -1;
1605
1606 mHasAudio = false;
1607 if (mNextVideoTimeMediaUs >= 0) {
1608 int64_t nowUs = ALooper::GetNowUs();
1609 mMediaClock->updateAnchor(
1610 mNextVideoTimeMediaUs, nowUs,
1611 mNextVideoTimeMediaUs + kDefaultVideoFrameIntervalUs);
1612 }
1613 } else {
1614 notifyComplete = mNotifyCompleteVideo;
1615 mNotifyCompleteVideo = false;
1616 }
1617
1618 // If we're currently syncing the queues, i.e. dropping audio while
1619 // aligning the first audio/video buffer times and only one of the
1620 // two queues has data, we may starve that queue by not requesting
1621 // more buffers from the decoder. If the other source then encounters
1622 // a discontinuity that leads to flushing, we'll never find the
1623 // corresponding discontinuity on the other queue.
1624 // Therefore we'll stop syncing the queues if at least one of them
1625 // is flushed.
1626 syncQueuesDone_l();
1627 }
1628 clearAnchorTime();
1629
1630 ALOGV("flushing %s", audio ? "audio" : "video");
1631 if (audio) {
1632 {
1633 Mutex::Autolock autoLock(mLock);
1634 flushQueue(&mAudioQueue);
1635
1636 ++mAudioDrainGeneration;
1637 ++mAudioEOSGeneration;
1638 prepareForMediaRenderingStart_l();
1639
1640 // the frame count will be reset after flush.
1641 clearAudioFirstAnchorTime_l();
1642 }
1643
1644 mDrainAudioQueuePending = false;
1645
1646 if (offloadingAudio()) {
1647 mAudioSink->pause();
1648 mAudioSink->flush();
1649 if (!mPaused) {
1650 mAudioSink->start();
1651 }
1652 } else {
1653 mAudioSink->pause();
1654 mAudioSink->flush();
1655 // Call stop() to signal to the AudioSink to completely fill the
1656 // internal buffer before resuming playback.
1657 // FIXME: this is ignored after flush().
1658 mAudioSink->stop();
1659 if (!mPaused) {
1660 mAudioSink->start();
1661 }
1662 mNumFramesWritten = 0;
1663 }
1664 mNextAudioClockUpdateTimeUs = -1;
1665 } else {
1666 flushQueue(&mVideoQueue);
1667
1668 mDrainVideoQueuePending = false;
1669
1670 if (mVideoScheduler != NULL) {
1671 mVideoScheduler->restart();
1672 }
1673
1674 Mutex::Autolock autoLock(mLock);
1675 ++mVideoDrainGeneration;
1676 prepareForMediaRenderingStart_l();
1677 }
1678
1679 mVideoSampleReceived = false;
1680
1681 if (notifyComplete) {
1682 notifyFlushComplete(audio);
1683 }
1684 }
1685
flushQueue(List<QueueEntry> * queue)1686 void NuPlayer::Renderer::flushQueue(List<QueueEntry> *queue) {
1687 while (!queue->empty()) {
1688 QueueEntry *entry = &*queue->begin();
1689
1690 if (entry->mBuffer != NULL) {
1691 entry->mNotifyConsumed->post();
1692 } else if (entry->mNotifyConsumed != nullptr) {
1693 // Is it needed to open audio sink now?
1694 onChangeAudioFormat(entry->mMeta, entry->mNotifyConsumed);
1695 }
1696
1697 queue->erase(queue->begin());
1698 entry = NULL;
1699 }
1700 }
1701
notifyFlushComplete(bool audio)1702 void NuPlayer::Renderer::notifyFlushComplete(bool audio) {
1703 sp<AMessage> notify = mNotify->dup();
1704 notify->setInt32("what", kWhatFlushComplete);
1705 notify->setInt32("audio", static_cast<int32_t>(audio));
1706 notify->post();
1707 }
1708
dropBufferIfStale(bool audio,const sp<AMessage> & msg)1709 bool NuPlayer::Renderer::dropBufferIfStale(
1710 bool audio, const sp<AMessage> &msg) {
1711 int32_t queueGeneration;
1712 CHECK(msg->findInt32("queueGeneration", &queueGeneration));
1713
1714 if (queueGeneration == getQueueGeneration(audio)) {
1715 return false;
1716 }
1717
1718 sp<AMessage> notifyConsumed;
1719 if (msg->findMessage("notifyConsumed", ¬ifyConsumed)) {
1720 notifyConsumed->post();
1721 }
1722
1723 return true;
1724 }
1725
onAudioSinkChanged()1726 void NuPlayer::Renderer::onAudioSinkChanged() {
1727 if (offloadingAudio()) {
1728 return;
1729 }
1730 CHECK(!mDrainAudioQueuePending);
1731 mNumFramesWritten = 0;
1732 mAnchorNumFramesWritten = -1;
1733 uint32_t written;
1734 if (mAudioSink->getFramesWritten(&written) == OK) {
1735 mNumFramesWritten = written;
1736 }
1737 }
1738
onDisableOffloadAudio()1739 void NuPlayer::Renderer::onDisableOffloadAudio() {
1740 Mutex::Autolock autoLock(mLock);
1741 mFlags &= ~FLAG_OFFLOAD_AUDIO;
1742 ++mAudioDrainGeneration;
1743 if (mAudioRenderingStartGeneration != -1) {
1744 prepareForMediaRenderingStart_l();
1745 // PauseTimeout is applied to offload mode only. Cancel pending timer.
1746 cancelAudioOffloadPauseTimeout();
1747 }
1748 }
1749
onEnableOffloadAudio()1750 void NuPlayer::Renderer::onEnableOffloadAudio() {
1751 Mutex::Autolock autoLock(mLock);
1752 mFlags |= FLAG_OFFLOAD_AUDIO;
1753 ++mAudioDrainGeneration;
1754 if (mAudioRenderingStartGeneration != -1) {
1755 prepareForMediaRenderingStart_l();
1756 }
1757 }
1758
onPause()1759 void NuPlayer::Renderer::onPause() {
1760 if (mPaused) {
1761 return;
1762 }
1763
1764 {
1765 Mutex::Autolock autoLock(mLock);
1766 // we do not increment audio drain generation so that we fill audio buffer during pause.
1767 ++mVideoDrainGeneration;
1768 prepareForMediaRenderingStart_l();
1769 mPaused = true;
1770 mMediaClock->setPlaybackRate(0.0);
1771 }
1772
1773 mDrainAudioQueuePending = false;
1774 mDrainVideoQueuePending = false;
1775
1776 // Note: audio data may not have been decoded, and the AudioSink may not be opened.
1777 mAudioSink->pause();
1778 startAudioOffloadPauseTimeout();
1779
1780 ALOGV("now paused audio queue has %zu entries, video has %zu entries",
1781 mAudioQueue.size(), mVideoQueue.size());
1782 }
1783
onResume()1784 void NuPlayer::Renderer::onResume() {
1785 if (!mPaused) {
1786 return;
1787 }
1788
1789 // Note: audio data may not have been decoded, and the AudioSink may not be opened.
1790 cancelAudioOffloadPauseTimeout();
1791 if (mAudioSink->ready()) {
1792 status_t err = mAudioSink->start();
1793 if (err != OK) {
1794 ALOGE("cannot start AudioSink err %d", err);
1795 notifyAudioTearDown(kDueToError);
1796 }
1797 }
1798
1799 {
1800 Mutex::Autolock autoLock(mLock);
1801 mPaused = false;
1802 // rendering started message may have been delayed if we were paused.
1803 if (mRenderingDataDelivered) {
1804 notifyIfMediaRenderingStarted_l();
1805 }
1806 // configure audiosink as we did not do it when pausing
1807 if (mAudioSink != NULL && mAudioSink->ready()) {
1808 mAudioSink->setPlaybackRate(mPlaybackSettings);
1809 }
1810
1811 mMediaClock->setPlaybackRate(mPlaybackRate);
1812
1813 if (!mAudioQueue.empty()) {
1814 postDrainAudioQueue_l();
1815 }
1816 }
1817
1818 if (!mVideoQueue.empty()) {
1819 postDrainVideoQueue();
1820 }
1821 }
1822
onSetVideoFrameRate(float fps)1823 void NuPlayer::Renderer::onSetVideoFrameRate(float fps) {
1824 if (mVideoScheduler == NULL) {
1825 mVideoScheduler = new VideoFrameScheduler();
1826 }
1827 mVideoScheduler->init(fps);
1828 }
1829
getQueueGeneration(bool audio)1830 int32_t NuPlayer::Renderer::getQueueGeneration(bool audio) {
1831 Mutex::Autolock autoLock(mLock);
1832 return (audio ? mAudioQueueGeneration : mVideoQueueGeneration);
1833 }
1834
getDrainGeneration(bool audio)1835 int32_t NuPlayer::Renderer::getDrainGeneration(bool audio) {
1836 Mutex::Autolock autoLock(mLock);
1837 return (audio ? mAudioDrainGeneration : mVideoDrainGeneration);
1838 }
1839
getSyncQueues()1840 bool NuPlayer::Renderer::getSyncQueues() {
1841 Mutex::Autolock autoLock(mLock);
1842 return mSyncQueues;
1843 }
1844
onAudioTearDown(AudioTearDownReason reason)1845 void NuPlayer::Renderer::onAudioTearDown(AudioTearDownReason reason) {
1846 if (mAudioTornDown) {
1847 return;
1848 }
1849
1850 // TimeoutWhenPaused is only for offload mode.
1851 if (reason == kDueToTimeout && !offloadingAudio()) {
1852 return;
1853 }
1854
1855 mAudioTornDown = true;
1856
1857 int64_t currentPositionUs;
1858 sp<AMessage> notify = mNotify->dup();
1859 if (getCurrentPosition(¤tPositionUs) == OK) {
1860 notify->setInt64("positionUs", currentPositionUs);
1861 }
1862
1863 mAudioSink->stop();
1864 mAudioSink->flush();
1865
1866 notify->setInt32("what", kWhatAudioTearDown);
1867 notify->setInt32("reason", reason);
1868 notify->post();
1869 }
1870
startAudioOffloadPauseTimeout()1871 void NuPlayer::Renderer::startAudioOffloadPauseTimeout() {
1872 if (offloadingAudio()) {
1873 mWakeLock->acquire();
1874 sp<AMessage> msg = new AMessage(kWhatAudioOffloadPauseTimeout, this);
1875 msg->setInt32("drainGeneration", mAudioOffloadPauseTimeoutGeneration);
1876 msg->post(kOffloadPauseMaxUs);
1877 }
1878 }
1879
cancelAudioOffloadPauseTimeout()1880 void NuPlayer::Renderer::cancelAudioOffloadPauseTimeout() {
1881 // We may have called startAudioOffloadPauseTimeout() without
1882 // the AudioSink open and with offloadingAudio enabled.
1883 //
1884 // When we cancel, it may be that offloadingAudio is subsequently disabled, so regardless
1885 // we always release the wakelock and increment the pause timeout generation.
1886 //
1887 // Note: The acquired wakelock prevents the device from suspending
1888 // immediately after offload pause (in case a resume happens shortly thereafter).
1889 mWakeLock->release(true);
1890 ++mAudioOffloadPauseTimeoutGeneration;
1891 }
1892
onOpenAudioSink(const sp<AMessage> & format,bool offloadOnly,bool hasVideo,uint32_t flags,bool isStreaming)1893 status_t NuPlayer::Renderer::onOpenAudioSink(
1894 const sp<AMessage> &format,
1895 bool offloadOnly,
1896 bool hasVideo,
1897 uint32_t flags,
1898 bool isStreaming) {
1899 ALOGV("openAudioSink: offloadOnly(%d) offloadingAudio(%d)",
1900 offloadOnly, offloadingAudio());
1901 bool audioSinkChanged = false;
1902
1903 int32_t numChannels;
1904 CHECK(format->findInt32("channel-count", &numChannels));
1905
1906 int32_t channelMask;
1907 if (!format->findInt32("channel-mask", &channelMask)) {
1908 // signal to the AudioSink to derive the mask from count.
1909 channelMask = CHANNEL_MASK_USE_CHANNEL_ORDER;
1910 }
1911
1912 int32_t sampleRate;
1913 CHECK(format->findInt32("sample-rate", &sampleRate));
1914
1915 // read pcm encoding from MediaCodec output format, if available
1916 int32_t pcmEncoding;
1917 audio_format_t audioFormat =
1918 format->findInt32(KEY_PCM_ENCODING, &pcmEncoding) ?
1919 audioFormatFromEncoding(pcmEncoding) : AUDIO_FORMAT_PCM_16_BIT;
1920
1921 if (offloadingAudio()) {
1922 AString mime;
1923 CHECK(format->findString("mime", &mime));
1924 status_t err = mapMimeToAudioFormat(audioFormat, mime.c_str());
1925
1926 if (err != OK) {
1927 ALOGE("Couldn't map mime \"%s\" to a valid "
1928 "audio_format", mime.c_str());
1929 onDisableOffloadAudio();
1930 } else {
1931 ALOGV("Mime \"%s\" mapped to audio_format 0x%x",
1932 mime.c_str(), audioFormat);
1933
1934 int avgBitRate = -1;
1935 format->findInt32("bitrate", &avgBitRate);
1936
1937 int32_t aacProfile = -1;
1938 if (audioFormat == AUDIO_FORMAT_AAC
1939 && format->findInt32("aac-profile", &aacProfile)) {
1940 // Redefine AAC format as per aac profile
1941 mapAACProfileToAudioFormat(
1942 audioFormat,
1943 aacProfile);
1944 }
1945
1946 audio_offload_info_t offloadInfo = AUDIO_INFO_INITIALIZER;
1947 offloadInfo.duration_us = -1;
1948 format->findInt64(
1949 "durationUs", &offloadInfo.duration_us);
1950 offloadInfo.sample_rate = sampleRate;
1951 offloadInfo.channel_mask = channelMask;
1952 offloadInfo.format = audioFormat;
1953 offloadInfo.stream_type = AUDIO_STREAM_MUSIC;
1954 offloadInfo.bit_rate = avgBitRate;
1955 offloadInfo.has_video = hasVideo;
1956 offloadInfo.is_streaming = isStreaming;
1957
1958 if (memcmp(&mCurrentOffloadInfo, &offloadInfo, sizeof(offloadInfo)) == 0) {
1959 ALOGV("openAudioSink: no change in offload mode");
1960 // no change from previous configuration, everything ok.
1961 return OK;
1962 }
1963 mCurrentPcmInfo = AUDIO_PCMINFO_INITIALIZER;
1964
1965 ALOGV("openAudioSink: try to open AudioSink in offload mode");
1966 uint32_t offloadFlags = flags;
1967 offloadFlags |= AUDIO_OUTPUT_FLAG_COMPRESS_OFFLOAD;
1968 offloadFlags &= ~AUDIO_OUTPUT_FLAG_DEEP_BUFFER;
1969 audioSinkChanged = true;
1970 mAudioSink->close();
1971
1972 err = mAudioSink->open(
1973 sampleRate,
1974 numChannels,
1975 (audio_channel_mask_t)channelMask,
1976 audioFormat,
1977 0 /* bufferCount - unused */,
1978 &NuPlayer::Renderer::AudioSinkCallback,
1979 this,
1980 (audio_output_flags_t)offloadFlags,
1981 &offloadInfo);
1982
1983 if (err == OK) {
1984 err = mAudioSink->setPlaybackRate(mPlaybackSettings);
1985 }
1986
1987 if (err == OK) {
1988 // If the playback is offloaded to h/w, we pass
1989 // the HAL some metadata information.
1990 // We don't want to do this for PCM because it
1991 // will be going through the AudioFlinger mixer
1992 // before reaching the hardware.
1993 // TODO
1994 mCurrentOffloadInfo = offloadInfo;
1995 if (!mPaused) { // for preview mode, don't start if paused
1996 err = mAudioSink->start();
1997 }
1998 ALOGV_IF(err == OK, "openAudioSink: offload succeeded");
1999 }
2000 if (err != OK) {
2001 // Clean up, fall back to non offload mode.
2002 mAudioSink->close();
2003 onDisableOffloadAudio();
2004 mCurrentOffloadInfo = AUDIO_INFO_INITIALIZER;
2005 ALOGV("openAudioSink: offload failed");
2006 if (offloadOnly) {
2007 notifyAudioTearDown(kForceNonOffload);
2008 }
2009 } else {
2010 mUseAudioCallback = true; // offload mode transfers data through callback
2011 ++mAudioDrainGeneration; // discard pending kWhatDrainAudioQueue message.
2012 }
2013 }
2014 }
2015 if (!offloadOnly && !offloadingAudio()) {
2016 ALOGV("openAudioSink: open AudioSink in NON-offload mode");
2017 uint32_t pcmFlags = flags;
2018 pcmFlags &= ~AUDIO_OUTPUT_FLAG_COMPRESS_OFFLOAD;
2019
2020 const PcmInfo info = {
2021 (audio_channel_mask_t)channelMask,
2022 (audio_output_flags_t)pcmFlags,
2023 audioFormat,
2024 numChannels,
2025 sampleRate
2026 };
2027 if (memcmp(&mCurrentPcmInfo, &info, sizeof(info)) == 0) {
2028 ALOGV("openAudioSink: no change in pcm mode");
2029 // no change from previous configuration, everything ok.
2030 return OK;
2031 }
2032
2033 audioSinkChanged = true;
2034 mAudioSink->close();
2035 mCurrentOffloadInfo = AUDIO_INFO_INITIALIZER;
2036 // Note: It is possible to set up the callback, but not use it to send audio data.
2037 // This requires a fix in AudioSink to explicitly specify the transfer mode.
2038 mUseAudioCallback = getUseAudioCallbackSetting();
2039 if (mUseAudioCallback) {
2040 ++mAudioDrainGeneration; // discard pending kWhatDrainAudioQueue message.
2041 }
2042
2043 // Compute the desired buffer size.
2044 // For callback mode, the amount of time before wakeup is about half the buffer size.
2045 const uint32_t frameCount =
2046 (unsigned long long)sampleRate * getAudioSinkPcmMsSetting() / 1000;
2047
2048 // The doNotReconnect means AudioSink will signal back and let NuPlayer to re-construct
2049 // AudioSink. We don't want this when there's video because it will cause a video seek to
2050 // the previous I frame. But we do want this when there's only audio because it will give
2051 // NuPlayer a chance to switch from non-offload mode to offload mode.
2052 // So we only set doNotReconnect when there's no video.
2053 const bool doNotReconnect = !hasVideo;
2054
2055 // We should always be able to set our playback settings if the sink is closed.
2056 LOG_ALWAYS_FATAL_IF(mAudioSink->setPlaybackRate(mPlaybackSettings) != OK,
2057 "onOpenAudioSink: can't set playback rate on closed sink");
2058 status_t err = mAudioSink->open(
2059 sampleRate,
2060 numChannels,
2061 (audio_channel_mask_t)channelMask,
2062 audioFormat,
2063 0 /* bufferCount - unused */,
2064 mUseAudioCallback ? &NuPlayer::Renderer::AudioSinkCallback : NULL,
2065 mUseAudioCallback ? this : NULL,
2066 (audio_output_flags_t)pcmFlags,
2067 NULL,
2068 doNotReconnect,
2069 frameCount);
2070 if (err != OK) {
2071 ALOGW("openAudioSink: non offloaded open failed status: %d", err);
2072 mAudioSink->close();
2073 mCurrentPcmInfo = AUDIO_PCMINFO_INITIALIZER;
2074 return err;
2075 }
2076 mCurrentPcmInfo = info;
2077 if (!mPaused) { // for preview mode, don't start if paused
2078 mAudioSink->start();
2079 }
2080 }
2081 if (audioSinkChanged) {
2082 onAudioSinkChanged();
2083 }
2084 mAudioTornDown = false;
2085 return OK;
2086 }
2087
onCloseAudioSink()2088 void NuPlayer::Renderer::onCloseAudioSink() {
2089 mAudioSink->close();
2090 mCurrentOffloadInfo = AUDIO_INFO_INITIALIZER;
2091 mCurrentPcmInfo = AUDIO_PCMINFO_INITIALIZER;
2092 }
2093
onChangeAudioFormat(const sp<AMessage> & meta,const sp<AMessage> & notify)2094 void NuPlayer::Renderer::onChangeAudioFormat(
2095 const sp<AMessage> &meta, const sp<AMessage> ¬ify) {
2096 sp<AMessage> format;
2097 CHECK(meta->findMessage("format", &format));
2098
2099 int32_t offloadOnly;
2100 CHECK(meta->findInt32("offload-only", &offloadOnly));
2101
2102 int32_t hasVideo;
2103 CHECK(meta->findInt32("has-video", &hasVideo));
2104
2105 uint32_t flags;
2106 CHECK(meta->findInt32("flags", (int32_t *)&flags));
2107
2108 uint32_t isStreaming;
2109 CHECK(meta->findInt32("isStreaming", (int32_t *)&isStreaming));
2110
2111 status_t err = onOpenAudioSink(format, offloadOnly, hasVideo, flags, isStreaming);
2112
2113 if (err != OK) {
2114 notify->setInt32("err", err);
2115 }
2116 notify->post();
2117 }
2118
2119 } // namespace android
2120