1 /*
2 * Copyright 2012, The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 //#define LOG_NDEBUG 0
18 #define LOG_TAG "MediaCodec"
19 #include <utils/Log.h>
20
21 #include <inttypes.h>
22 #include <stdlib.h>
23
24 #include "include/SecureBuffer.h"
25 #include "include/SharedMemoryBuffer.h"
26 #include "include/SoftwareRenderer.h"
27 #include "StagefrightPluginLoader.h"
28
29 #include <android/hardware/cas/native/1.0/IDescrambler.h>
30
31 #include <binder/IMemory.h>
32 #include <binder/IPCThreadState.h>
33 #include <binder/IServiceManager.h>
34 #include <binder/MemoryDealer.h>
35 #include <cutils/properties.h>
36 #include <gui/BufferQueue.h>
37 #include <gui/Surface.h>
38 #include <mediadrm/ICrypto.h>
39 #include <media/IOMX.h>
40 #include <media/IResourceManagerService.h>
41 #include <media/MediaCodecBuffer.h>
42 #include <media/MediaAnalyticsItem.h>
43 #include <media/stagefright/foundation/ABuffer.h>
44 #include <media/stagefright/foundation/ADebug.h>
45 #include <media/stagefright/foundation/AMessage.h>
46 #include <media/stagefright/foundation/AString.h>
47 #include <media/stagefright/foundation/AUtils.h>
48 #include <media/stagefright/foundation/avc_utils.h>
49 #include <media/stagefright/foundation/hexdump.h>
50 #include <media/stagefright/ACodec.h>
51 #include <media/stagefright/BatteryChecker.h>
52 #include <media/stagefright/BufferProducerWrapper.h>
53 #include <media/stagefright/MediaCodec.h>
54 #include <media/stagefright/MediaCodecList.h>
55 #include <media/stagefright/MediaDefs.h>
56 #include <media/stagefright/MediaErrors.h>
57 #include <media/stagefright/MediaFilter.h>
58 #include <media/stagefright/OMXClient.h>
59 #include <media/stagefright/PersistentSurface.h>
60 #include <media/stagefright/SurfaceUtils.h>
61 #include <private/android_filesystem_config.h>
62 #include <utils/Singleton.h>
63
64 namespace android {
65
66 // key for media statistics
67 static const char *kCodecKeyName = "codec";
68 // attrs for media statistics
69 // NB: these are matched with public Java API constants defined
70 // in frameworks/base/media/java/android/media/MediaCodec.java
71 // These must be kept synchronized with the constants there.
72 static const char *kCodecCodec = "android.media.mediacodec.codec"; /* e.g. OMX.google.aac.decoder */
73 static const char *kCodecMime = "android.media.mediacodec.mime"; /* e.g. audio/mime */
74 static const char *kCodecMode = "android.media.mediacodec.mode"; /* audio, video */
75 static const char *kCodecModeVideo = "video"; /* values returned for kCodecMode */
76 static const char *kCodecModeAudio = "audio";
77 static const char *kCodecEncoder = "android.media.mediacodec.encoder"; /* 0,1 */
78 static const char *kCodecSecure = "android.media.mediacodec.secure"; /* 0, 1 */
79 static const char *kCodecWidth = "android.media.mediacodec.width"; /* 0..n */
80 static const char *kCodecHeight = "android.media.mediacodec.height"; /* 0..n */
81 static const char *kCodecRotation = "android.media.mediacodec.rotation-degrees"; /* 0/90/180/270 */
82
83 // NB: These are not yet exposed as public Java API constants.
84 static const char *kCodecCrypto = "android.media.mediacodec.crypto"; /* 0,1 */
85 static const char *kCodecProfile = "android.media.mediacodec.profile"; /* 0..n */
86 static const char *kCodecLevel = "android.media.mediacodec.level"; /* 0..n */
87 static const char *kCodecMaxWidth = "android.media.mediacodec.maxwidth"; /* 0..n */
88 static const char *kCodecMaxHeight = "android.media.mediacodec.maxheight"; /* 0..n */
89 static const char *kCodecError = "android.media.mediacodec.errcode";
90 static const char *kCodecErrorState = "android.media.mediacodec.errstate";
91 static const char *kCodecLatencyMax = "android.media.mediacodec.latency.max"; /* in us */
92 static const char *kCodecLatencyMin = "android.media.mediacodec.latency.min"; /* in us */
93 static const char *kCodecLatencyAvg = "android.media.mediacodec.latency.avg"; /* in us */
94 static const char *kCodecLatencyCount = "android.media.mediacodec.latency.n";
95 static const char *kCodecLatencyHist = "android.media.mediacodec.latency.hist"; /* in us */
96 static const char *kCodecLatencyUnknown = "android.media.mediacodec.latency.unknown";
97
98 // the kCodecRecent* fields appear only in getMetrics() results
99 static const char *kCodecRecentLatencyMax = "android.media.mediacodec.recent.max"; /* in us */
100 static const char *kCodecRecentLatencyMin = "android.media.mediacodec.recent.min"; /* in us */
101 static const char *kCodecRecentLatencyAvg = "android.media.mediacodec.recent.avg"; /* in us */
102 static const char *kCodecRecentLatencyCount = "android.media.mediacodec.recent.n";
103 static const char *kCodecRecentLatencyHist = "android.media.mediacodec.recent.hist"; /* in us */
104
105 // XXX suppress until we get our representation right
106 static bool kEmitHistogram = false;
107
108
getId(const sp<IResourceManagerClient> & client)109 static int64_t getId(const sp<IResourceManagerClient> &client) {
110 return (int64_t) client.get();
111 }
112
isResourceError(status_t err)113 static bool isResourceError(status_t err) {
114 return (err == NO_MEMORY);
115 }
116
117 static const int kMaxRetry = 2;
118 static const int kMaxReclaimWaitTimeInUs = 500000; // 0.5s
119 static const int kNumBuffersAlign = 16;
120
121 ////////////////////////////////////////////////////////////////////////////////
122
123 struct ResourceManagerClient : public BnResourceManagerClient {
ResourceManagerClientandroid::ResourceManagerClient124 explicit ResourceManagerClient(MediaCodec* codec) : mMediaCodec(codec) {}
125
reclaimResourceandroid::ResourceManagerClient126 virtual bool reclaimResource() {
127 sp<MediaCodec> codec = mMediaCodec.promote();
128 if (codec == NULL) {
129 // codec is already gone.
130 return true;
131 }
132 status_t err = codec->reclaim();
133 if (err == WOULD_BLOCK) {
134 ALOGD("Wait for the client to release codec.");
135 usleep(kMaxReclaimWaitTimeInUs);
136 ALOGD("Try to reclaim again.");
137 err = codec->reclaim(true /* force */);
138 }
139 if (err != OK) {
140 ALOGW("ResourceManagerClient failed to release codec with err %d", err);
141 }
142 return (err == OK);
143 }
144
getNameandroid::ResourceManagerClient145 virtual String8 getName() {
146 String8 ret;
147 sp<MediaCodec> codec = mMediaCodec.promote();
148 if (codec == NULL) {
149 // codec is already gone.
150 return ret;
151 }
152
153 AString name;
154 if (codec->getName(&name) == OK) {
155 ret.setTo(name.c_str());
156 }
157 return ret;
158 }
159
160 protected:
~ResourceManagerClientandroid::ResourceManagerClient161 virtual ~ResourceManagerClient() {}
162
163 private:
164 wp<MediaCodec> mMediaCodec;
165
166 DISALLOW_EVIL_CONSTRUCTORS(ResourceManagerClient);
167 };
168
ResourceManagerServiceProxy(pid_t pid,uid_t uid)169 MediaCodec::ResourceManagerServiceProxy::ResourceManagerServiceProxy(
170 pid_t pid, uid_t uid)
171 : mPid(pid), mUid(uid) {
172 if (mPid == MediaCodec::kNoPid) {
173 mPid = IPCThreadState::self()->getCallingPid();
174 }
175 }
176
~ResourceManagerServiceProxy()177 MediaCodec::ResourceManagerServiceProxy::~ResourceManagerServiceProxy() {
178 if (mService != NULL) {
179 IInterface::asBinder(mService)->unlinkToDeath(this);
180 }
181 }
182
init()183 void MediaCodec::ResourceManagerServiceProxy::init() {
184 sp<IServiceManager> sm = defaultServiceManager();
185 sp<IBinder> binder = sm->getService(String16("media.resource_manager"));
186 mService = interface_cast<IResourceManagerService>(binder);
187 if (mService == NULL) {
188 ALOGE("Failed to get ResourceManagerService");
189 return;
190 }
191 IInterface::asBinder(mService)->linkToDeath(this);
192 }
193
binderDied(const wp<IBinder> &)194 void MediaCodec::ResourceManagerServiceProxy::binderDied(const wp<IBinder>& /*who*/) {
195 ALOGW("ResourceManagerService died.");
196 Mutex::Autolock _l(mLock);
197 mService.clear();
198 }
199
addResource(int64_t clientId,const sp<IResourceManagerClient> & client,const Vector<MediaResource> & resources)200 void MediaCodec::ResourceManagerServiceProxy::addResource(
201 int64_t clientId,
202 const sp<IResourceManagerClient> &client,
203 const Vector<MediaResource> &resources) {
204 Mutex::Autolock _l(mLock);
205 if (mService == NULL) {
206 return;
207 }
208 mService->addResource(mPid, mUid, clientId, client, resources);
209 }
210
removeResource(int64_t clientId,const Vector<MediaResource> & resources)211 void MediaCodec::ResourceManagerServiceProxy::removeResource(
212 int64_t clientId,
213 const Vector<MediaResource> &resources) {
214 Mutex::Autolock _l(mLock);
215 if (mService == NULL) {
216 return;
217 }
218 mService->removeResource(mPid, clientId, resources);
219 }
220
removeClient(int64_t clientId)221 void MediaCodec::ResourceManagerServiceProxy::removeClient(int64_t clientId) {
222 Mutex::Autolock _l(mLock);
223 if (mService == NULL) {
224 return;
225 }
226 mService->removeClient(mPid, clientId);
227 }
228
reclaimResource(const Vector<MediaResource> & resources)229 bool MediaCodec::ResourceManagerServiceProxy::reclaimResource(
230 const Vector<MediaResource> &resources) {
231 Mutex::Autolock _l(mLock);
232 if (mService == NULL) {
233 return false;
234 }
235 return mService->reclaimResource(mPid, resources);
236 }
237
238 ////////////////////////////////////////////////////////////////////////////////
239
BufferInfo()240 MediaCodec::BufferInfo::BufferInfo() : mOwnedByClient(false) {}
241
242 ////////////////////////////////////////////////////////////////////////////////
243
244 namespace {
245
246 enum {
247 kWhatFillThisBuffer = 'fill',
248 kWhatDrainThisBuffer = 'drai',
249 kWhatEOS = 'eos ',
250 kWhatStartCompleted = 'Scom',
251 kWhatStopCompleted = 'scom',
252 kWhatReleaseCompleted = 'rcom',
253 kWhatFlushCompleted = 'fcom',
254 kWhatError = 'erro',
255 kWhatComponentAllocated = 'cAll',
256 kWhatComponentConfigured = 'cCon',
257 kWhatInputSurfaceCreated = 'isfc',
258 kWhatInputSurfaceAccepted = 'isfa',
259 kWhatSignaledInputEOS = 'seos',
260 kWhatOutputFramesRendered = 'outR',
261 kWhatOutputBuffersChanged = 'outC',
262 };
263
264 class BufferCallback : public CodecBase::BufferCallback {
265 public:
266 explicit BufferCallback(const sp<AMessage> ¬ify);
267 virtual ~BufferCallback() = default;
268
269 virtual void onInputBufferAvailable(
270 size_t index, const sp<MediaCodecBuffer> &buffer) override;
271 virtual void onOutputBufferAvailable(
272 size_t index, const sp<MediaCodecBuffer> &buffer) override;
273 private:
274 const sp<AMessage> mNotify;
275 };
276
BufferCallback(const sp<AMessage> & notify)277 BufferCallback::BufferCallback(const sp<AMessage> ¬ify)
278 : mNotify(notify) {}
279
onInputBufferAvailable(size_t index,const sp<MediaCodecBuffer> & buffer)280 void BufferCallback::onInputBufferAvailable(
281 size_t index, const sp<MediaCodecBuffer> &buffer) {
282 sp<AMessage> notify(mNotify->dup());
283 notify->setInt32("what", kWhatFillThisBuffer);
284 notify->setSize("index", index);
285 notify->setObject("buffer", buffer);
286 notify->post();
287 }
288
onOutputBufferAvailable(size_t index,const sp<MediaCodecBuffer> & buffer)289 void BufferCallback::onOutputBufferAvailable(
290 size_t index, const sp<MediaCodecBuffer> &buffer) {
291 sp<AMessage> notify(mNotify->dup());
292 notify->setInt32("what", kWhatDrainThisBuffer);
293 notify->setSize("index", index);
294 notify->setObject("buffer", buffer);
295 notify->post();
296 }
297
298 class CodecCallback : public CodecBase::CodecCallback {
299 public:
300 explicit CodecCallback(const sp<AMessage> ¬ify);
301 virtual ~CodecCallback() = default;
302
303 virtual void onEos(status_t err) override;
304 virtual void onStartCompleted() override;
305 virtual void onStopCompleted() override;
306 virtual void onReleaseCompleted() override;
307 virtual void onFlushCompleted() override;
308 virtual void onError(status_t err, enum ActionCode actionCode) override;
309 virtual void onComponentAllocated(const char *componentName) override;
310 virtual void onComponentConfigured(
311 const sp<AMessage> &inputFormat, const sp<AMessage> &outputFormat) override;
312 virtual void onInputSurfaceCreated(
313 const sp<AMessage> &inputFormat,
314 const sp<AMessage> &outputFormat,
315 const sp<BufferProducerWrapper> &inputSurface) override;
316 virtual void onInputSurfaceCreationFailed(status_t err) override;
317 virtual void onInputSurfaceAccepted(
318 const sp<AMessage> &inputFormat,
319 const sp<AMessage> &outputFormat) override;
320 virtual void onInputSurfaceDeclined(status_t err) override;
321 virtual void onSignaledInputEOS(status_t err) override;
322 virtual void onOutputFramesRendered(const std::list<FrameRenderTracker::Info> &done) override;
323 virtual void onOutputBuffersChanged() override;
324 private:
325 const sp<AMessage> mNotify;
326 };
327
CodecCallback(const sp<AMessage> & notify)328 CodecCallback::CodecCallback(const sp<AMessage> ¬ify) : mNotify(notify) {}
329
onEos(status_t err)330 void CodecCallback::onEos(status_t err) {
331 sp<AMessage> notify(mNotify->dup());
332 notify->setInt32("what", kWhatEOS);
333 notify->setInt32("err", err);
334 notify->post();
335 }
336
onStartCompleted()337 void CodecCallback::onStartCompleted() {
338 sp<AMessage> notify(mNotify->dup());
339 notify->setInt32("what", kWhatStartCompleted);
340 notify->post();
341 }
342
onStopCompleted()343 void CodecCallback::onStopCompleted() {
344 sp<AMessage> notify(mNotify->dup());
345 notify->setInt32("what", kWhatStopCompleted);
346 notify->post();
347 }
348
onReleaseCompleted()349 void CodecCallback::onReleaseCompleted() {
350 sp<AMessage> notify(mNotify->dup());
351 notify->setInt32("what", kWhatReleaseCompleted);
352 notify->post();
353 }
354
onFlushCompleted()355 void CodecCallback::onFlushCompleted() {
356 sp<AMessage> notify(mNotify->dup());
357 notify->setInt32("what", kWhatFlushCompleted);
358 notify->post();
359 }
360
onError(status_t err,enum ActionCode actionCode)361 void CodecCallback::onError(status_t err, enum ActionCode actionCode) {
362 sp<AMessage> notify(mNotify->dup());
363 notify->setInt32("what", kWhatError);
364 notify->setInt32("err", err);
365 notify->setInt32("actionCode", actionCode);
366 notify->post();
367 }
368
onComponentAllocated(const char * componentName)369 void CodecCallback::onComponentAllocated(const char *componentName) {
370 sp<AMessage> notify(mNotify->dup());
371 notify->setInt32("what", kWhatComponentAllocated);
372 notify->setString("componentName", componentName);
373 notify->post();
374 }
375
onComponentConfigured(const sp<AMessage> & inputFormat,const sp<AMessage> & outputFormat)376 void CodecCallback::onComponentConfigured(
377 const sp<AMessage> &inputFormat, const sp<AMessage> &outputFormat) {
378 sp<AMessage> notify(mNotify->dup());
379 notify->setInt32("what", kWhatComponentConfigured);
380 notify->setMessage("input-format", inputFormat);
381 notify->setMessage("output-format", outputFormat);
382 notify->post();
383 }
384
onInputSurfaceCreated(const sp<AMessage> & inputFormat,const sp<AMessage> & outputFormat,const sp<BufferProducerWrapper> & inputSurface)385 void CodecCallback::onInputSurfaceCreated(
386 const sp<AMessage> &inputFormat,
387 const sp<AMessage> &outputFormat,
388 const sp<BufferProducerWrapper> &inputSurface) {
389 sp<AMessage> notify(mNotify->dup());
390 notify->setInt32("what", kWhatInputSurfaceCreated);
391 notify->setMessage("input-format", inputFormat);
392 notify->setMessage("output-format", outputFormat);
393 notify->setObject("input-surface", inputSurface);
394 notify->post();
395 }
396
onInputSurfaceCreationFailed(status_t err)397 void CodecCallback::onInputSurfaceCreationFailed(status_t err) {
398 sp<AMessage> notify(mNotify->dup());
399 notify->setInt32("what", kWhatInputSurfaceCreated);
400 notify->setInt32("err", err);
401 notify->post();
402 }
403
onInputSurfaceAccepted(const sp<AMessage> & inputFormat,const sp<AMessage> & outputFormat)404 void CodecCallback::onInputSurfaceAccepted(
405 const sp<AMessage> &inputFormat,
406 const sp<AMessage> &outputFormat) {
407 sp<AMessage> notify(mNotify->dup());
408 notify->setInt32("what", kWhatInputSurfaceAccepted);
409 notify->setMessage("input-format", inputFormat);
410 notify->setMessage("output-format", outputFormat);
411 notify->post();
412 }
413
onInputSurfaceDeclined(status_t err)414 void CodecCallback::onInputSurfaceDeclined(status_t err) {
415 sp<AMessage> notify(mNotify->dup());
416 notify->setInt32("what", kWhatInputSurfaceAccepted);
417 notify->setInt32("err", err);
418 notify->post();
419 }
420
onSignaledInputEOS(status_t err)421 void CodecCallback::onSignaledInputEOS(status_t err) {
422 sp<AMessage> notify(mNotify->dup());
423 notify->setInt32("what", kWhatSignaledInputEOS);
424 if (err != OK) {
425 notify->setInt32("err", err);
426 }
427 notify->post();
428 }
429
onOutputFramesRendered(const std::list<FrameRenderTracker::Info> & done)430 void CodecCallback::onOutputFramesRendered(const std::list<FrameRenderTracker::Info> &done) {
431 sp<AMessage> notify(mNotify->dup());
432 notify->setInt32("what", kWhatOutputFramesRendered);
433 if (MediaCodec::CreateFramesRenderedMessage(done, notify)) {
434 notify->post();
435 }
436 }
437
onOutputBuffersChanged()438 void CodecCallback::onOutputBuffersChanged() {
439 sp<AMessage> notify(mNotify->dup());
440 notify->setInt32("what", kWhatOutputBuffersChanged);
441 notify->post();
442 }
443
444 } // namespace
445
446 ////////////////////////////////////////////////////////////////////////////////
447
448 // static
CreateByType(const sp<ALooper> & looper,const AString & mime,bool encoder,status_t * err,pid_t pid,uid_t uid)449 sp<MediaCodec> MediaCodec::CreateByType(
450 const sp<ALooper> &looper, const AString &mime, bool encoder, status_t *err, pid_t pid,
451 uid_t uid) {
452 Vector<AString> matchingCodecs;
453
454 MediaCodecList::findMatchingCodecs(
455 mime.c_str(),
456 encoder,
457 0,
458 &matchingCodecs);
459
460 if (err != NULL) {
461 *err = NAME_NOT_FOUND;
462 }
463 for (size_t i = 0; i < matchingCodecs.size(); ++i) {
464 sp<MediaCodec> codec = new MediaCodec(looper, pid, uid);
465 AString componentName = matchingCodecs[i];
466 status_t ret = codec->init(componentName);
467 if (err != NULL) {
468 *err = ret;
469 }
470 if (ret == OK) {
471 return codec;
472 }
473 ALOGD("Allocating component '%s' failed (%d), try next one.",
474 componentName.c_str(), ret);
475 }
476 return NULL;
477 }
478
479 // static
CreateByComponentName(const sp<ALooper> & looper,const AString & name,status_t * err,pid_t pid,uid_t uid)480 sp<MediaCodec> MediaCodec::CreateByComponentName(
481 const sp<ALooper> &looper, const AString &name, status_t *err, pid_t pid, uid_t uid) {
482 sp<MediaCodec> codec = new MediaCodec(looper, pid, uid);
483
484 const status_t ret = codec->init(name);
485 if (err != NULL) {
486 *err = ret;
487 }
488 return ret == OK ? codec : NULL; // NULL deallocates codec.
489 }
490
491 // static
CreatePersistentInputSurface()492 sp<PersistentSurface> MediaCodec::CreatePersistentInputSurface() {
493 // allow plugin to create surface
494 sp<PersistentSurface> pluginSurface =
495 StagefrightPluginLoader::GetCCodecInstance()->createInputSurface();
496 if (pluginSurface != nullptr) {
497 return pluginSurface;
498 }
499
500 OMXClient client;
501 if (client.connect() != OK) {
502 ALOGE("Failed to connect to OMX to create persistent input surface.");
503 return NULL;
504 }
505
506 sp<IOMX> omx = client.interface();
507
508 sp<IGraphicBufferProducer> bufferProducer;
509 sp<IGraphicBufferSource> bufferSource;
510
511 status_t err = omx->createInputSurface(&bufferProducer, &bufferSource);
512
513 if (err != OK) {
514 ALOGE("Failed to create persistent input surface.");
515 return NULL;
516 }
517
518 return new PersistentSurface(bufferProducer, bufferSource);
519 }
520
MediaCodec(const sp<ALooper> & looper,pid_t pid,uid_t uid)521 MediaCodec::MediaCodec(const sp<ALooper> &looper, pid_t pid, uid_t uid)
522 : mState(UNINITIALIZED),
523 mReleasedByResourceManager(false),
524 mLooper(looper),
525 mCodec(NULL),
526 mReplyID(0),
527 mFlags(0),
528 mStickyError(OK),
529 mSoftRenderer(NULL),
530 mMetricsHandle(0),
531 mIsVideo(false),
532 mVideoWidth(0),
533 mVideoHeight(0),
534 mRotationDegrees(0),
535 mDequeueInputTimeoutGeneration(0),
536 mDequeueInputReplyID(0),
537 mDequeueOutputTimeoutGeneration(0),
538 mDequeueOutputReplyID(0),
539 mHaveInputSurface(false),
540 mHavePendingInputBuffers(false),
541 mCpuBoostRequested(false),
542 mLatencyUnknown(0) {
543 if (uid == kNoUid) {
544 mUid = IPCThreadState::self()->getCallingUid();
545 } else {
546 mUid = uid;
547 }
548 mResourceManagerClient = new ResourceManagerClient(this);
549 mResourceManagerService = new ResourceManagerServiceProxy(pid, mUid);
550
551 initMediametrics();
552 }
553
~MediaCodec()554 MediaCodec::~MediaCodec() {
555 CHECK_EQ(mState, UNINITIALIZED);
556 mResourceManagerService->removeClient(getId(mResourceManagerClient));
557
558 flushMediametrics();
559 }
560
initMediametrics()561 void MediaCodec::initMediametrics() {
562 if (mMetricsHandle == 0) {
563 mMetricsHandle = mediametrics_create(kCodecKeyName);
564 }
565
566 mLatencyHist.setup(kLatencyHistBuckets, kLatencyHistWidth, kLatencyHistFloor);
567
568 {
569 Mutex::Autolock al(mRecentLock);
570 for (int i = 0; i<kRecentLatencyFrames; i++) {
571 mRecentSamples[i] = kRecentSampleInvalid;
572 }
573 mRecentHead = 0;
574 }
575 }
576
updateMediametrics()577 void MediaCodec::updateMediametrics() {
578 ALOGV("MediaCodec::updateMediametrics");
579 if (mMetricsHandle == 0) {
580 return;
581 }
582
583
584 if (mLatencyHist.getCount() != 0 ) {
585 mediametrics_setInt64(mMetricsHandle, kCodecLatencyMax, mLatencyHist.getMax());
586 mediametrics_setInt64(mMetricsHandle, kCodecLatencyMin, mLatencyHist.getMin());
587 mediametrics_setInt64(mMetricsHandle, kCodecLatencyAvg, mLatencyHist.getAvg());
588 mediametrics_setInt64(mMetricsHandle, kCodecLatencyCount, mLatencyHist.getCount());
589
590 if (kEmitHistogram) {
591 // and the histogram itself
592 std::string hist = mLatencyHist.emit();
593 mediametrics_setCString(mMetricsHandle, kCodecLatencyHist, hist.c_str());
594 }
595 }
596 if (mLatencyUnknown > 0) {
597 mediametrics_setInt64(mMetricsHandle, kCodecLatencyUnknown, mLatencyUnknown);
598 }
599
600 #if 0
601 // enable for short term, only while debugging
602 updateEphemeralMediametrics(mMetricsHandle);
603 #endif
604 }
605
updateEphemeralMediametrics(mediametrics_handle_t item)606 void MediaCodec::updateEphemeralMediametrics(mediametrics_handle_t item) {
607 ALOGD("MediaCodec::updateEphemeralMediametrics()");
608
609 if (item == 0) {
610 return;
611 }
612
613 Histogram recentHist;
614
615 // build an empty histogram
616 recentHist.setup(kLatencyHistBuckets, kLatencyHistWidth, kLatencyHistFloor);
617
618 // stuff it with the samples in the ring buffer
619 {
620 Mutex::Autolock al(mRecentLock);
621
622 for (int i=0; i<kRecentLatencyFrames; i++) {
623 if (mRecentSamples[i] != kRecentSampleInvalid) {
624 recentHist.insert(mRecentSamples[i]);
625 }
626 }
627 }
628
629
630 // spit the data (if any) into the supplied analytics record
631 if (recentHist.getCount()!= 0 ) {
632 mediametrics_setInt64(item, kCodecRecentLatencyMax, recentHist.getMax());
633 mediametrics_setInt64(item, kCodecRecentLatencyMin, recentHist.getMin());
634 mediametrics_setInt64(item, kCodecRecentLatencyAvg, recentHist.getAvg());
635 mediametrics_setInt64(item, kCodecRecentLatencyCount, recentHist.getCount());
636
637 if (kEmitHistogram) {
638 // and the histogram itself
639 std::string hist = recentHist.emit();
640 mediametrics_setCString(item, kCodecRecentLatencyHist, hist.c_str());
641 }
642 }
643 }
644
flushMediametrics()645 void MediaCodec::flushMediametrics() {
646 updateMediametrics();
647 if (mMetricsHandle != 0) {
648 if (mediametrics_count(mMetricsHandle) > 0) {
649 mediametrics_selfRecord(mMetricsHandle);
650 }
651 mediametrics_delete(mMetricsHandle);
652 mMetricsHandle = 0;
653 }
654 }
655
setup(int nbuckets,int64_t width,int64_t floor)656 bool MediaCodec::Histogram::setup(int nbuckets, int64_t width, int64_t floor)
657 {
658 if (nbuckets <= 0 || width <= 0) {
659 return false;
660 }
661
662 // get histogram buckets
663 if (nbuckets == mBucketCount && mBuckets != NULL) {
664 // reuse our existing buffer
665 memset(mBuckets, 0, sizeof(*mBuckets) * mBucketCount);
666 } else {
667 // get a new pre-zeroed buffer
668 int64_t *newbuckets = (int64_t *)calloc(nbuckets, sizeof (*mBuckets));
669 if (newbuckets == NULL) {
670 goto bad;
671 }
672 if (mBuckets != NULL)
673 free(mBuckets);
674 mBuckets = newbuckets;
675 }
676
677 mWidth = width;
678 mFloor = floor;
679 mCeiling = floor + nbuckets * width;
680 mBucketCount = nbuckets;
681
682 mMin = INT64_MAX;
683 mMax = INT64_MIN;
684 mSum = 0;
685 mCount = 0;
686 mBelow = mAbove = 0;
687
688 return true;
689
690 bad:
691 if (mBuckets != NULL) {
692 free(mBuckets);
693 mBuckets = NULL;
694 }
695
696 return false;
697 }
698
insert(int64_t sample)699 void MediaCodec::Histogram::insert(int64_t sample)
700 {
701 // histogram is not set up
702 if (mBuckets == NULL) {
703 return;
704 }
705
706 mCount++;
707 mSum += sample;
708 if (mMin > sample) mMin = sample;
709 if (mMax < sample) mMax = sample;
710
711 if (sample < mFloor) {
712 mBelow++;
713 } else if (sample >= mCeiling) {
714 mAbove++;
715 } else {
716 int64_t slot = (sample - mFloor) / mWidth;
717 CHECK(slot < mBucketCount);
718 mBuckets[slot]++;
719 }
720 return;
721 }
722
emit()723 std::string MediaCodec::Histogram::emit()
724 {
725 std::string value;
726 char buffer[64];
727
728 // emits: width,Below{bucket0,bucket1,...., bucketN}above
729 // unconfigured will emit: 0,0{}0
730 // XXX: is this best representation?
731 snprintf(buffer, sizeof(buffer), "%" PRId64 ",%" PRId64 ",%" PRId64 "{",
732 mFloor, mWidth, mBelow);
733 value = buffer;
734 for (int i = 0; i < mBucketCount; i++) {
735 if (i != 0) {
736 value = value + ",";
737 }
738 snprintf(buffer, sizeof(buffer), "%" PRId64, mBuckets[i]);
739 value = value + buffer;
740 }
741 snprintf(buffer, sizeof(buffer), "}%" PRId64 , mAbove);
742 value = value + buffer;
743 return value;
744 }
745
746 // when we send a buffer to the codec;
statsBufferSent(int64_t presentationUs)747 void MediaCodec::statsBufferSent(int64_t presentationUs) {
748
749 // only enqueue if we have a legitimate time
750 if (presentationUs <= 0) {
751 ALOGV("presentation time: %" PRId64, presentationUs);
752 return;
753 }
754
755 if (mBatteryChecker != nullptr) {
756 mBatteryChecker->onCodecActivity([this] () {
757 addResource(MediaResource::kBattery, MediaResource::kVideoCodec, 1);
758 });
759 }
760
761 const int64_t nowNs = systemTime(SYSTEM_TIME_MONOTONIC);
762 BufferFlightTiming_t startdata = { presentationUs, nowNs };
763
764 {
765 // mutex access to mBuffersInFlight and other stats
766 Mutex::Autolock al(mLatencyLock);
767
768
769 // XXX: we *could* make sure that the time is later than the end of queue
770 // as part of a consistency check...
771 mBuffersInFlight.push_back(startdata);
772 }
773 }
774
775 // when we get a buffer back from the codec
statsBufferReceived(int64_t presentationUs)776 void MediaCodec::statsBufferReceived(int64_t presentationUs) {
777
778 CHECK_NE(mState, UNINITIALIZED);
779
780 // mutex access to mBuffersInFlight and other stats
781 Mutex::Autolock al(mLatencyLock);
782
783 // how long this buffer took for the round trip through the codec
784 // NB: pipelining can/will make these times larger. e.g., if each packet
785 // is always 2 msec and we have 3 in flight at any given time, we're going to
786 // see "6 msec" as an answer.
787
788 // ignore stuff with no presentation time
789 if (presentationUs <= 0) {
790 ALOGV("-- returned buffer timestamp %" PRId64 " <= 0, ignore it", presentationUs);
791 mLatencyUnknown++;
792 return;
793 }
794
795 if (mBatteryChecker != nullptr) {
796 mBatteryChecker->onCodecActivity([this] () {
797 addResource(MediaResource::kBattery, MediaResource::kVideoCodec, 1);
798 });
799 }
800
801 BufferFlightTiming_t startdata;
802 bool valid = false;
803 while (mBuffersInFlight.size() > 0) {
804 startdata = *mBuffersInFlight.begin();
805 ALOGV("-- Looking at startdata. presentation %" PRId64 ", start %" PRId64,
806 startdata.presentationUs, startdata.startedNs);
807 if (startdata.presentationUs == presentationUs) {
808 // a match
809 ALOGV("-- match entry for %" PRId64 ", hits our frame of %" PRId64,
810 startdata.presentationUs, presentationUs);
811 mBuffersInFlight.pop_front();
812 valid = true;
813 break;
814 } else if (startdata.presentationUs < presentationUs) {
815 // we must have missed the match for this, drop it and keep looking
816 ALOGV("-- drop entry for %" PRId64 ", before our frame of %" PRId64,
817 startdata.presentationUs, presentationUs);
818 mBuffersInFlight.pop_front();
819 continue;
820 } else {
821 // head is after, so we don't have a frame for ourselves
822 ALOGV("-- found entry for %" PRId64 ", AFTER our frame of %" PRId64
823 " we have nothing to pair with",
824 startdata.presentationUs, presentationUs);
825 mLatencyUnknown++;
826 return;
827 }
828 }
829 if (!valid) {
830 ALOGV("-- empty queue, so ignore that.");
831 mLatencyUnknown++;
832 return;
833 }
834
835 // nowNs start our calculations
836 const int64_t nowNs = systemTime(SYSTEM_TIME_MONOTONIC);
837 int64_t latencyUs = (nowNs - startdata.startedNs + 500) / 1000;
838
839 mLatencyHist.insert(latencyUs);
840
841 // push into the recent samples
842 {
843 Mutex::Autolock al(mRecentLock);
844
845 if (mRecentHead >= kRecentLatencyFrames) {
846 mRecentHead = 0;
847 }
848 mRecentSamples[mRecentHead++] = latencyUs;
849 }
850 }
851
852 // static
PostAndAwaitResponse(const sp<AMessage> & msg,sp<AMessage> * response)853 status_t MediaCodec::PostAndAwaitResponse(
854 const sp<AMessage> &msg, sp<AMessage> *response) {
855 status_t err = msg->postAndAwaitResponse(response);
856
857 if (err != OK) {
858 return err;
859 }
860
861 if (!(*response)->findInt32("err", &err)) {
862 err = OK;
863 }
864
865 return err;
866 }
867
PostReplyWithError(const sp<AReplyToken> & replyID,int32_t err)868 void MediaCodec::PostReplyWithError(const sp<AReplyToken> &replyID, int32_t err) {
869 int32_t finalErr = err;
870 if (mReleasedByResourceManager) {
871 // override the err code if MediaCodec has been released by ResourceManager.
872 finalErr = DEAD_OBJECT;
873 }
874
875 sp<AMessage> response = new AMessage;
876 response->setInt32("err", finalErr);
877 response->postReply(replyID);
878 }
879
CreateCCodec()880 static CodecBase *CreateCCodec() {
881 return StagefrightPluginLoader::GetCCodecInstance()->createCodec();
882 }
883
884 //static
GetCodecBase(const AString & name,const char * owner)885 sp<CodecBase> MediaCodec::GetCodecBase(const AString &name, const char *owner) {
886 if (owner) {
887 if (strcmp(owner, "default") == 0) {
888 return new ACodec;
889 } else if (strncmp(owner, "codec2", 6) == 0) {
890 return CreateCCodec();
891 }
892 }
893
894 if (name.startsWithIgnoreCase("c2.")) {
895 return CreateCCodec();
896 } else if (name.startsWithIgnoreCase("omx.")) {
897 // at this time only ACodec specifies a mime type.
898 return new ACodec;
899 } else if (name.startsWithIgnoreCase("android.filter.")) {
900 return new MediaFilter;
901 } else {
902 return NULL;
903 }
904 }
905
init(const AString & name)906 status_t MediaCodec::init(const AString &name) {
907 mResourceManagerService->init();
908
909 // save init parameters for reset
910 mInitName = name;
911
912 // Current video decoders do not return from OMX_FillThisBuffer
913 // quickly, violating the OpenMAX specs, until that is remedied
914 // we need to invest in an extra looper to free the main event
915 // queue.
916
917 mCodecInfo.clear();
918
919 bool secureCodec = false;
920 AString tmp = name;
921 if (tmp.endsWith(".secure")) {
922 secureCodec = true;
923 tmp.erase(tmp.size() - 7, 7);
924 }
925 const sp<IMediaCodecList> mcl = MediaCodecList::getInstance();
926 if (mcl == NULL) {
927 mCodec = NULL; // remove the codec.
928 return NO_INIT; // if called from Java should raise IOException
929 }
930 for (const AString &codecName : { name, tmp }) {
931 ssize_t codecIdx = mcl->findCodecByName(codecName.c_str());
932 if (codecIdx < 0) {
933 continue;
934 }
935 mCodecInfo = mcl->getCodecInfo(codecIdx);
936 Vector<AString> mediaTypes;
937 mCodecInfo->getSupportedMediaTypes(&mediaTypes);
938 for (size_t i = 0; i < mediaTypes.size(); i++) {
939 if (mediaTypes[i].startsWith("video/")) {
940 mIsVideo = true;
941 break;
942 }
943 }
944 break;
945 }
946 if (mCodecInfo == nullptr) {
947 return NAME_NOT_FOUND;
948 }
949
950 mCodec = GetCodecBase(name, mCodecInfo->getOwnerName());
951 if (mCodec == NULL) {
952 return NAME_NOT_FOUND;
953 }
954
955 if (mIsVideo) {
956 // video codec needs dedicated looper
957 if (mCodecLooper == NULL) {
958 mCodecLooper = new ALooper;
959 mCodecLooper->setName("CodecLooper");
960 mCodecLooper->start(false, false, ANDROID_PRIORITY_AUDIO);
961 }
962
963 mCodecLooper->registerHandler(mCodec);
964 } else {
965 mLooper->registerHandler(mCodec);
966 }
967
968 mLooper->registerHandler(this);
969
970 mCodec->setCallback(
971 std::unique_ptr<CodecBase::CodecCallback>(
972 new CodecCallback(new AMessage(kWhatCodecNotify, this))));
973 mBufferChannel = mCodec->getBufferChannel();
974 mBufferChannel->setCallback(
975 std::unique_ptr<CodecBase::BufferCallback>(
976 new BufferCallback(new AMessage(kWhatCodecNotify, this))));
977
978 sp<AMessage> msg = new AMessage(kWhatInit, this);
979 msg->setObject("codecInfo", mCodecInfo);
980 // name may be different from mCodecInfo->getCodecName() if we stripped
981 // ".secure"
982 msg->setString("name", name);
983
984 if (mMetricsHandle != 0) {
985 mediametrics_setCString(mMetricsHandle, kCodecCodec, name.c_str());
986 mediametrics_setCString(mMetricsHandle, kCodecMode,
987 mIsVideo ? kCodecModeVideo : kCodecModeAudio);
988 }
989
990 if (mIsVideo) {
991 mBatteryChecker = new BatteryChecker(new AMessage(kWhatCheckBatteryStats, this));
992 }
993
994 status_t err;
995 Vector<MediaResource> resources;
996 MediaResource::Type type =
997 secureCodec ? MediaResource::kSecureCodec : MediaResource::kNonSecureCodec;
998 MediaResource::SubType subtype =
999 mIsVideo ? MediaResource::kVideoCodec : MediaResource::kAudioCodec;
1000 resources.push_back(MediaResource(type, subtype, 1));
1001 for (int i = 0; i <= kMaxRetry; ++i) {
1002 if (i > 0) {
1003 // Don't try to reclaim resource for the first time.
1004 if (!mResourceManagerService->reclaimResource(resources)) {
1005 break;
1006 }
1007 }
1008
1009 sp<AMessage> response;
1010 err = PostAndAwaitResponse(msg, &response);
1011 if (!isResourceError(err)) {
1012 break;
1013 }
1014 }
1015 return err;
1016 }
1017
setCallback(const sp<AMessage> & callback)1018 status_t MediaCodec::setCallback(const sp<AMessage> &callback) {
1019 sp<AMessage> msg = new AMessage(kWhatSetCallback, this);
1020 msg->setMessage("callback", callback);
1021
1022 sp<AMessage> response;
1023 return PostAndAwaitResponse(msg, &response);
1024 }
1025
setOnFrameRenderedNotification(const sp<AMessage> & notify)1026 status_t MediaCodec::setOnFrameRenderedNotification(const sp<AMessage> ¬ify) {
1027 sp<AMessage> msg = new AMessage(kWhatSetNotification, this);
1028 msg->setMessage("on-frame-rendered", notify);
1029 return msg->post();
1030 }
1031
configure(const sp<AMessage> & format,const sp<Surface> & nativeWindow,const sp<ICrypto> & crypto,uint32_t flags)1032 status_t MediaCodec::configure(
1033 const sp<AMessage> &format,
1034 const sp<Surface> &nativeWindow,
1035 const sp<ICrypto> &crypto,
1036 uint32_t flags) {
1037 return configure(format, nativeWindow, crypto, NULL, flags);
1038 }
1039
configure(const sp<AMessage> & format,const sp<Surface> & surface,const sp<ICrypto> & crypto,const sp<IDescrambler> & descrambler,uint32_t flags)1040 status_t MediaCodec::configure(
1041 const sp<AMessage> &format,
1042 const sp<Surface> &surface,
1043 const sp<ICrypto> &crypto,
1044 const sp<IDescrambler> &descrambler,
1045 uint32_t flags) {
1046 sp<AMessage> msg = new AMessage(kWhatConfigure, this);
1047
1048 if (mMetricsHandle != 0) {
1049 int32_t profile = 0;
1050 if (format->findInt32("profile", &profile)) {
1051 mediametrics_setInt32(mMetricsHandle, kCodecProfile, profile);
1052 }
1053 int32_t level = 0;
1054 if (format->findInt32("level", &level)) {
1055 mediametrics_setInt32(mMetricsHandle, kCodecLevel, level);
1056 }
1057 mediametrics_setInt32(mMetricsHandle, kCodecEncoder,
1058 (flags & CONFIGURE_FLAG_ENCODE) ? 1 : 0);
1059 }
1060
1061 if (mIsVideo) {
1062 format->findInt32("width", &mVideoWidth);
1063 format->findInt32("height", &mVideoHeight);
1064 if (!format->findInt32("rotation-degrees", &mRotationDegrees)) {
1065 mRotationDegrees = 0;
1066 }
1067
1068 if (mMetricsHandle != 0) {
1069 mediametrics_setInt32(mMetricsHandle, kCodecWidth, mVideoWidth);
1070 mediametrics_setInt32(mMetricsHandle, kCodecHeight, mVideoHeight);
1071 mediametrics_setInt32(mMetricsHandle, kCodecRotation, mRotationDegrees);
1072 int32_t maxWidth = 0;
1073 if (format->findInt32("max-width", &maxWidth)) {
1074 mediametrics_setInt32(mMetricsHandle, kCodecMaxWidth, maxWidth);
1075 }
1076 int32_t maxHeight = 0;
1077 if (format->findInt32("max-height", &maxHeight)) {
1078 mediametrics_setInt32(mMetricsHandle, kCodecMaxHeight, maxHeight);
1079 }
1080 }
1081
1082 // Prevent possible integer overflow in downstream code.
1083 if (mVideoWidth < 0 || mVideoHeight < 0 ||
1084 (uint64_t)mVideoWidth * mVideoHeight > (uint64_t)INT32_MAX / 4) {
1085 ALOGE("Invalid size(s), width=%d, height=%d", mVideoWidth, mVideoHeight);
1086 return BAD_VALUE;
1087 }
1088 }
1089
1090 msg->setMessage("format", format);
1091 msg->setInt32("flags", flags);
1092 msg->setObject("surface", surface);
1093
1094 if (crypto != NULL || descrambler != NULL) {
1095 if (crypto != NULL) {
1096 msg->setPointer("crypto", crypto.get());
1097 } else {
1098 msg->setPointer("descrambler", descrambler.get());
1099 }
1100 if (mMetricsHandle != 0) {
1101 mediametrics_setInt32(mMetricsHandle, kCodecCrypto, 1);
1102 }
1103 } else if (mFlags & kFlagIsSecure) {
1104 ALOGW("Crypto or descrambler should be given for secure codec");
1105 }
1106
1107 // save msg for reset
1108 mConfigureMsg = msg;
1109
1110 status_t err;
1111 Vector<MediaResource> resources;
1112 MediaResource::Type type = (mFlags & kFlagIsSecure) ?
1113 MediaResource::kSecureCodec : MediaResource::kNonSecureCodec;
1114 MediaResource::SubType subtype =
1115 mIsVideo ? MediaResource::kVideoCodec : MediaResource::kAudioCodec;
1116 resources.push_back(MediaResource(type, subtype, 1));
1117 // Don't know the buffer size at this point, but it's fine to use 1 because
1118 // the reclaimResource call doesn't consider the requester's buffer size for now.
1119 resources.push_back(MediaResource(MediaResource::kGraphicMemory, 1));
1120 for (int i = 0; i <= kMaxRetry; ++i) {
1121 if (i > 0) {
1122 // Don't try to reclaim resource for the first time.
1123 if (!mResourceManagerService->reclaimResource(resources)) {
1124 break;
1125 }
1126 }
1127
1128 sp<AMessage> response;
1129 err = PostAndAwaitResponse(msg, &response);
1130 if (err != OK && err != INVALID_OPERATION) {
1131 // MediaCodec now set state to UNINITIALIZED upon any fatal error.
1132 // To maintain backward-compatibility, do a reset() to put codec
1133 // back into INITIALIZED state.
1134 // But don't reset if the err is INVALID_OPERATION, which means
1135 // the configure failure is due to wrong state.
1136
1137 ALOGE("configure failed with err 0x%08x, resetting...", err);
1138 reset();
1139 }
1140 if (!isResourceError(err)) {
1141 break;
1142 }
1143 }
1144
1145 return err;
1146 }
1147
releaseCrypto()1148 status_t MediaCodec::releaseCrypto()
1149 {
1150 ALOGV("releaseCrypto");
1151
1152 sp<AMessage> msg = new AMessage(kWhatDrmReleaseCrypto, this);
1153
1154 sp<AMessage> response;
1155 status_t status = msg->postAndAwaitResponse(&response);
1156
1157 if (status == OK && response != NULL) {
1158 CHECK(response->findInt32("status", &status));
1159 ALOGV("releaseCrypto ret: %d ", status);
1160 }
1161 else {
1162 ALOGE("releaseCrypto err: %d", status);
1163 }
1164
1165 return status;
1166 }
1167
onReleaseCrypto(const sp<AMessage> & msg)1168 void MediaCodec::onReleaseCrypto(const sp<AMessage>& msg)
1169 {
1170 status_t status = INVALID_OPERATION;
1171 if (mCrypto != NULL) {
1172 ALOGV("onReleaseCrypto: mCrypto: %p (%d)", mCrypto.get(), mCrypto->getStrongCount());
1173 mBufferChannel->setCrypto(NULL);
1174 // TODO change to ALOGV
1175 ALOGD("onReleaseCrypto: [before clear] mCrypto: %p (%d)",
1176 mCrypto.get(), mCrypto->getStrongCount());
1177 mCrypto.clear();
1178
1179 status = OK;
1180 }
1181 else {
1182 ALOGW("onReleaseCrypto: No mCrypto. err: %d", status);
1183 }
1184
1185 sp<AMessage> response = new AMessage;
1186 response->setInt32("status", status);
1187
1188 sp<AReplyToken> replyID;
1189 CHECK(msg->senderAwaitsResponse(&replyID));
1190 response->postReply(replyID);
1191 }
1192
setInputSurface(const sp<PersistentSurface> & surface)1193 status_t MediaCodec::setInputSurface(
1194 const sp<PersistentSurface> &surface) {
1195 sp<AMessage> msg = new AMessage(kWhatSetInputSurface, this);
1196 msg->setObject("input-surface", surface.get());
1197
1198 sp<AMessage> response;
1199 return PostAndAwaitResponse(msg, &response);
1200 }
1201
setSurface(const sp<Surface> & surface)1202 status_t MediaCodec::setSurface(const sp<Surface> &surface) {
1203 sp<AMessage> msg = new AMessage(kWhatSetSurface, this);
1204 msg->setObject("surface", surface);
1205
1206 sp<AMessage> response;
1207 return PostAndAwaitResponse(msg, &response);
1208 }
1209
createInputSurface(sp<IGraphicBufferProducer> * bufferProducer)1210 status_t MediaCodec::createInputSurface(
1211 sp<IGraphicBufferProducer>* bufferProducer) {
1212 sp<AMessage> msg = new AMessage(kWhatCreateInputSurface, this);
1213
1214 sp<AMessage> response;
1215 status_t err = PostAndAwaitResponse(msg, &response);
1216 if (err == NO_ERROR) {
1217 // unwrap the sp<IGraphicBufferProducer>
1218 sp<RefBase> obj;
1219 bool found = response->findObject("input-surface", &obj);
1220 CHECK(found);
1221 sp<BufferProducerWrapper> wrapper(
1222 static_cast<BufferProducerWrapper*>(obj.get()));
1223 *bufferProducer = wrapper->getBufferProducer();
1224 } else {
1225 ALOGW("createInputSurface failed, err=%d", err);
1226 }
1227 return err;
1228 }
1229
getGraphicBufferSize()1230 uint64_t MediaCodec::getGraphicBufferSize() {
1231 if (!mIsVideo) {
1232 return 0;
1233 }
1234
1235 uint64_t size = 0;
1236 size_t portNum = sizeof(mPortBuffers) / sizeof((mPortBuffers)[0]);
1237 for (size_t i = 0; i < portNum; ++i) {
1238 // TODO: this is just an estimation, we should get the real buffer size from ACodec.
1239 size += mPortBuffers[i].size() * mVideoWidth * mVideoHeight * 3 / 2;
1240 }
1241 return size;
1242 }
1243
addResource(MediaResource::Type type,MediaResource::SubType subtype,uint64_t value)1244 void MediaCodec::addResource(
1245 MediaResource::Type type, MediaResource::SubType subtype, uint64_t value) {
1246 Vector<MediaResource> resources;
1247 resources.push_back(MediaResource(type, subtype, value));
1248 mResourceManagerService->addResource(
1249 getId(mResourceManagerClient), mResourceManagerClient, resources);
1250 }
1251
removeResource(MediaResource::Type type,MediaResource::SubType subtype,uint64_t value)1252 void MediaCodec::removeResource(
1253 MediaResource::Type type, MediaResource::SubType subtype, uint64_t value) {
1254 Vector<MediaResource> resources;
1255 resources.push_back(MediaResource(type, subtype, value));
1256 mResourceManagerService->removeResource(getId(mResourceManagerClient), resources);
1257 }
1258
start()1259 status_t MediaCodec::start() {
1260 sp<AMessage> msg = new AMessage(kWhatStart, this);
1261
1262 status_t err;
1263 Vector<MediaResource> resources;
1264 MediaResource::Type type = (mFlags & kFlagIsSecure) ?
1265 MediaResource::kSecureCodec : MediaResource::kNonSecureCodec;
1266 MediaResource::SubType subtype =
1267 mIsVideo ? MediaResource::kVideoCodec : MediaResource::kAudioCodec;
1268 resources.push_back(MediaResource(type, subtype, 1));
1269 // Don't know the buffer size at this point, but it's fine to use 1 because
1270 // the reclaimResource call doesn't consider the requester's buffer size for now.
1271 resources.push_back(MediaResource(MediaResource::kGraphicMemory, 1));
1272 for (int i = 0; i <= kMaxRetry; ++i) {
1273 if (i > 0) {
1274 // Don't try to reclaim resource for the first time.
1275 if (!mResourceManagerService->reclaimResource(resources)) {
1276 break;
1277 }
1278 // Recover codec from previous error before retry start.
1279 err = reset();
1280 if (err != OK) {
1281 ALOGE("retrying start: failed to reset codec");
1282 break;
1283 }
1284 sp<AMessage> response;
1285 err = PostAndAwaitResponse(mConfigureMsg, &response);
1286 if (err != OK) {
1287 ALOGE("retrying start: failed to configure codec");
1288 break;
1289 }
1290 }
1291
1292 sp<AMessage> response;
1293 err = PostAndAwaitResponse(msg, &response);
1294 if (!isResourceError(err)) {
1295 break;
1296 }
1297 }
1298 return err;
1299 }
1300
stop()1301 status_t MediaCodec::stop() {
1302 sp<AMessage> msg = new AMessage(kWhatStop, this);
1303
1304 sp<AMessage> response;
1305 return PostAndAwaitResponse(msg, &response);
1306 }
1307
hasPendingBuffer(int portIndex)1308 bool MediaCodec::hasPendingBuffer(int portIndex) {
1309 return std::any_of(
1310 mPortBuffers[portIndex].begin(), mPortBuffers[portIndex].end(),
1311 [](const BufferInfo &info) { return info.mOwnedByClient; });
1312 }
1313
hasPendingBuffer()1314 bool MediaCodec::hasPendingBuffer() {
1315 return hasPendingBuffer(kPortIndexInput) || hasPendingBuffer(kPortIndexOutput);
1316 }
1317
reclaim(bool force)1318 status_t MediaCodec::reclaim(bool force) {
1319 ALOGD("MediaCodec::reclaim(%p) %s", this, mInitName.c_str());
1320 sp<AMessage> msg = new AMessage(kWhatRelease, this);
1321 msg->setInt32("reclaimed", 1);
1322 msg->setInt32("force", force ? 1 : 0);
1323
1324 sp<AMessage> response;
1325 status_t ret = PostAndAwaitResponse(msg, &response);
1326 if (ret == -ENOENT) {
1327 ALOGD("MediaCodec looper is gone, skip reclaim");
1328 ret = OK;
1329 }
1330 return ret;
1331 }
1332
release()1333 status_t MediaCodec::release() {
1334 sp<AMessage> msg = new AMessage(kWhatRelease, this);
1335
1336 sp<AMessage> response;
1337 return PostAndAwaitResponse(msg, &response);
1338 }
1339
reset()1340 status_t MediaCodec::reset() {
1341 /* When external-facing MediaCodec object is created,
1342 it is already initialized. Thus, reset is essentially
1343 release() followed by init(), plus clearing the state */
1344
1345 status_t err = release();
1346
1347 // unregister handlers
1348 if (mCodec != NULL) {
1349 if (mCodecLooper != NULL) {
1350 mCodecLooper->unregisterHandler(mCodec->id());
1351 } else {
1352 mLooper->unregisterHandler(mCodec->id());
1353 }
1354 mCodec = NULL;
1355 }
1356 mLooper->unregisterHandler(id());
1357
1358 mFlags = 0; // clear all flags
1359 mStickyError = OK;
1360
1361 // reset state not reset by setState(UNINITIALIZED)
1362 mReplyID = 0;
1363 mDequeueInputReplyID = 0;
1364 mDequeueOutputReplyID = 0;
1365 mDequeueInputTimeoutGeneration = 0;
1366 mDequeueOutputTimeoutGeneration = 0;
1367 mHaveInputSurface = false;
1368
1369 if (err == OK) {
1370 err = init(mInitName);
1371 }
1372 return err;
1373 }
1374
queueInputBuffer(size_t index,size_t offset,size_t size,int64_t presentationTimeUs,uint32_t flags,AString * errorDetailMsg)1375 status_t MediaCodec::queueInputBuffer(
1376 size_t index,
1377 size_t offset,
1378 size_t size,
1379 int64_t presentationTimeUs,
1380 uint32_t flags,
1381 AString *errorDetailMsg) {
1382 if (errorDetailMsg != NULL) {
1383 errorDetailMsg->clear();
1384 }
1385
1386 sp<AMessage> msg = new AMessage(kWhatQueueInputBuffer, this);
1387 msg->setSize("index", index);
1388 msg->setSize("offset", offset);
1389 msg->setSize("size", size);
1390 msg->setInt64("timeUs", presentationTimeUs);
1391 msg->setInt32("flags", flags);
1392 msg->setPointer("errorDetailMsg", errorDetailMsg);
1393
1394 sp<AMessage> response;
1395 return PostAndAwaitResponse(msg, &response);
1396 }
1397
queueSecureInputBuffer(size_t index,size_t offset,const CryptoPlugin::SubSample * subSamples,size_t numSubSamples,const uint8_t key[16],const uint8_t iv[16],CryptoPlugin::Mode mode,const CryptoPlugin::Pattern & pattern,int64_t presentationTimeUs,uint32_t flags,AString * errorDetailMsg)1398 status_t MediaCodec::queueSecureInputBuffer(
1399 size_t index,
1400 size_t offset,
1401 const CryptoPlugin::SubSample *subSamples,
1402 size_t numSubSamples,
1403 const uint8_t key[16],
1404 const uint8_t iv[16],
1405 CryptoPlugin::Mode mode,
1406 const CryptoPlugin::Pattern &pattern,
1407 int64_t presentationTimeUs,
1408 uint32_t flags,
1409 AString *errorDetailMsg) {
1410 if (errorDetailMsg != NULL) {
1411 errorDetailMsg->clear();
1412 }
1413
1414 sp<AMessage> msg = new AMessage(kWhatQueueInputBuffer, this);
1415 msg->setSize("index", index);
1416 msg->setSize("offset", offset);
1417 msg->setPointer("subSamples", (void *)subSamples);
1418 msg->setSize("numSubSamples", numSubSamples);
1419 msg->setPointer("key", (void *)key);
1420 msg->setPointer("iv", (void *)iv);
1421 msg->setInt32("mode", mode);
1422 msg->setInt32("encryptBlocks", pattern.mEncryptBlocks);
1423 msg->setInt32("skipBlocks", pattern.mSkipBlocks);
1424 msg->setInt64("timeUs", presentationTimeUs);
1425 msg->setInt32("flags", flags);
1426 msg->setPointer("errorDetailMsg", errorDetailMsg);
1427
1428 sp<AMessage> response;
1429 status_t err = PostAndAwaitResponse(msg, &response);
1430
1431 return err;
1432 }
1433
dequeueInputBuffer(size_t * index,int64_t timeoutUs)1434 status_t MediaCodec::dequeueInputBuffer(size_t *index, int64_t timeoutUs) {
1435 sp<AMessage> msg = new AMessage(kWhatDequeueInputBuffer, this);
1436 msg->setInt64("timeoutUs", timeoutUs);
1437
1438 sp<AMessage> response;
1439 status_t err;
1440 if ((err = PostAndAwaitResponse(msg, &response)) != OK) {
1441 return err;
1442 }
1443
1444 CHECK(response->findSize("index", index));
1445
1446 return OK;
1447 }
1448
dequeueOutputBuffer(size_t * index,size_t * offset,size_t * size,int64_t * presentationTimeUs,uint32_t * flags,int64_t timeoutUs)1449 status_t MediaCodec::dequeueOutputBuffer(
1450 size_t *index,
1451 size_t *offset,
1452 size_t *size,
1453 int64_t *presentationTimeUs,
1454 uint32_t *flags,
1455 int64_t timeoutUs) {
1456 sp<AMessage> msg = new AMessage(kWhatDequeueOutputBuffer, this);
1457 msg->setInt64("timeoutUs", timeoutUs);
1458
1459 sp<AMessage> response;
1460 status_t err;
1461 if ((err = PostAndAwaitResponse(msg, &response)) != OK) {
1462 return err;
1463 }
1464
1465 CHECK(response->findSize("index", index));
1466 CHECK(response->findSize("offset", offset));
1467 CHECK(response->findSize("size", size));
1468 CHECK(response->findInt64("timeUs", presentationTimeUs));
1469 CHECK(response->findInt32("flags", (int32_t *)flags));
1470
1471 return OK;
1472 }
1473
renderOutputBufferAndRelease(size_t index)1474 status_t MediaCodec::renderOutputBufferAndRelease(size_t index) {
1475 sp<AMessage> msg = new AMessage(kWhatReleaseOutputBuffer, this);
1476 msg->setSize("index", index);
1477 msg->setInt32("render", true);
1478
1479 sp<AMessage> response;
1480 return PostAndAwaitResponse(msg, &response);
1481 }
1482
renderOutputBufferAndRelease(size_t index,int64_t timestampNs)1483 status_t MediaCodec::renderOutputBufferAndRelease(size_t index, int64_t timestampNs) {
1484 sp<AMessage> msg = new AMessage(kWhatReleaseOutputBuffer, this);
1485 msg->setSize("index", index);
1486 msg->setInt32("render", true);
1487 msg->setInt64("timestampNs", timestampNs);
1488
1489 sp<AMessage> response;
1490 return PostAndAwaitResponse(msg, &response);
1491 }
1492
releaseOutputBuffer(size_t index)1493 status_t MediaCodec::releaseOutputBuffer(size_t index) {
1494 sp<AMessage> msg = new AMessage(kWhatReleaseOutputBuffer, this);
1495 msg->setSize("index", index);
1496
1497 sp<AMessage> response;
1498 return PostAndAwaitResponse(msg, &response);
1499 }
1500
signalEndOfInputStream()1501 status_t MediaCodec::signalEndOfInputStream() {
1502 sp<AMessage> msg = new AMessage(kWhatSignalEndOfInputStream, this);
1503
1504 sp<AMessage> response;
1505 return PostAndAwaitResponse(msg, &response);
1506 }
1507
getOutputFormat(sp<AMessage> * format) const1508 status_t MediaCodec::getOutputFormat(sp<AMessage> *format) const {
1509 sp<AMessage> msg = new AMessage(kWhatGetOutputFormat, this);
1510
1511 sp<AMessage> response;
1512 status_t err;
1513 if ((err = PostAndAwaitResponse(msg, &response)) != OK) {
1514 return err;
1515 }
1516
1517 CHECK(response->findMessage("format", format));
1518
1519 return OK;
1520 }
1521
getInputFormat(sp<AMessage> * format) const1522 status_t MediaCodec::getInputFormat(sp<AMessage> *format) const {
1523 sp<AMessage> msg = new AMessage(kWhatGetInputFormat, this);
1524
1525 sp<AMessage> response;
1526 status_t err;
1527 if ((err = PostAndAwaitResponse(msg, &response)) != OK) {
1528 return err;
1529 }
1530
1531 CHECK(response->findMessage("format", format));
1532
1533 return OK;
1534 }
1535
getName(AString * name) const1536 status_t MediaCodec::getName(AString *name) const {
1537 sp<AMessage> msg = new AMessage(kWhatGetName, this);
1538
1539 sp<AMessage> response;
1540 status_t err;
1541 if ((err = PostAndAwaitResponse(msg, &response)) != OK) {
1542 return err;
1543 }
1544
1545 CHECK(response->findString("name", name));
1546
1547 return OK;
1548 }
1549
getCodecInfo(sp<MediaCodecInfo> * codecInfo) const1550 status_t MediaCodec::getCodecInfo(sp<MediaCodecInfo> *codecInfo) const {
1551 sp<AMessage> msg = new AMessage(kWhatGetCodecInfo, this);
1552
1553 sp<AMessage> response;
1554 status_t err;
1555 if ((err = PostAndAwaitResponse(msg, &response)) != OK) {
1556 return err;
1557 }
1558
1559 sp<RefBase> obj;
1560 CHECK(response->findObject("codecInfo", &obj));
1561 *codecInfo = static_cast<MediaCodecInfo *>(obj.get());
1562
1563 return OK;
1564 }
1565
getMetrics(mediametrics_handle_t & reply)1566 status_t MediaCodec::getMetrics(mediametrics_handle_t &reply) {
1567
1568 reply = 0;
1569
1570 // shouldn't happen, but be safe
1571 if (mMetricsHandle == 0) {
1572 return UNKNOWN_ERROR;
1573 }
1574
1575 // update any in-flight data that's not carried within the record
1576 updateMediametrics();
1577
1578 // send it back to the caller.
1579 reply = mediametrics_dup(mMetricsHandle);
1580
1581 updateEphemeralMediametrics(reply);
1582
1583 return OK;
1584 }
1585
getInputBuffers(Vector<sp<MediaCodecBuffer>> * buffers) const1586 status_t MediaCodec::getInputBuffers(Vector<sp<MediaCodecBuffer> > *buffers) const {
1587 sp<AMessage> msg = new AMessage(kWhatGetBuffers, this);
1588 msg->setInt32("portIndex", kPortIndexInput);
1589 msg->setPointer("buffers", buffers);
1590
1591 sp<AMessage> response;
1592 return PostAndAwaitResponse(msg, &response);
1593 }
1594
getOutputBuffers(Vector<sp<MediaCodecBuffer>> * buffers) const1595 status_t MediaCodec::getOutputBuffers(Vector<sp<MediaCodecBuffer> > *buffers) const {
1596 sp<AMessage> msg = new AMessage(kWhatGetBuffers, this);
1597 msg->setInt32("portIndex", kPortIndexOutput);
1598 msg->setPointer("buffers", buffers);
1599
1600 sp<AMessage> response;
1601 return PostAndAwaitResponse(msg, &response);
1602 }
1603
getOutputBuffer(size_t index,sp<MediaCodecBuffer> * buffer)1604 status_t MediaCodec::getOutputBuffer(size_t index, sp<MediaCodecBuffer> *buffer) {
1605 sp<AMessage> format;
1606 return getBufferAndFormat(kPortIndexOutput, index, buffer, &format);
1607 }
1608
getOutputFormat(size_t index,sp<AMessage> * format)1609 status_t MediaCodec::getOutputFormat(size_t index, sp<AMessage> *format) {
1610 sp<MediaCodecBuffer> buffer;
1611 return getBufferAndFormat(kPortIndexOutput, index, &buffer, format);
1612 }
1613
getInputBuffer(size_t index,sp<MediaCodecBuffer> * buffer)1614 status_t MediaCodec::getInputBuffer(size_t index, sp<MediaCodecBuffer> *buffer) {
1615 sp<AMessage> format;
1616 return getBufferAndFormat(kPortIndexInput, index, buffer, &format);
1617 }
1618
isExecuting() const1619 bool MediaCodec::isExecuting() const {
1620 return mState == STARTED || mState == FLUSHED;
1621 }
1622
getBufferAndFormat(size_t portIndex,size_t index,sp<MediaCodecBuffer> * buffer,sp<AMessage> * format)1623 status_t MediaCodec::getBufferAndFormat(
1624 size_t portIndex, size_t index,
1625 sp<MediaCodecBuffer> *buffer, sp<AMessage> *format) {
1626 // use mutex instead of a context switch
1627 if (mReleasedByResourceManager) {
1628 ALOGE("getBufferAndFormat - resource already released");
1629 return DEAD_OBJECT;
1630 }
1631
1632 if (buffer == NULL) {
1633 ALOGE("getBufferAndFormat - null MediaCodecBuffer");
1634 return INVALID_OPERATION;
1635 }
1636
1637 if (format == NULL) {
1638 ALOGE("getBufferAndFormat - null AMessage");
1639 return INVALID_OPERATION;
1640 }
1641
1642 buffer->clear();
1643 format->clear();
1644
1645 if (!isExecuting()) {
1646 ALOGE("getBufferAndFormat - not executing");
1647 return INVALID_OPERATION;
1648 }
1649
1650 // we do not want mPortBuffers to change during this section
1651 // we also don't want mOwnedByClient to change during this
1652 Mutex::Autolock al(mBufferLock);
1653
1654 std::vector<BufferInfo> &buffers = mPortBuffers[portIndex];
1655 if (index >= buffers.size()) {
1656 ALOGE("getBufferAndFormat - trying to get buffer with "
1657 "bad index (index=%zu buffer_size=%zu)", index, buffers.size());
1658 return INVALID_OPERATION;
1659 }
1660
1661 const BufferInfo &info = buffers[index];
1662 if (!info.mOwnedByClient) {
1663 ALOGE("getBufferAndFormat - invalid operation "
1664 "(the index %zu is not owned by client)", index);
1665 return INVALID_OPERATION;
1666 }
1667
1668 *buffer = info.mData;
1669 *format = info.mData->format();
1670
1671 return OK;
1672 }
1673
flush()1674 status_t MediaCodec::flush() {
1675 sp<AMessage> msg = new AMessage(kWhatFlush, this);
1676
1677 sp<AMessage> response;
1678 return PostAndAwaitResponse(msg, &response);
1679 }
1680
requestIDRFrame()1681 status_t MediaCodec::requestIDRFrame() {
1682 (new AMessage(kWhatRequestIDRFrame, this))->post();
1683
1684 return OK;
1685 }
1686
requestActivityNotification(const sp<AMessage> & notify)1687 void MediaCodec::requestActivityNotification(const sp<AMessage> ¬ify) {
1688 sp<AMessage> msg = new AMessage(kWhatRequestActivityNotification, this);
1689 msg->setMessage("notify", notify);
1690 msg->post();
1691 }
1692
requestCpuBoostIfNeeded()1693 void MediaCodec::requestCpuBoostIfNeeded() {
1694 if (mCpuBoostRequested) {
1695 return;
1696 }
1697 int32_t colorFormat;
1698 if (mOutputFormat->contains("hdr-static-info")
1699 && mOutputFormat->findInt32("color-format", &colorFormat)
1700 // check format for OMX only, for C2 the format is always opaque since the
1701 // software rendering doesn't go through client
1702 && ((mSoftRenderer != NULL && colorFormat == OMX_COLOR_FormatYUV420Planar16)
1703 || mOwnerName.equalsIgnoreCase("codec2::software"))) {
1704 int32_t left, top, right, bottom, width, height;
1705 int64_t totalPixel = 0;
1706 if (mOutputFormat->findRect("crop", &left, &top, &right, &bottom)) {
1707 totalPixel = (right - left + 1) * (bottom - top + 1);
1708 } else if (mOutputFormat->findInt32("width", &width)
1709 && mOutputFormat->findInt32("height", &height)) {
1710 totalPixel = width * height;
1711 }
1712 if (totalPixel >= 1920 * 1080) {
1713 addResource(MediaResource::kCpuBoost,
1714 MediaResource::kUnspecifiedSubType, 1);
1715 mCpuBoostRequested = true;
1716 }
1717 }
1718 }
1719
BatteryChecker(const sp<AMessage> & msg,int64_t timeoutUs)1720 BatteryChecker::BatteryChecker(const sp<AMessage> &msg, int64_t timeoutUs)
1721 : mTimeoutUs(timeoutUs)
1722 , mLastActivityTimeUs(-1ll)
1723 , mBatteryStatNotified(false)
1724 , mBatteryCheckerGeneration(0)
1725 , mIsExecuting(false)
1726 , mBatteryCheckerMsg(msg) {}
1727
onCodecActivity(std::function<void ()> batteryOnCb)1728 void BatteryChecker::onCodecActivity(std::function<void()> batteryOnCb) {
1729 if (!isExecuting()) {
1730 // ignore if not executing
1731 return;
1732 }
1733 if (!mBatteryStatNotified) {
1734 batteryOnCb();
1735 mBatteryStatNotified = true;
1736 sp<AMessage> msg = mBatteryCheckerMsg->dup();
1737 msg->setInt32("generation", mBatteryCheckerGeneration);
1738
1739 // post checker and clear last activity time
1740 msg->post(mTimeoutUs);
1741 mLastActivityTimeUs = -1ll;
1742 } else {
1743 // update last activity time
1744 mLastActivityTimeUs = ALooper::GetNowUs();
1745 }
1746 }
1747
onCheckBatteryTimer(const sp<AMessage> & msg,std::function<void ()> batteryOffCb)1748 void BatteryChecker::onCheckBatteryTimer(
1749 const sp<AMessage> &msg, std::function<void()> batteryOffCb) {
1750 // ignore if this checker already expired because the client resource was removed
1751 int32_t generation;
1752 if (!msg->findInt32("generation", &generation)
1753 || generation != mBatteryCheckerGeneration) {
1754 return;
1755 }
1756
1757 if (mLastActivityTimeUs < 0ll) {
1758 // timed out inactive, do not repost checker
1759 batteryOffCb();
1760 mBatteryStatNotified = false;
1761 } else {
1762 // repost checker and clear last activity time
1763 msg->post(mTimeoutUs + mLastActivityTimeUs - ALooper::GetNowUs());
1764 mLastActivityTimeUs = -1ll;
1765 }
1766 }
1767
onClientRemoved()1768 void BatteryChecker::onClientRemoved() {
1769 mBatteryStatNotified = false;
1770 mBatteryCheckerGeneration++;
1771 }
1772
1773 ////////////////////////////////////////////////////////////////////////////////
1774
cancelPendingDequeueOperations()1775 void MediaCodec::cancelPendingDequeueOperations() {
1776 if (mFlags & kFlagDequeueInputPending) {
1777 PostReplyWithError(mDequeueInputReplyID, INVALID_OPERATION);
1778
1779 ++mDequeueInputTimeoutGeneration;
1780 mDequeueInputReplyID = 0;
1781 mFlags &= ~kFlagDequeueInputPending;
1782 }
1783
1784 if (mFlags & kFlagDequeueOutputPending) {
1785 PostReplyWithError(mDequeueOutputReplyID, INVALID_OPERATION);
1786
1787 ++mDequeueOutputTimeoutGeneration;
1788 mDequeueOutputReplyID = 0;
1789 mFlags &= ~kFlagDequeueOutputPending;
1790 }
1791 }
1792
handleDequeueInputBuffer(const sp<AReplyToken> & replyID,bool newRequest)1793 bool MediaCodec::handleDequeueInputBuffer(const sp<AReplyToken> &replyID, bool newRequest) {
1794 if (!isExecuting() || (mFlags & kFlagIsAsync)
1795 || (newRequest && (mFlags & kFlagDequeueInputPending))) {
1796 PostReplyWithError(replyID, INVALID_OPERATION);
1797 return true;
1798 } else if (mFlags & kFlagStickyError) {
1799 PostReplyWithError(replyID, getStickyError());
1800 return true;
1801 }
1802
1803 ssize_t index = dequeuePortBuffer(kPortIndexInput);
1804
1805 if (index < 0) {
1806 CHECK_EQ(index, -EAGAIN);
1807 return false;
1808 }
1809
1810 sp<AMessage> response = new AMessage;
1811 response->setSize("index", index);
1812 response->postReply(replyID);
1813
1814 return true;
1815 }
1816
handleDequeueOutputBuffer(const sp<AReplyToken> & replyID,bool newRequest)1817 bool MediaCodec::handleDequeueOutputBuffer(const sp<AReplyToken> &replyID, bool newRequest) {
1818 if (!isExecuting() || (mFlags & kFlagIsAsync)
1819 || (newRequest && (mFlags & kFlagDequeueOutputPending))) {
1820 PostReplyWithError(replyID, INVALID_OPERATION);
1821 } else if (mFlags & kFlagStickyError) {
1822 PostReplyWithError(replyID, getStickyError());
1823 } else if (mFlags & kFlagOutputBuffersChanged) {
1824 PostReplyWithError(replyID, INFO_OUTPUT_BUFFERS_CHANGED);
1825 mFlags &= ~kFlagOutputBuffersChanged;
1826 } else if (mFlags & kFlagOutputFormatChanged) {
1827 PostReplyWithError(replyID, INFO_FORMAT_CHANGED);
1828 mFlags &= ~kFlagOutputFormatChanged;
1829 } else {
1830 sp<AMessage> response = new AMessage;
1831 ssize_t index = dequeuePortBuffer(kPortIndexOutput);
1832
1833 if (index < 0) {
1834 CHECK_EQ(index, -EAGAIN);
1835 return false;
1836 }
1837
1838 const sp<MediaCodecBuffer> &buffer =
1839 mPortBuffers[kPortIndexOutput][index].mData;
1840
1841 response->setSize("index", index);
1842 response->setSize("offset", buffer->offset());
1843 response->setSize("size", buffer->size());
1844
1845 int64_t timeUs;
1846 CHECK(buffer->meta()->findInt64("timeUs", &timeUs));
1847
1848 statsBufferReceived(timeUs);
1849
1850 response->setInt64("timeUs", timeUs);
1851
1852 int32_t flags;
1853 CHECK(buffer->meta()->findInt32("flags", &flags));
1854
1855 response->setInt32("flags", flags);
1856 response->postReply(replyID);
1857 }
1858
1859 return true;
1860 }
1861
onMessageReceived(const sp<AMessage> & msg)1862 void MediaCodec::onMessageReceived(const sp<AMessage> &msg) {
1863 switch (msg->what()) {
1864 case kWhatCodecNotify:
1865 {
1866 int32_t what;
1867 CHECK(msg->findInt32("what", &what));
1868
1869 switch (what) {
1870 case kWhatError:
1871 {
1872 int32_t err, actionCode;
1873 CHECK(msg->findInt32("err", &err));
1874 CHECK(msg->findInt32("actionCode", &actionCode));
1875
1876 ALOGE("Codec reported err %#x, actionCode %d, while in state %d",
1877 err, actionCode, mState);
1878 if (err == DEAD_OBJECT) {
1879 mFlags |= kFlagSawMediaServerDie;
1880 mFlags &= ~kFlagIsComponentAllocated;
1881 }
1882
1883 bool sendErrorResponse = true;
1884
1885 switch (mState) {
1886 case INITIALIZING:
1887 {
1888 setState(UNINITIALIZED);
1889 break;
1890 }
1891
1892 case CONFIGURING:
1893 {
1894 if (actionCode == ACTION_CODE_FATAL) {
1895 mediametrics_setInt32(mMetricsHandle, kCodecError, err);
1896 mediametrics_setCString(mMetricsHandle, kCodecErrorState,
1897 stateString(mState).c_str());
1898 flushMediametrics();
1899 initMediametrics();
1900 }
1901 setState(actionCode == ACTION_CODE_FATAL ?
1902 UNINITIALIZED : INITIALIZED);
1903 break;
1904 }
1905
1906 case STARTING:
1907 {
1908 if (actionCode == ACTION_CODE_FATAL) {
1909 mediametrics_setInt32(mMetricsHandle, kCodecError, err);
1910 mediametrics_setCString(mMetricsHandle, kCodecErrorState,
1911 stateString(mState).c_str());
1912 flushMediametrics();
1913 initMediametrics();
1914 }
1915 setState(actionCode == ACTION_CODE_FATAL ?
1916 UNINITIALIZED : CONFIGURED);
1917 break;
1918 }
1919
1920 case RELEASING:
1921 {
1922 // Ignore the error, assuming we'll still get
1923 // the shutdown complete notification. If we
1924 // don't, we'll timeout and force release.
1925 sendErrorResponse = false;
1926 FALLTHROUGH_INTENDED;
1927 }
1928 case STOPPING:
1929 {
1930 if (mFlags & kFlagSawMediaServerDie) {
1931 // MediaServer died, there definitely won't
1932 // be a shutdown complete notification after
1933 // all.
1934
1935 // note that we're directly going from
1936 // STOPPING->UNINITIALIZED, instead of the
1937 // usual STOPPING->INITIALIZED state.
1938 setState(UNINITIALIZED);
1939 if (mState == RELEASING) {
1940 mComponentName.clear();
1941 }
1942 (new AMessage)->postReply(mReplyID);
1943 sendErrorResponse = false;
1944 }
1945 break;
1946 }
1947
1948 case FLUSHING:
1949 {
1950 if (actionCode == ACTION_CODE_FATAL) {
1951 mediametrics_setInt32(mMetricsHandle, kCodecError, err);
1952 mediametrics_setCString(mMetricsHandle, kCodecErrorState,
1953 stateString(mState).c_str());
1954 flushMediametrics();
1955 initMediametrics();
1956
1957 setState(UNINITIALIZED);
1958 } else {
1959 setState(
1960 (mFlags & kFlagIsAsync) ? FLUSHED : STARTED);
1961 }
1962 break;
1963 }
1964
1965 case FLUSHED:
1966 case STARTED:
1967 {
1968 sendErrorResponse = false;
1969
1970 setStickyError(err);
1971 postActivityNotificationIfPossible();
1972
1973 cancelPendingDequeueOperations();
1974
1975 if (mFlags & kFlagIsAsync) {
1976 onError(err, actionCode);
1977 }
1978 switch (actionCode) {
1979 case ACTION_CODE_TRANSIENT:
1980 break;
1981 case ACTION_CODE_RECOVERABLE:
1982 setState(INITIALIZED);
1983 break;
1984 default:
1985 mediametrics_setInt32(mMetricsHandle, kCodecError, err);
1986 mediametrics_setCString(mMetricsHandle, kCodecErrorState,
1987 stateString(mState).c_str());
1988 flushMediametrics();
1989 initMediametrics();
1990 setState(UNINITIALIZED);
1991 break;
1992 }
1993 break;
1994 }
1995
1996 default:
1997 {
1998 sendErrorResponse = false;
1999
2000 setStickyError(err);
2001 postActivityNotificationIfPossible();
2002
2003 // actionCode in an uninitialized state is always fatal.
2004 if (mState == UNINITIALIZED) {
2005 actionCode = ACTION_CODE_FATAL;
2006 }
2007 if (mFlags & kFlagIsAsync) {
2008 onError(err, actionCode);
2009 }
2010 switch (actionCode) {
2011 case ACTION_CODE_TRANSIENT:
2012 break;
2013 case ACTION_CODE_RECOVERABLE:
2014 setState(INITIALIZED);
2015 break;
2016 default:
2017 setState(UNINITIALIZED);
2018 break;
2019 }
2020 break;
2021 }
2022 }
2023
2024 if (sendErrorResponse) {
2025 PostReplyWithError(mReplyID, err);
2026 }
2027 break;
2028 }
2029
2030 case kWhatComponentAllocated:
2031 {
2032 if (mState == RELEASING || mState == UNINITIALIZED) {
2033 // In case a kWhatError or kWhatRelease message came in and replied,
2034 // we log a warning and ignore.
2035 ALOGW("allocate interrupted by error or release, current state %d",
2036 mState);
2037 break;
2038 }
2039 CHECK_EQ(mState, INITIALIZING);
2040 setState(INITIALIZED);
2041 mFlags |= kFlagIsComponentAllocated;
2042
2043 CHECK(msg->findString("componentName", &mComponentName));
2044
2045 if (mComponentName.c_str()) {
2046 mediametrics_setCString(mMetricsHandle, kCodecCodec,
2047 mComponentName.c_str());
2048 }
2049
2050 const char *owner = mCodecInfo->getOwnerName();
2051 if (mComponentName.startsWith("OMX.google.")
2052 && (owner == nullptr || strncmp(owner, "default", 8) == 0)) {
2053 mFlags |= kFlagUsesSoftwareRenderer;
2054 } else {
2055 mFlags &= ~kFlagUsesSoftwareRenderer;
2056 }
2057 mOwnerName = owner;
2058
2059 MediaResource::Type resourceType;
2060 if (mComponentName.endsWith(".secure")) {
2061 mFlags |= kFlagIsSecure;
2062 resourceType = MediaResource::kSecureCodec;
2063 mediametrics_setInt32(mMetricsHandle, kCodecSecure, 1);
2064 } else {
2065 mFlags &= ~kFlagIsSecure;
2066 resourceType = MediaResource::kNonSecureCodec;
2067 mediametrics_setInt32(mMetricsHandle, kCodecSecure, 0);
2068 }
2069
2070 if (mIsVideo) {
2071 // audio codec is currently ignored.
2072 addResource(resourceType, MediaResource::kVideoCodec, 1);
2073 }
2074
2075 (new AMessage)->postReply(mReplyID);
2076 break;
2077 }
2078
2079 case kWhatComponentConfigured:
2080 {
2081 if (mState == RELEASING || mState == UNINITIALIZED || mState == INITIALIZED) {
2082 // In case a kWhatError or kWhatRelease message came in and replied,
2083 // we log a warning and ignore.
2084 ALOGW("configure interrupted by error or release, current state %d",
2085 mState);
2086 break;
2087 }
2088 CHECK_EQ(mState, CONFIGURING);
2089
2090 // reset input surface flag
2091 mHaveInputSurface = false;
2092
2093 CHECK(msg->findMessage("input-format", &mInputFormat));
2094 CHECK(msg->findMessage("output-format", &mOutputFormat));
2095
2096 // limit to confirming the opt-in behavior to minimize any behavioral change
2097 if (mSurface != nullptr && !mAllowFrameDroppingBySurface) {
2098 // signal frame dropping mode in the input format as this may also be
2099 // meaningful and confusing for an encoder in a transcoder scenario
2100 mInputFormat->setInt32("allow-frame-drop", mAllowFrameDroppingBySurface);
2101 }
2102 ALOGV("[%s] configured as input format: %s, output format: %s",
2103 mComponentName.c_str(),
2104 mInputFormat->debugString(4).c_str(),
2105 mOutputFormat->debugString(4).c_str());
2106 int32_t usingSwRenderer;
2107 if (mOutputFormat->findInt32("using-sw-renderer", &usingSwRenderer)
2108 && usingSwRenderer) {
2109 mFlags |= kFlagUsesSoftwareRenderer;
2110 }
2111 setState(CONFIGURED);
2112 (new AMessage)->postReply(mReplyID);
2113
2114 // augment our media metrics info, now that we know more things
2115 if (mMetricsHandle != 0) {
2116 sp<AMessage> format;
2117 if (mConfigureMsg != NULL &&
2118 mConfigureMsg->findMessage("format", &format)) {
2119 // format includes: mime
2120 AString mime;
2121 if (format->findString("mime", &mime)) {
2122 mediametrics_setCString(mMetricsHandle, kCodecMime,
2123 mime.c_str());
2124 }
2125 }
2126 }
2127 break;
2128 }
2129
2130 case kWhatInputSurfaceCreated:
2131 {
2132 // response to initiateCreateInputSurface()
2133 status_t err = NO_ERROR;
2134 sp<AMessage> response = new AMessage;
2135 if (!msg->findInt32("err", &err)) {
2136 sp<RefBase> obj;
2137 msg->findObject("input-surface", &obj);
2138 CHECK(msg->findMessage("input-format", &mInputFormat));
2139 CHECK(msg->findMessage("output-format", &mOutputFormat));
2140 ALOGV("[%s] input surface created as input format: %s, output format: %s",
2141 mComponentName.c_str(),
2142 mInputFormat->debugString(4).c_str(),
2143 mOutputFormat->debugString(4).c_str());
2144 CHECK(obj != NULL);
2145 response->setObject("input-surface", obj);
2146 mHaveInputSurface = true;
2147 } else {
2148 response->setInt32("err", err);
2149 }
2150 response->postReply(mReplyID);
2151 break;
2152 }
2153
2154 case kWhatInputSurfaceAccepted:
2155 {
2156 // response to initiateSetInputSurface()
2157 status_t err = NO_ERROR;
2158 sp<AMessage> response = new AMessage();
2159 if (!msg->findInt32("err", &err)) {
2160 CHECK(msg->findMessage("input-format", &mInputFormat));
2161 CHECK(msg->findMessage("output-format", &mOutputFormat));
2162 mHaveInputSurface = true;
2163 } else {
2164 response->setInt32("err", err);
2165 }
2166 response->postReply(mReplyID);
2167 break;
2168 }
2169
2170 case kWhatSignaledInputEOS:
2171 {
2172 // response to signalEndOfInputStream()
2173 sp<AMessage> response = new AMessage;
2174 status_t err;
2175 if (msg->findInt32("err", &err)) {
2176 response->setInt32("err", err);
2177 }
2178 response->postReply(mReplyID);
2179 break;
2180 }
2181
2182 case kWhatStartCompleted:
2183 {
2184 if (mState == RELEASING || mState == UNINITIALIZED) {
2185 // In case a kWhatRelease message came in and replied,
2186 // we log a warning and ignore.
2187 ALOGW("start interrupted by release, current state %d", mState);
2188 break;
2189 }
2190
2191 CHECK_EQ(mState, STARTING);
2192 if (mIsVideo) {
2193 addResource(
2194 MediaResource::kGraphicMemory,
2195 MediaResource::kUnspecifiedSubType,
2196 getGraphicBufferSize());
2197 }
2198 setState(STARTED);
2199 (new AMessage)->postReply(mReplyID);
2200 break;
2201 }
2202
2203 case kWhatOutputBuffersChanged:
2204 {
2205 mFlags |= kFlagOutputBuffersChanged;
2206 postActivityNotificationIfPossible();
2207 break;
2208 }
2209
2210 case kWhatOutputFramesRendered:
2211 {
2212 // ignore these in all states except running, and check that we have a
2213 // notification set
2214 if (mState == STARTED && mOnFrameRenderedNotification != NULL) {
2215 sp<AMessage> notify = mOnFrameRenderedNotification->dup();
2216 notify->setMessage("data", msg);
2217 notify->post();
2218 }
2219 break;
2220 }
2221
2222 case kWhatFillThisBuffer:
2223 {
2224 /* size_t index = */updateBuffers(kPortIndexInput, msg);
2225
2226 if (mState == FLUSHING
2227 || mState == STOPPING
2228 || mState == RELEASING) {
2229 returnBuffersToCodecOnPort(kPortIndexInput);
2230 break;
2231 }
2232
2233 if (!mCSD.empty()) {
2234 ssize_t index = dequeuePortBuffer(kPortIndexInput);
2235 CHECK_GE(index, 0);
2236
2237 // If codec specific data had been specified as
2238 // part of the format in the call to configure and
2239 // if there's more csd left, we submit it here
2240 // clients only get access to input buffers once
2241 // this data has been exhausted.
2242
2243 status_t err = queueCSDInputBuffer(index);
2244
2245 if (err != OK) {
2246 ALOGE("queueCSDInputBuffer failed w/ error %d",
2247 err);
2248
2249 setStickyError(err);
2250 postActivityNotificationIfPossible();
2251
2252 cancelPendingDequeueOperations();
2253 }
2254 break;
2255 }
2256
2257 if (mFlags & kFlagIsAsync) {
2258 if (!mHaveInputSurface) {
2259 if (mState == FLUSHED) {
2260 mHavePendingInputBuffers = true;
2261 } else {
2262 onInputBufferAvailable();
2263 }
2264 }
2265 } else if (mFlags & kFlagDequeueInputPending) {
2266 CHECK(handleDequeueInputBuffer(mDequeueInputReplyID));
2267
2268 ++mDequeueInputTimeoutGeneration;
2269 mFlags &= ~kFlagDequeueInputPending;
2270 mDequeueInputReplyID = 0;
2271 } else {
2272 postActivityNotificationIfPossible();
2273 }
2274 break;
2275 }
2276
2277 case kWhatDrainThisBuffer:
2278 {
2279 /* size_t index = */updateBuffers(kPortIndexOutput, msg);
2280
2281 if (mState == FLUSHING
2282 || mState == STOPPING
2283 || mState == RELEASING) {
2284 returnBuffersToCodecOnPort(kPortIndexOutput);
2285 break;
2286 }
2287
2288 sp<RefBase> obj;
2289 CHECK(msg->findObject("buffer", &obj));
2290 sp<MediaCodecBuffer> buffer = static_cast<MediaCodecBuffer *>(obj.get());
2291
2292 if (mOutputFormat != buffer->format()) {
2293 mOutputFormat = buffer->format();
2294 ALOGV("[%s] output format changed to: %s",
2295 mComponentName.c_str(), mOutputFormat->debugString(4).c_str());
2296
2297 if (mSoftRenderer == NULL &&
2298 mSurface != NULL &&
2299 (mFlags & kFlagUsesSoftwareRenderer)) {
2300 AString mime;
2301 CHECK(mOutputFormat->findString("mime", &mime));
2302
2303 // TODO: propagate color aspects to software renderer to allow better
2304 // color conversion to RGB. For now, just mark dataspace for YUV
2305 // rendering.
2306 int32_t dataSpace;
2307 if (mOutputFormat->findInt32("android._dataspace", &dataSpace)) {
2308 ALOGD("[%s] setting dataspace on output surface to #%x",
2309 mComponentName.c_str(), dataSpace);
2310 int err = native_window_set_buffers_data_space(
2311 mSurface.get(), (android_dataspace)dataSpace);
2312 ALOGW_IF(err != 0, "failed to set dataspace on surface (%d)", err);
2313 }
2314 if (mOutputFormat->contains("hdr-static-info")) {
2315 HDRStaticInfo info;
2316 if (ColorUtils::getHDRStaticInfoFromFormat(mOutputFormat, &info)) {
2317 setNativeWindowHdrMetadata(mSurface.get(), &info);
2318 }
2319 }
2320
2321 sp<ABuffer> hdr10PlusInfo;
2322 if (mOutputFormat->findBuffer("hdr10-plus-info", &hdr10PlusInfo)
2323 && hdr10PlusInfo != nullptr && hdr10PlusInfo->size() > 0) {
2324 native_window_set_buffers_hdr10_plus_metadata(mSurface.get(),
2325 hdr10PlusInfo->size(), hdr10PlusInfo->data());
2326 }
2327
2328 if (mime.startsWithIgnoreCase("video/")) {
2329 mSurface->setDequeueTimeout(-1);
2330 mSoftRenderer = new SoftwareRenderer(mSurface, mRotationDegrees);
2331 }
2332 }
2333
2334 requestCpuBoostIfNeeded();
2335
2336 if (mFlags & kFlagIsEncoder) {
2337 // Before we announce the format change we should
2338 // collect codec specific data and amend the output
2339 // format as necessary.
2340 int32_t flags = 0;
2341 (void) buffer->meta()->findInt32("flags", &flags);
2342 if (flags & BUFFER_FLAG_CODECCONFIG) {
2343 status_t err =
2344 amendOutputFormatWithCodecSpecificData(buffer);
2345
2346 if (err != OK) {
2347 ALOGE("Codec spit out malformed codec "
2348 "specific data!");
2349 }
2350 }
2351 }
2352 if (mFlags & kFlagIsAsync) {
2353 onOutputFormatChanged();
2354 } else {
2355 mFlags |= kFlagOutputFormatChanged;
2356 postActivityNotificationIfPossible();
2357 }
2358
2359 // Notify mCrypto of video resolution changes
2360 if (mCrypto != NULL) {
2361 int32_t left, top, right, bottom, width, height;
2362 if (mOutputFormat->findRect("crop", &left, &top, &right, &bottom)) {
2363 mCrypto->notifyResolution(right - left + 1, bottom - top + 1);
2364 } else if (mOutputFormat->findInt32("width", &width)
2365 && mOutputFormat->findInt32("height", &height)) {
2366 mCrypto->notifyResolution(width, height);
2367 }
2368 }
2369 }
2370
2371 if (mFlags & kFlagIsAsync) {
2372 onOutputBufferAvailable();
2373 } else if (mFlags & kFlagDequeueOutputPending) {
2374 CHECK(handleDequeueOutputBuffer(mDequeueOutputReplyID));
2375
2376 ++mDequeueOutputTimeoutGeneration;
2377 mFlags &= ~kFlagDequeueOutputPending;
2378 mDequeueOutputReplyID = 0;
2379 } else {
2380 postActivityNotificationIfPossible();
2381 }
2382
2383 break;
2384 }
2385
2386 case kWhatEOS:
2387 {
2388 // We already notify the client of this by using the
2389 // corresponding flag in "onOutputBufferReady".
2390 break;
2391 }
2392
2393 case kWhatStopCompleted:
2394 {
2395 if (mState != STOPPING) {
2396 ALOGW("Received kWhatStopCompleted in state %d", mState);
2397 break;
2398 }
2399 setState(INITIALIZED);
2400 (new AMessage)->postReply(mReplyID);
2401 break;
2402 }
2403
2404 case kWhatReleaseCompleted:
2405 {
2406 if (mState != RELEASING) {
2407 ALOGW("Received kWhatReleaseCompleted in state %d", mState);
2408 break;
2409 }
2410 setState(UNINITIALIZED);
2411 mComponentName.clear();
2412
2413 mFlags &= ~kFlagIsComponentAllocated;
2414
2415 // off since we're removing all resources including the battery on
2416 if (mBatteryChecker != nullptr) {
2417 mBatteryChecker->onClientRemoved();
2418 }
2419
2420 mResourceManagerService->removeClient(getId(mResourceManagerClient));
2421
2422 (new AMessage)->postReply(mReplyID);
2423 break;
2424 }
2425
2426 case kWhatFlushCompleted:
2427 {
2428 if (mState != FLUSHING) {
2429 ALOGW("received FlushCompleted message in state %d",
2430 mState);
2431 break;
2432 }
2433
2434 if (mFlags & kFlagIsAsync) {
2435 setState(FLUSHED);
2436 } else {
2437 setState(STARTED);
2438 mCodec->signalResume();
2439 }
2440
2441 (new AMessage)->postReply(mReplyID);
2442 break;
2443 }
2444
2445 default:
2446 TRESPASS();
2447 }
2448 break;
2449 }
2450
2451 case kWhatInit:
2452 {
2453 sp<AReplyToken> replyID;
2454 CHECK(msg->senderAwaitsResponse(&replyID));
2455
2456 if (mState != UNINITIALIZED) {
2457 PostReplyWithError(replyID, INVALID_OPERATION);
2458 break;
2459 }
2460
2461 mReplyID = replyID;
2462 setState(INITIALIZING);
2463
2464 sp<RefBase> codecInfo;
2465 CHECK(msg->findObject("codecInfo", &codecInfo));
2466 AString name;
2467 CHECK(msg->findString("name", &name));
2468
2469 sp<AMessage> format = new AMessage;
2470 format->setObject("codecInfo", codecInfo);
2471 format->setString("componentName", name);
2472
2473 mCodec->initiateAllocateComponent(format);
2474 break;
2475 }
2476
2477 case kWhatSetNotification:
2478 {
2479 sp<AMessage> notify;
2480 if (msg->findMessage("on-frame-rendered", ¬ify)) {
2481 mOnFrameRenderedNotification = notify;
2482 }
2483 break;
2484 }
2485
2486 case kWhatSetCallback:
2487 {
2488 sp<AReplyToken> replyID;
2489 CHECK(msg->senderAwaitsResponse(&replyID));
2490
2491 if (mState == UNINITIALIZED
2492 || mState == INITIALIZING
2493 || isExecuting()) {
2494 // callback can't be set after codec is executing,
2495 // or before it's initialized (as the callback
2496 // will be cleared when it goes to INITIALIZED)
2497 PostReplyWithError(replyID, INVALID_OPERATION);
2498 break;
2499 }
2500
2501 sp<AMessage> callback;
2502 CHECK(msg->findMessage("callback", &callback));
2503
2504 mCallback = callback;
2505
2506 if (mCallback != NULL) {
2507 ALOGI("MediaCodec will operate in async mode");
2508 mFlags |= kFlagIsAsync;
2509 } else {
2510 mFlags &= ~kFlagIsAsync;
2511 }
2512
2513 sp<AMessage> response = new AMessage;
2514 response->postReply(replyID);
2515 break;
2516 }
2517
2518 case kWhatConfigure:
2519 {
2520 sp<AReplyToken> replyID;
2521 CHECK(msg->senderAwaitsResponse(&replyID));
2522
2523 if (mState != INITIALIZED) {
2524 PostReplyWithError(replyID, INVALID_OPERATION);
2525 break;
2526 }
2527
2528 sp<RefBase> obj;
2529 CHECK(msg->findObject("surface", &obj));
2530
2531 sp<AMessage> format;
2532 CHECK(msg->findMessage("format", &format));
2533
2534 int32_t push;
2535 if (msg->findInt32("push-blank-buffers-on-shutdown", &push) && push != 0) {
2536 mFlags |= kFlagPushBlankBuffersOnShutdown;
2537 }
2538
2539 if (obj != NULL) {
2540 if (!format->findInt32("allow-frame-drop", &mAllowFrameDroppingBySurface)) {
2541 // allow frame dropping by surface by default
2542 mAllowFrameDroppingBySurface = true;
2543 }
2544
2545 format->setObject("native-window", obj);
2546 status_t err = handleSetSurface(static_cast<Surface *>(obj.get()));
2547 if (err != OK) {
2548 PostReplyWithError(replyID, err);
2549 break;
2550 }
2551 } else {
2552 // we are not using surface so this variable is not used, but initialize sensibly anyway
2553 mAllowFrameDroppingBySurface = false;
2554
2555 handleSetSurface(NULL);
2556 }
2557
2558 mReplyID = replyID;
2559 setState(CONFIGURING);
2560
2561 void *crypto;
2562 if (!msg->findPointer("crypto", &crypto)) {
2563 crypto = NULL;
2564 }
2565
2566 ALOGV("kWhatConfigure: Old mCrypto: %p (%d)",
2567 mCrypto.get(), (mCrypto != NULL ? mCrypto->getStrongCount() : 0));
2568
2569 mCrypto = static_cast<ICrypto *>(crypto);
2570 mBufferChannel->setCrypto(mCrypto);
2571
2572 ALOGV("kWhatConfigure: New mCrypto: %p (%d)",
2573 mCrypto.get(), (mCrypto != NULL ? mCrypto->getStrongCount() : 0));
2574
2575 void *descrambler;
2576 if (!msg->findPointer("descrambler", &descrambler)) {
2577 descrambler = NULL;
2578 }
2579
2580 mDescrambler = static_cast<IDescrambler *>(descrambler);
2581 mBufferChannel->setDescrambler(mDescrambler);
2582
2583 uint32_t flags;
2584 CHECK(msg->findInt32("flags", (int32_t *)&flags));
2585
2586 if (flags & CONFIGURE_FLAG_ENCODE) {
2587 format->setInt32("encoder", true);
2588 mFlags |= kFlagIsEncoder;
2589 }
2590
2591 extractCSD(format);
2592
2593 mCodec->initiateConfigureComponent(format);
2594 break;
2595 }
2596
2597 case kWhatSetSurface:
2598 {
2599 sp<AReplyToken> replyID;
2600 CHECK(msg->senderAwaitsResponse(&replyID));
2601
2602 status_t err = OK;
2603
2604 switch (mState) {
2605 case CONFIGURED:
2606 case STARTED:
2607 case FLUSHED:
2608 {
2609 sp<RefBase> obj;
2610 (void)msg->findObject("surface", &obj);
2611 sp<Surface> surface = static_cast<Surface *>(obj.get());
2612 if (mSurface == NULL) {
2613 // do not support setting surface if it was not set
2614 err = INVALID_OPERATION;
2615 } else if (obj == NULL) {
2616 // do not support unsetting surface
2617 err = BAD_VALUE;
2618 } else {
2619 err = connectToSurface(surface);
2620 if (err == ALREADY_EXISTS) {
2621 // reconnecting to same surface
2622 err = OK;
2623 } else {
2624 if (err == OK) {
2625 if (mFlags & kFlagUsesSoftwareRenderer) {
2626 if (mSoftRenderer != NULL
2627 && (mFlags & kFlagPushBlankBuffersOnShutdown)) {
2628 pushBlankBuffersToNativeWindow(mSurface.get());
2629 }
2630 surface->setDequeueTimeout(-1);
2631 mSoftRenderer = new SoftwareRenderer(surface);
2632 // TODO: check if this was successful
2633 } else {
2634 err = mCodec->setSurface(surface);
2635 }
2636 }
2637 if (err == OK) {
2638 (void)disconnectFromSurface();
2639 mSurface = surface;
2640 }
2641 }
2642 }
2643 break;
2644 }
2645
2646 default:
2647 err = INVALID_OPERATION;
2648 break;
2649 }
2650
2651 PostReplyWithError(replyID, err);
2652 break;
2653 }
2654
2655 case kWhatCreateInputSurface:
2656 case kWhatSetInputSurface:
2657 {
2658 sp<AReplyToken> replyID;
2659 CHECK(msg->senderAwaitsResponse(&replyID));
2660
2661 // Must be configured, but can't have been started yet.
2662 if (mState != CONFIGURED) {
2663 PostReplyWithError(replyID, INVALID_OPERATION);
2664 break;
2665 }
2666
2667 mReplyID = replyID;
2668 if (msg->what() == kWhatCreateInputSurface) {
2669 mCodec->initiateCreateInputSurface();
2670 } else {
2671 sp<RefBase> obj;
2672 CHECK(msg->findObject("input-surface", &obj));
2673
2674 mCodec->initiateSetInputSurface(
2675 static_cast<PersistentSurface *>(obj.get()));
2676 }
2677 break;
2678 }
2679 case kWhatStart:
2680 {
2681 sp<AReplyToken> replyID;
2682 CHECK(msg->senderAwaitsResponse(&replyID));
2683
2684 if (mState == FLUSHED) {
2685 setState(STARTED);
2686 if (mHavePendingInputBuffers) {
2687 onInputBufferAvailable();
2688 mHavePendingInputBuffers = false;
2689 }
2690 mCodec->signalResume();
2691 PostReplyWithError(replyID, OK);
2692 break;
2693 } else if (mState != CONFIGURED) {
2694 PostReplyWithError(replyID, INVALID_OPERATION);
2695 break;
2696 }
2697
2698 mReplyID = replyID;
2699 setState(STARTING);
2700
2701 mCodec->initiateStart();
2702 break;
2703 }
2704
2705 case kWhatStop:
2706 case kWhatRelease:
2707 {
2708 State targetState =
2709 (msg->what() == kWhatStop) ? INITIALIZED : UNINITIALIZED;
2710
2711 sp<AReplyToken> replyID;
2712 CHECK(msg->senderAwaitsResponse(&replyID));
2713
2714 // already stopped/released
2715 if (mState == UNINITIALIZED && mReleasedByResourceManager) {
2716 sp<AMessage> response = new AMessage;
2717 response->setInt32("err", OK);
2718 response->postReply(replyID);
2719 break;
2720 }
2721
2722 int32_t reclaimed = 0;
2723 msg->findInt32("reclaimed", &reclaimed);
2724 if (reclaimed) {
2725 mReleasedByResourceManager = true;
2726
2727 int32_t force = 0;
2728 msg->findInt32("force", &force);
2729 if (!force && hasPendingBuffer()) {
2730 ALOGW("Can't reclaim codec right now due to pending buffers.");
2731
2732 // return WOULD_BLOCK to ask resource manager to retry later.
2733 sp<AMessage> response = new AMessage;
2734 response->setInt32("err", WOULD_BLOCK);
2735 response->postReply(replyID);
2736
2737 // notify the async client
2738 if (mFlags & kFlagIsAsync) {
2739 onError(DEAD_OBJECT, ACTION_CODE_FATAL);
2740 }
2741 break;
2742 }
2743 }
2744
2745 bool isReleasingAllocatedComponent =
2746 (mFlags & kFlagIsComponentAllocated) && targetState == UNINITIALIZED;
2747 if (!isReleasingAllocatedComponent // See 1
2748 && mState != INITIALIZED
2749 && mState != CONFIGURED && !isExecuting()) {
2750 // 1) Permit release to shut down the component if allocated.
2751 //
2752 // 2) We may be in "UNINITIALIZED" state already and
2753 // also shutdown the encoder/decoder without the
2754 // client being aware of this if media server died while
2755 // we were being stopped. The client would assume that
2756 // after stop() returned, it would be safe to call release()
2757 // and it should be in this case, no harm to allow a release()
2758 // if we're already uninitialized.
2759 sp<AMessage> response = new AMessage;
2760 // TODO: we shouldn't throw an exception for stop/release. Change this to wait until
2761 // the previous stop/release completes and then reply with OK.
2762 status_t err = mState == targetState ? OK : INVALID_OPERATION;
2763 response->setInt32("err", err);
2764 if (err == OK && targetState == UNINITIALIZED) {
2765 mComponentName.clear();
2766 }
2767 response->postReply(replyID);
2768 break;
2769 }
2770
2771 // If we're flushing, stopping, configuring or starting but
2772 // received a release request, post the reply for the pending call
2773 // first, and consider it done. The reply token will be replaced
2774 // after this, and we'll no longer be able to reply.
2775 if (mState == FLUSHING || mState == STOPPING
2776 || mState == CONFIGURING || mState == STARTING) {
2777 (new AMessage)->postReply(mReplyID);
2778 }
2779
2780 if (mFlags & kFlagSawMediaServerDie) {
2781 // It's dead, Jim. Don't expect initiateShutdown to yield
2782 // any useful results now...
2783 setState(UNINITIALIZED);
2784 if (targetState == UNINITIALIZED) {
2785 mComponentName.clear();
2786 }
2787 (new AMessage)->postReply(replyID);
2788 break;
2789 }
2790
2791 // If we already have an error, component may not be able to
2792 // complete the shutdown properly. If we're stopping, post the
2793 // reply now with an error to unblock the client, client can
2794 // release after the failure (instead of ANR).
2795 if (msg->what() == kWhatStop && (mFlags & kFlagStickyError)) {
2796 PostReplyWithError(replyID, getStickyError());
2797 break;
2798 }
2799
2800 mReplyID = replyID;
2801 setState(msg->what() == kWhatStop ? STOPPING : RELEASING);
2802
2803 mCodec->initiateShutdown(
2804 msg->what() == kWhatStop /* keepComponentAllocated */);
2805
2806 returnBuffersToCodec(reclaimed);
2807
2808 if (mSoftRenderer != NULL && (mFlags & kFlagPushBlankBuffersOnShutdown)) {
2809 pushBlankBuffersToNativeWindow(mSurface.get());
2810 }
2811
2812 break;
2813 }
2814
2815 case kWhatDequeueInputBuffer:
2816 {
2817 sp<AReplyToken> replyID;
2818 CHECK(msg->senderAwaitsResponse(&replyID));
2819
2820 if (mFlags & kFlagIsAsync) {
2821 ALOGE("dequeueInputBuffer can't be used in async mode");
2822 PostReplyWithError(replyID, INVALID_OPERATION);
2823 break;
2824 }
2825
2826 if (mHaveInputSurface) {
2827 ALOGE("dequeueInputBuffer can't be used with input surface");
2828 PostReplyWithError(replyID, INVALID_OPERATION);
2829 break;
2830 }
2831
2832 if (handleDequeueInputBuffer(replyID, true /* new request */)) {
2833 break;
2834 }
2835
2836 int64_t timeoutUs;
2837 CHECK(msg->findInt64("timeoutUs", &timeoutUs));
2838
2839 if (timeoutUs == 0LL) {
2840 PostReplyWithError(replyID, -EAGAIN);
2841 break;
2842 }
2843
2844 mFlags |= kFlagDequeueInputPending;
2845 mDequeueInputReplyID = replyID;
2846
2847 if (timeoutUs > 0LL) {
2848 sp<AMessage> timeoutMsg =
2849 new AMessage(kWhatDequeueInputTimedOut, this);
2850 timeoutMsg->setInt32(
2851 "generation", ++mDequeueInputTimeoutGeneration);
2852 timeoutMsg->post(timeoutUs);
2853 }
2854 break;
2855 }
2856
2857 case kWhatDequeueInputTimedOut:
2858 {
2859 int32_t generation;
2860 CHECK(msg->findInt32("generation", &generation));
2861
2862 if (generation != mDequeueInputTimeoutGeneration) {
2863 // Obsolete
2864 break;
2865 }
2866
2867 CHECK(mFlags & kFlagDequeueInputPending);
2868
2869 PostReplyWithError(mDequeueInputReplyID, -EAGAIN);
2870
2871 mFlags &= ~kFlagDequeueInputPending;
2872 mDequeueInputReplyID = 0;
2873 break;
2874 }
2875
2876 case kWhatQueueInputBuffer:
2877 {
2878 sp<AReplyToken> replyID;
2879 CHECK(msg->senderAwaitsResponse(&replyID));
2880
2881 if (!isExecuting()) {
2882 PostReplyWithError(replyID, INVALID_OPERATION);
2883 break;
2884 } else if (mFlags & kFlagStickyError) {
2885 PostReplyWithError(replyID, getStickyError());
2886 break;
2887 }
2888
2889 status_t err = onQueueInputBuffer(msg);
2890
2891 PostReplyWithError(replyID, err);
2892 break;
2893 }
2894
2895 case kWhatDequeueOutputBuffer:
2896 {
2897 sp<AReplyToken> replyID;
2898 CHECK(msg->senderAwaitsResponse(&replyID));
2899
2900 if (mFlags & kFlagIsAsync) {
2901 ALOGE("dequeueOutputBuffer can't be used in async mode");
2902 PostReplyWithError(replyID, INVALID_OPERATION);
2903 break;
2904 }
2905
2906 if (handleDequeueOutputBuffer(replyID, true /* new request */)) {
2907 break;
2908 }
2909
2910 int64_t timeoutUs;
2911 CHECK(msg->findInt64("timeoutUs", &timeoutUs));
2912
2913 if (timeoutUs == 0LL) {
2914 PostReplyWithError(replyID, -EAGAIN);
2915 break;
2916 }
2917
2918 mFlags |= kFlagDequeueOutputPending;
2919 mDequeueOutputReplyID = replyID;
2920
2921 if (timeoutUs > 0LL) {
2922 sp<AMessage> timeoutMsg =
2923 new AMessage(kWhatDequeueOutputTimedOut, this);
2924 timeoutMsg->setInt32(
2925 "generation", ++mDequeueOutputTimeoutGeneration);
2926 timeoutMsg->post(timeoutUs);
2927 }
2928 break;
2929 }
2930
2931 case kWhatDequeueOutputTimedOut:
2932 {
2933 int32_t generation;
2934 CHECK(msg->findInt32("generation", &generation));
2935
2936 if (generation != mDequeueOutputTimeoutGeneration) {
2937 // Obsolete
2938 break;
2939 }
2940
2941 CHECK(mFlags & kFlagDequeueOutputPending);
2942
2943 PostReplyWithError(mDequeueOutputReplyID, -EAGAIN);
2944
2945 mFlags &= ~kFlagDequeueOutputPending;
2946 mDequeueOutputReplyID = 0;
2947 break;
2948 }
2949
2950 case kWhatReleaseOutputBuffer:
2951 {
2952 sp<AReplyToken> replyID;
2953 CHECK(msg->senderAwaitsResponse(&replyID));
2954
2955 if (!isExecuting()) {
2956 PostReplyWithError(replyID, INVALID_OPERATION);
2957 break;
2958 } else if (mFlags & kFlagStickyError) {
2959 PostReplyWithError(replyID, getStickyError());
2960 break;
2961 }
2962
2963 status_t err = onReleaseOutputBuffer(msg);
2964
2965 PostReplyWithError(replyID, err);
2966 break;
2967 }
2968
2969 case kWhatSignalEndOfInputStream:
2970 {
2971 sp<AReplyToken> replyID;
2972 CHECK(msg->senderAwaitsResponse(&replyID));
2973
2974 if (!isExecuting() || !mHaveInputSurface) {
2975 PostReplyWithError(replyID, INVALID_OPERATION);
2976 break;
2977 } else if (mFlags & kFlagStickyError) {
2978 PostReplyWithError(replyID, getStickyError());
2979 break;
2980 }
2981
2982 mReplyID = replyID;
2983 mCodec->signalEndOfInputStream();
2984 break;
2985 }
2986
2987 case kWhatGetBuffers:
2988 {
2989 sp<AReplyToken> replyID;
2990 CHECK(msg->senderAwaitsResponse(&replyID));
2991 if (!isExecuting() || (mFlags & kFlagIsAsync)) {
2992 PostReplyWithError(replyID, INVALID_OPERATION);
2993 break;
2994 } else if (mFlags & kFlagStickyError) {
2995 PostReplyWithError(replyID, getStickyError());
2996 break;
2997 }
2998
2999 int32_t portIndex;
3000 CHECK(msg->findInt32("portIndex", &portIndex));
3001
3002 Vector<sp<MediaCodecBuffer> > *dstBuffers;
3003 CHECK(msg->findPointer("buffers", (void **)&dstBuffers));
3004
3005 dstBuffers->clear();
3006 // If we're using input surface (either non-persistent created by
3007 // createInputSurface(), or persistent set by setInputSurface()),
3008 // give the client an empty input buffers array.
3009 if (portIndex != kPortIndexInput || !mHaveInputSurface) {
3010 if (portIndex == kPortIndexInput) {
3011 mBufferChannel->getInputBufferArray(dstBuffers);
3012 } else {
3013 mBufferChannel->getOutputBufferArray(dstBuffers);
3014 }
3015 }
3016
3017 (new AMessage)->postReply(replyID);
3018 break;
3019 }
3020
3021 case kWhatFlush:
3022 {
3023 sp<AReplyToken> replyID;
3024 CHECK(msg->senderAwaitsResponse(&replyID));
3025
3026 if (!isExecuting()) {
3027 PostReplyWithError(replyID, INVALID_OPERATION);
3028 break;
3029 } else if (mFlags & kFlagStickyError) {
3030 PostReplyWithError(replyID, getStickyError());
3031 break;
3032 }
3033
3034 mReplyID = replyID;
3035 // TODO: skip flushing if already FLUSHED
3036 setState(FLUSHING);
3037
3038 mCodec->signalFlush();
3039 returnBuffersToCodec();
3040 break;
3041 }
3042
3043 case kWhatGetInputFormat:
3044 case kWhatGetOutputFormat:
3045 {
3046 sp<AMessage> format =
3047 (msg->what() == kWhatGetOutputFormat ? mOutputFormat : mInputFormat);
3048
3049 sp<AReplyToken> replyID;
3050 CHECK(msg->senderAwaitsResponse(&replyID));
3051
3052 if ((mState != CONFIGURED && mState != STARTING &&
3053 mState != STARTED && mState != FLUSHING &&
3054 mState != FLUSHED)
3055 || format == NULL) {
3056 PostReplyWithError(replyID, INVALID_OPERATION);
3057 break;
3058 } else if (mFlags & kFlagStickyError) {
3059 PostReplyWithError(replyID, getStickyError());
3060 break;
3061 }
3062
3063 sp<AMessage> response = new AMessage;
3064 response->setMessage("format", format);
3065 response->postReply(replyID);
3066 break;
3067 }
3068
3069 case kWhatRequestIDRFrame:
3070 {
3071 mCodec->signalRequestIDRFrame();
3072 break;
3073 }
3074
3075 case kWhatRequestActivityNotification:
3076 {
3077 CHECK(mActivityNotify == NULL);
3078 CHECK(msg->findMessage("notify", &mActivityNotify));
3079
3080 postActivityNotificationIfPossible();
3081 break;
3082 }
3083
3084 case kWhatGetName:
3085 {
3086 sp<AReplyToken> replyID;
3087 CHECK(msg->senderAwaitsResponse(&replyID));
3088
3089 if (mComponentName.empty()) {
3090 PostReplyWithError(replyID, INVALID_OPERATION);
3091 break;
3092 }
3093
3094 sp<AMessage> response = new AMessage;
3095 response->setString("name", mComponentName.c_str());
3096 response->postReply(replyID);
3097 break;
3098 }
3099
3100 case kWhatGetCodecInfo:
3101 {
3102 sp<AReplyToken> replyID;
3103 CHECK(msg->senderAwaitsResponse(&replyID));
3104
3105 sp<AMessage> response = new AMessage;
3106 response->setObject("codecInfo", mCodecInfo);
3107 response->postReply(replyID);
3108 break;
3109 }
3110
3111 case kWhatSetParameters:
3112 {
3113 sp<AReplyToken> replyID;
3114 CHECK(msg->senderAwaitsResponse(&replyID));
3115
3116 sp<AMessage> params;
3117 CHECK(msg->findMessage("params", ¶ms));
3118
3119 status_t err = onSetParameters(params);
3120
3121 PostReplyWithError(replyID, err);
3122 break;
3123 }
3124
3125 case kWhatDrmReleaseCrypto:
3126 {
3127 onReleaseCrypto(msg);
3128 break;
3129 }
3130
3131 case kWhatCheckBatteryStats:
3132 {
3133 if (mBatteryChecker != nullptr) {
3134 mBatteryChecker->onCheckBatteryTimer(msg, [this] () {
3135 removeResource(MediaResource::kBattery, MediaResource::kVideoCodec, 1);
3136 });
3137 }
3138 break;
3139 }
3140
3141 default:
3142 TRESPASS();
3143 }
3144 }
3145
extractCSD(const sp<AMessage> & format)3146 void MediaCodec::extractCSD(const sp<AMessage> &format) {
3147 mCSD.clear();
3148
3149 size_t i = 0;
3150 for (;;) {
3151 sp<ABuffer> csd;
3152 if (!format->findBuffer(AStringPrintf("csd-%u", i).c_str(), &csd)) {
3153 break;
3154 }
3155 if (csd->size() == 0) {
3156 ALOGW("csd-%zu size is 0", i);
3157 }
3158
3159 mCSD.push_back(csd);
3160 ++i;
3161 }
3162
3163 ALOGV("Found %zu pieces of codec specific data.", mCSD.size());
3164 }
3165
queueCSDInputBuffer(size_t bufferIndex)3166 status_t MediaCodec::queueCSDInputBuffer(size_t bufferIndex) {
3167 CHECK(!mCSD.empty());
3168
3169 const BufferInfo &info = mPortBuffers[kPortIndexInput][bufferIndex];
3170
3171 sp<ABuffer> csd = *mCSD.begin();
3172 mCSD.erase(mCSD.begin());
3173
3174 const sp<MediaCodecBuffer> &codecInputData = info.mData;
3175
3176 if (csd->size() > codecInputData->capacity()) {
3177 return -EINVAL;
3178 }
3179 if (codecInputData->data() == NULL) {
3180 ALOGV("Input buffer %zu is not properly allocated", bufferIndex);
3181 return -EINVAL;
3182 }
3183
3184 memcpy(codecInputData->data(), csd->data(), csd->size());
3185
3186 AString errorDetailMsg;
3187
3188 sp<AMessage> msg = new AMessage(kWhatQueueInputBuffer, this);
3189 msg->setSize("index", bufferIndex);
3190 msg->setSize("offset", 0);
3191 msg->setSize("size", csd->size());
3192 msg->setInt64("timeUs", 0LL);
3193 msg->setInt32("flags", BUFFER_FLAG_CODECCONFIG);
3194 msg->setPointer("errorDetailMsg", &errorDetailMsg);
3195
3196 return onQueueInputBuffer(msg);
3197 }
3198
setState(State newState)3199 void MediaCodec::setState(State newState) {
3200 if (newState == INITIALIZED || newState == UNINITIALIZED) {
3201 delete mSoftRenderer;
3202 mSoftRenderer = NULL;
3203
3204 if ( mCrypto != NULL ) {
3205 ALOGV("setState: ~mCrypto: %p (%d)",
3206 mCrypto.get(), (mCrypto != NULL ? mCrypto->getStrongCount() : 0));
3207 }
3208 mCrypto.clear();
3209 mDescrambler.clear();
3210 handleSetSurface(NULL);
3211
3212 mInputFormat.clear();
3213 mOutputFormat.clear();
3214 mFlags &= ~kFlagOutputFormatChanged;
3215 mFlags &= ~kFlagOutputBuffersChanged;
3216 mFlags &= ~kFlagStickyError;
3217 mFlags &= ~kFlagIsEncoder;
3218 mFlags &= ~kFlagIsAsync;
3219 mStickyError = OK;
3220
3221 mActivityNotify.clear();
3222 mCallback.clear();
3223 }
3224
3225 if (newState == UNINITIALIZED) {
3226 // return any straggling buffers, e.g. if we got here on an error
3227 returnBuffersToCodec();
3228
3229 // The component is gone, mediaserver's probably back up already
3230 // but should definitely be back up should we try to instantiate
3231 // another component.. and the cycle continues.
3232 mFlags &= ~kFlagSawMediaServerDie;
3233 }
3234
3235 mState = newState;
3236
3237 if (mBatteryChecker != nullptr) {
3238 mBatteryChecker->setExecuting(isExecuting());
3239 }
3240
3241 cancelPendingDequeueOperations();
3242 }
3243
returnBuffersToCodec(bool isReclaim)3244 void MediaCodec::returnBuffersToCodec(bool isReclaim) {
3245 returnBuffersToCodecOnPort(kPortIndexInput, isReclaim);
3246 returnBuffersToCodecOnPort(kPortIndexOutput, isReclaim);
3247 }
3248
returnBuffersToCodecOnPort(int32_t portIndex,bool isReclaim)3249 void MediaCodec::returnBuffersToCodecOnPort(int32_t portIndex, bool isReclaim) {
3250 CHECK(portIndex == kPortIndexInput || portIndex == kPortIndexOutput);
3251 Mutex::Autolock al(mBufferLock);
3252
3253 for (size_t i = 0; i < mPortBuffers[portIndex].size(); ++i) {
3254 BufferInfo *info = &mPortBuffers[portIndex][i];
3255
3256 if (info->mData != nullptr) {
3257 sp<MediaCodecBuffer> buffer = info->mData;
3258 if (isReclaim && info->mOwnedByClient) {
3259 ALOGD("port %d buffer %zu still owned by client when codec is reclaimed",
3260 portIndex, i);
3261 } else {
3262 info->mOwnedByClient = false;
3263 info->mData.clear();
3264 }
3265 mBufferChannel->discardBuffer(buffer);
3266 }
3267 }
3268
3269 mAvailPortBuffers[portIndex].clear();
3270 }
3271
updateBuffers(int32_t portIndex,const sp<AMessage> & msg)3272 size_t MediaCodec::updateBuffers(
3273 int32_t portIndex, const sp<AMessage> &msg) {
3274 CHECK(portIndex == kPortIndexInput || portIndex == kPortIndexOutput);
3275 size_t index;
3276 CHECK(msg->findSize("index", &index));
3277 sp<RefBase> obj;
3278 CHECK(msg->findObject("buffer", &obj));
3279 sp<MediaCodecBuffer> buffer = static_cast<MediaCodecBuffer *>(obj.get());
3280
3281 {
3282 Mutex::Autolock al(mBufferLock);
3283 if (mPortBuffers[portIndex].size() <= index) {
3284 mPortBuffers[portIndex].resize(align(index + 1, kNumBuffersAlign));
3285 }
3286 mPortBuffers[portIndex][index].mData = buffer;
3287 }
3288 mAvailPortBuffers[portIndex].push_back(index);
3289
3290 return index;
3291 }
3292
onQueueInputBuffer(const sp<AMessage> & msg)3293 status_t MediaCodec::onQueueInputBuffer(const sp<AMessage> &msg) {
3294 size_t index;
3295 size_t offset;
3296 size_t size;
3297 int64_t timeUs;
3298 uint32_t flags;
3299 CHECK(msg->findSize("index", &index));
3300 CHECK(msg->findSize("offset", &offset));
3301 CHECK(msg->findInt64("timeUs", &timeUs));
3302 CHECK(msg->findInt32("flags", (int32_t *)&flags));
3303
3304 const CryptoPlugin::SubSample *subSamples;
3305 size_t numSubSamples;
3306 const uint8_t *key;
3307 const uint8_t *iv;
3308 CryptoPlugin::Mode mode = CryptoPlugin::kMode_Unencrypted;
3309
3310 // We allow the simpler queueInputBuffer API to be used even in
3311 // secure mode, by fabricating a single unencrypted subSample.
3312 CryptoPlugin::SubSample ss;
3313 CryptoPlugin::Pattern pattern;
3314
3315 if (msg->findSize("size", &size)) {
3316 if (hasCryptoOrDescrambler()) {
3317 ss.mNumBytesOfClearData = size;
3318 ss.mNumBytesOfEncryptedData = 0;
3319
3320 subSamples = &ss;
3321 numSubSamples = 1;
3322 key = NULL;
3323 iv = NULL;
3324 pattern.mEncryptBlocks = 0;
3325 pattern.mSkipBlocks = 0;
3326 }
3327 } else {
3328 if (!hasCryptoOrDescrambler()) {
3329 ALOGE("[%s] queuing secure buffer without mCrypto or mDescrambler!",
3330 mComponentName.c_str());
3331 return -EINVAL;
3332 }
3333
3334 CHECK(msg->findPointer("subSamples", (void **)&subSamples));
3335 CHECK(msg->findSize("numSubSamples", &numSubSamples));
3336 CHECK(msg->findPointer("key", (void **)&key));
3337 CHECK(msg->findPointer("iv", (void **)&iv));
3338 CHECK(msg->findInt32("encryptBlocks", (int32_t *)&pattern.mEncryptBlocks));
3339 CHECK(msg->findInt32("skipBlocks", (int32_t *)&pattern.mSkipBlocks));
3340
3341 int32_t tmp;
3342 CHECK(msg->findInt32("mode", &tmp));
3343
3344 mode = (CryptoPlugin::Mode)tmp;
3345
3346 size = 0;
3347 for (size_t i = 0; i < numSubSamples; ++i) {
3348 size += subSamples[i].mNumBytesOfClearData;
3349 size += subSamples[i].mNumBytesOfEncryptedData;
3350 }
3351 }
3352
3353 if (index >= mPortBuffers[kPortIndexInput].size()) {
3354 return -ERANGE;
3355 }
3356
3357 BufferInfo *info = &mPortBuffers[kPortIndexInput][index];
3358
3359 if (info->mData == nullptr || !info->mOwnedByClient) {
3360 return -EACCES;
3361 }
3362
3363 if (offset + size > info->mData->capacity()) {
3364 return -EINVAL;
3365 }
3366
3367 info->mData->setRange(offset, size);
3368 info->mData->meta()->setInt64("timeUs", timeUs);
3369 if (flags & BUFFER_FLAG_EOS) {
3370 info->mData->meta()->setInt32("eos", true);
3371 }
3372
3373 if (flags & BUFFER_FLAG_CODECCONFIG) {
3374 info->mData->meta()->setInt32("csd", true);
3375 }
3376
3377 sp<MediaCodecBuffer> buffer = info->mData;
3378 status_t err = OK;
3379 if (hasCryptoOrDescrambler()) {
3380 AString *errorDetailMsg;
3381 CHECK(msg->findPointer("errorDetailMsg", (void **)&errorDetailMsg));
3382
3383 err = mBufferChannel->queueSecureInputBuffer(
3384 buffer,
3385 (mFlags & kFlagIsSecure),
3386 key,
3387 iv,
3388 mode,
3389 pattern,
3390 subSamples,
3391 numSubSamples,
3392 errorDetailMsg);
3393 } else {
3394 err = mBufferChannel->queueInputBuffer(buffer);
3395 }
3396
3397 if (err == OK) {
3398 // synchronization boundary for getBufferAndFormat
3399 Mutex::Autolock al(mBufferLock);
3400 info->mOwnedByClient = false;
3401 info->mData.clear();
3402
3403 statsBufferSent(timeUs);
3404 }
3405
3406 return err;
3407 }
3408
3409 //static
CreateFramesRenderedMessage(const std::list<FrameRenderTracker::Info> & done,sp<AMessage> & msg)3410 size_t MediaCodec::CreateFramesRenderedMessage(
3411 const std::list<FrameRenderTracker::Info> &done, sp<AMessage> &msg) {
3412 size_t index = 0;
3413
3414 for (std::list<FrameRenderTracker::Info>::const_iterator it = done.cbegin();
3415 it != done.cend(); ++it) {
3416 if (it->getRenderTimeNs() < 0) {
3417 continue; // dropped frame from tracking
3418 }
3419 msg->setInt64(AStringPrintf("%zu-media-time-us", index).c_str(), it->getMediaTimeUs());
3420 msg->setInt64(AStringPrintf("%zu-system-nano", index).c_str(), it->getRenderTimeNs());
3421 ++index;
3422 }
3423 return index;
3424 }
3425
onReleaseOutputBuffer(const sp<AMessage> & msg)3426 status_t MediaCodec::onReleaseOutputBuffer(const sp<AMessage> &msg) {
3427 size_t index;
3428 CHECK(msg->findSize("index", &index));
3429
3430 int32_t render;
3431 if (!msg->findInt32("render", &render)) {
3432 render = 0;
3433 }
3434
3435 if (!isExecuting()) {
3436 return -EINVAL;
3437 }
3438
3439 if (index >= mPortBuffers[kPortIndexOutput].size()) {
3440 return -ERANGE;
3441 }
3442
3443 BufferInfo *info = &mPortBuffers[kPortIndexOutput][index];
3444
3445 if (info->mData == nullptr || !info->mOwnedByClient) {
3446 return -EACCES;
3447 }
3448
3449 // synchronization boundary for getBufferAndFormat
3450 sp<MediaCodecBuffer> buffer;
3451 {
3452 Mutex::Autolock al(mBufferLock);
3453 info->mOwnedByClient = false;
3454 buffer = info->mData;
3455 info->mData.clear();
3456 }
3457
3458 if (render && buffer->size() != 0) {
3459 int64_t mediaTimeUs = -1;
3460 buffer->meta()->findInt64("timeUs", &mediaTimeUs);
3461
3462 int64_t renderTimeNs = 0;
3463 if (!msg->findInt64("timestampNs", &renderTimeNs)) {
3464 // use media timestamp if client did not request a specific render timestamp
3465 ALOGV("using buffer PTS of %lld", (long long)mediaTimeUs);
3466 renderTimeNs = mediaTimeUs * 1000;
3467 }
3468
3469 if (mSoftRenderer != NULL) {
3470 std::list<FrameRenderTracker::Info> doneFrames = mSoftRenderer->render(
3471 buffer->data(), buffer->size(), mediaTimeUs, renderTimeNs,
3472 mPortBuffers[kPortIndexOutput].size(), buffer->format());
3473
3474 // if we are running, notify rendered frames
3475 if (!doneFrames.empty() && mState == STARTED && mOnFrameRenderedNotification != NULL) {
3476 sp<AMessage> notify = mOnFrameRenderedNotification->dup();
3477 sp<AMessage> data = new AMessage;
3478 if (CreateFramesRenderedMessage(doneFrames, data)) {
3479 notify->setMessage("data", data);
3480 notify->post();
3481 }
3482 }
3483 }
3484 mBufferChannel->renderOutputBuffer(buffer, renderTimeNs);
3485 } else {
3486 mBufferChannel->discardBuffer(buffer);
3487 }
3488
3489 return OK;
3490 }
3491
dequeuePortBuffer(int32_t portIndex)3492 ssize_t MediaCodec::dequeuePortBuffer(int32_t portIndex) {
3493 CHECK(portIndex == kPortIndexInput || portIndex == kPortIndexOutput);
3494
3495 List<size_t> *availBuffers = &mAvailPortBuffers[portIndex];
3496
3497 if (availBuffers->empty()) {
3498 return -EAGAIN;
3499 }
3500
3501 size_t index = *availBuffers->begin();
3502 availBuffers->erase(availBuffers->begin());
3503
3504 BufferInfo *info = &mPortBuffers[portIndex][index];
3505 CHECK(!info->mOwnedByClient);
3506 {
3507 Mutex::Autolock al(mBufferLock);
3508 info->mOwnedByClient = true;
3509
3510 // set image-data
3511 if (info->mData->format() != NULL) {
3512 sp<ABuffer> imageData;
3513 if (info->mData->format()->findBuffer("image-data", &imageData)) {
3514 info->mData->meta()->setBuffer("image-data", imageData);
3515 }
3516 int32_t left, top, right, bottom;
3517 if (info->mData->format()->findRect("crop", &left, &top, &right, &bottom)) {
3518 info->mData->meta()->setRect("crop-rect", left, top, right, bottom);
3519 }
3520 }
3521 }
3522
3523 return index;
3524 }
3525
connectToSurface(const sp<Surface> & surface)3526 status_t MediaCodec::connectToSurface(const sp<Surface> &surface) {
3527 status_t err = OK;
3528 if (surface != NULL) {
3529 uint64_t oldId, newId;
3530 if (mSurface != NULL
3531 && surface->getUniqueId(&newId) == NO_ERROR
3532 && mSurface->getUniqueId(&oldId) == NO_ERROR
3533 && newId == oldId) {
3534 ALOGI("[%s] connecting to the same surface. Nothing to do.", mComponentName.c_str());
3535 return ALREADY_EXISTS;
3536 }
3537
3538 err = nativeWindowConnect(surface.get(), "connectToSurface");
3539 if (err == OK) {
3540 // Require a fresh set of buffers after each connect by using a unique generation
3541 // number. Rely on the fact that max supported process id by Linux is 2^22.
3542 // PID is never 0 so we don't have to worry that we use the default generation of 0.
3543 // TODO: come up with a unique scheme if other producers also set the generation number.
3544 static uint32_t mSurfaceGeneration = 0;
3545 uint32_t generation = (getpid() << 10) | (++mSurfaceGeneration & ((1 << 10) - 1));
3546 surface->setGenerationNumber(generation);
3547 ALOGI("[%s] setting surface generation to %u", mComponentName.c_str(), generation);
3548
3549 // HACK: clear any free buffers. Remove when connect will automatically do this.
3550 // This is needed as the consumer may be holding onto stale frames that it can reattach
3551 // to this surface after disconnect/connect, and those free frames would inherit the new
3552 // generation number. Disconnecting after setting a unique generation prevents this.
3553 nativeWindowDisconnect(surface.get(), "connectToSurface(reconnect)");
3554 err = nativeWindowConnect(surface.get(), "connectToSurface(reconnect)");
3555 }
3556
3557 if (err != OK) {
3558 ALOGE("nativeWindowConnect returned an error: %s (%d)", strerror(-err), err);
3559 } else {
3560 if (!mAllowFrameDroppingBySurface) {
3561 disableLegacyBufferDropPostQ(surface);
3562 }
3563 }
3564 }
3565 // do not return ALREADY_EXISTS unless surfaces are the same
3566 return err == ALREADY_EXISTS ? BAD_VALUE : err;
3567 }
3568
disconnectFromSurface()3569 status_t MediaCodec::disconnectFromSurface() {
3570 status_t err = OK;
3571 if (mSurface != NULL) {
3572 // Resetting generation is not technically needed, but there is no need to keep it either
3573 mSurface->setGenerationNumber(0);
3574 err = nativeWindowDisconnect(mSurface.get(), "disconnectFromSurface");
3575 if (err != OK) {
3576 ALOGW("nativeWindowDisconnect returned an error: %s (%d)", strerror(-err), err);
3577 }
3578 // assume disconnected even on error
3579 mSurface.clear();
3580 }
3581 return err;
3582 }
3583
handleSetSurface(const sp<Surface> & surface)3584 status_t MediaCodec::handleSetSurface(const sp<Surface> &surface) {
3585 status_t err = OK;
3586 if (mSurface != NULL) {
3587 (void)disconnectFromSurface();
3588 }
3589 if (surface != NULL) {
3590 err = connectToSurface(surface);
3591 if (err == OK) {
3592 mSurface = surface;
3593 }
3594 }
3595 return err;
3596 }
3597
onInputBufferAvailable()3598 void MediaCodec::onInputBufferAvailable() {
3599 int32_t index;
3600 while ((index = dequeuePortBuffer(kPortIndexInput)) >= 0) {
3601 sp<AMessage> msg = mCallback->dup();
3602 msg->setInt32("callbackID", CB_INPUT_AVAILABLE);
3603 msg->setInt32("index", index);
3604 msg->post();
3605 }
3606 }
3607
onOutputBufferAvailable()3608 void MediaCodec::onOutputBufferAvailable() {
3609 int32_t index;
3610 while ((index = dequeuePortBuffer(kPortIndexOutput)) >= 0) {
3611 const sp<MediaCodecBuffer> &buffer =
3612 mPortBuffers[kPortIndexOutput][index].mData;
3613 sp<AMessage> msg = mCallback->dup();
3614 msg->setInt32("callbackID", CB_OUTPUT_AVAILABLE);
3615 msg->setInt32("index", index);
3616 msg->setSize("offset", buffer->offset());
3617 msg->setSize("size", buffer->size());
3618
3619 int64_t timeUs;
3620 CHECK(buffer->meta()->findInt64("timeUs", &timeUs));
3621
3622 msg->setInt64("timeUs", timeUs);
3623
3624 statsBufferReceived(timeUs);
3625
3626 int32_t flags;
3627 CHECK(buffer->meta()->findInt32("flags", &flags));
3628
3629 msg->setInt32("flags", flags);
3630
3631 msg->post();
3632 }
3633 }
3634
onError(status_t err,int32_t actionCode,const char * detail)3635 void MediaCodec::onError(status_t err, int32_t actionCode, const char *detail) {
3636 if (mCallback != NULL) {
3637 sp<AMessage> msg = mCallback->dup();
3638 msg->setInt32("callbackID", CB_ERROR);
3639 msg->setInt32("err", err);
3640 msg->setInt32("actionCode", actionCode);
3641
3642 if (detail != NULL) {
3643 msg->setString("detail", detail);
3644 }
3645
3646 msg->post();
3647 }
3648 }
3649
onOutputFormatChanged()3650 void MediaCodec::onOutputFormatChanged() {
3651 if (mCallback != NULL) {
3652 sp<AMessage> msg = mCallback->dup();
3653 msg->setInt32("callbackID", CB_OUTPUT_FORMAT_CHANGED);
3654 msg->setMessage("format", mOutputFormat);
3655 msg->post();
3656 }
3657 }
3658
postActivityNotificationIfPossible()3659 void MediaCodec::postActivityNotificationIfPossible() {
3660 if (mActivityNotify == NULL) {
3661 return;
3662 }
3663
3664 bool isErrorOrOutputChanged =
3665 (mFlags & (kFlagStickyError
3666 | kFlagOutputBuffersChanged
3667 | kFlagOutputFormatChanged));
3668
3669 if (isErrorOrOutputChanged
3670 || !mAvailPortBuffers[kPortIndexInput].empty()
3671 || !mAvailPortBuffers[kPortIndexOutput].empty()) {
3672 mActivityNotify->setInt32("input-buffers",
3673 mAvailPortBuffers[kPortIndexInput].size());
3674
3675 if (isErrorOrOutputChanged) {
3676 // we want consumer to dequeue as many times as it can
3677 mActivityNotify->setInt32("output-buffers", INT32_MAX);
3678 } else {
3679 mActivityNotify->setInt32("output-buffers",
3680 mAvailPortBuffers[kPortIndexOutput].size());
3681 }
3682 mActivityNotify->post();
3683 mActivityNotify.clear();
3684 }
3685 }
3686
setParameters(const sp<AMessage> & params)3687 status_t MediaCodec::setParameters(const sp<AMessage> ¶ms) {
3688 sp<AMessage> msg = new AMessage(kWhatSetParameters, this);
3689 msg->setMessage("params", params);
3690
3691 sp<AMessage> response;
3692 return PostAndAwaitResponse(msg, &response);
3693 }
3694
onSetParameters(const sp<AMessage> & params)3695 status_t MediaCodec::onSetParameters(const sp<AMessage> ¶ms) {
3696 mCodec->signalSetParameters(params);
3697
3698 return OK;
3699 }
3700
amendOutputFormatWithCodecSpecificData(const sp<MediaCodecBuffer> & buffer)3701 status_t MediaCodec::amendOutputFormatWithCodecSpecificData(
3702 const sp<MediaCodecBuffer> &buffer) {
3703 AString mime;
3704 CHECK(mOutputFormat->findString("mime", &mime));
3705
3706 if (!strcasecmp(mime.c_str(), MEDIA_MIMETYPE_VIDEO_AVC)) {
3707 // Codec specific data should be SPS and PPS in a single buffer,
3708 // each prefixed by a startcode (0x00 0x00 0x00 0x01).
3709 // We separate the two and put them into the output format
3710 // under the keys "csd-0" and "csd-1".
3711
3712 unsigned csdIndex = 0;
3713
3714 const uint8_t *data = buffer->data();
3715 size_t size = buffer->size();
3716
3717 const uint8_t *nalStart;
3718 size_t nalSize;
3719 while (getNextNALUnit(&data, &size, &nalStart, &nalSize, true) == OK) {
3720 sp<ABuffer> csd = new ABuffer(nalSize + 4);
3721 memcpy(csd->data(), "\x00\x00\x00\x01", 4);
3722 memcpy(csd->data() + 4, nalStart, nalSize);
3723
3724 mOutputFormat->setBuffer(
3725 AStringPrintf("csd-%u", csdIndex).c_str(), csd);
3726
3727 ++csdIndex;
3728 }
3729
3730 if (csdIndex != 2) {
3731 return ERROR_MALFORMED;
3732 }
3733 } else {
3734 // For everything else we just stash the codec specific data into
3735 // the output format as a single piece of csd under "csd-0".
3736 sp<ABuffer> csd = new ABuffer(buffer->size());
3737 memcpy(csd->data(), buffer->data(), buffer->size());
3738 csd->setRange(0, buffer->size());
3739 mOutputFormat->setBuffer("csd-0", csd);
3740 }
3741
3742 return OK;
3743 }
3744
stateString(State state)3745 std::string MediaCodec::stateString(State state) {
3746 const char *rval = NULL;
3747 char rawbuffer[16]; // room for "%d"
3748
3749 switch (state) {
3750 case UNINITIALIZED: rval = "UNINITIALIZED"; break;
3751 case INITIALIZING: rval = "INITIALIZING"; break;
3752 case INITIALIZED: rval = "INITIALIZED"; break;
3753 case CONFIGURING: rval = "CONFIGURING"; break;
3754 case CONFIGURED: rval = "CONFIGURED"; break;
3755 case STARTING: rval = "STARTING"; break;
3756 case STARTED: rval = "STARTED"; break;
3757 case FLUSHING: rval = "FLUSHING"; break;
3758 case FLUSHED: rval = "FLUSHED"; break;
3759 case STOPPING: rval = "STOPPING"; break;
3760 case RELEASING: rval = "RELEASING"; break;
3761 default:
3762 snprintf(rawbuffer, sizeof(rawbuffer), "%d", state);
3763 rval = rawbuffer;
3764 break;
3765 }
3766 return rval;
3767 }
3768
3769 } // namespace android
3770