1 /*
2 * Copyright 2017, The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 //#define LOG_NDEBUG 0
18 #define LOG_TAG "CCodecBufferChannel"
19 #include <utils/Log.h>
20
21 #include <numeric>
22
23 #include <C2AllocatorGralloc.h>
24 #include <C2PlatformSupport.h>
25 #include <C2BlockInternal.h>
26 #include <C2Config.h>
27 #include <C2Debug.h>
28
29 #include <android/hardware/cas/native/1.0/IDescrambler.h>
30 #include <android-base/stringprintf.h>
31 #include <binder/MemoryDealer.h>
32 #include <cutils/properties.h>
33 #include <gui/Surface.h>
34 #include <media/openmax/OMX_Core.h>
35 #include <media/stagefright/foundation/ABuffer.h>
36 #include <media/stagefright/foundation/ALookup.h>
37 #include <media/stagefright/foundation/AMessage.h>
38 #include <media/stagefright/foundation/AUtils.h>
39 #include <media/stagefright/foundation/hexdump.h>
40 #include <media/stagefright/MediaCodec.h>
41 #include <media/stagefright/MediaCodecConstants.h>
42 #include <media/MediaCodecBuffer.h>
43 #include <system/window.h>
44
45 #include "CCodecBufferChannel.h"
46 #include "Codec2Buffer.h"
47 #include "SkipCutBuffer.h"
48
49 namespace android {
50
51 using android::base::StringPrintf;
52 using hardware::hidl_handle;
53 using hardware::hidl_string;
54 using hardware::hidl_vec;
55 using namespace hardware::cas::V1_0;
56 using namespace hardware::cas::native::V1_0;
57
58 using CasStatus = hardware::cas::V1_0::Status;
59
60 namespace {
61
62 constexpr size_t kSmoothnessFactor = 4;
63 constexpr size_t kRenderingDepth = 3;
64
65 // This is for keeping IGBP's buffer dropping logic in legacy mode other
66 // than making it non-blocking. Do not change this value.
67 const static size_t kDequeueTimeoutNs = 0;
68
69 } // namespace
70
QueueGuard(CCodecBufferChannel::QueueSync & sync)71 CCodecBufferChannel::QueueGuard::QueueGuard(
72 CCodecBufferChannel::QueueSync &sync) : mSync(sync) {
73 Mutex::Autolock l(mSync.mGuardLock);
74 // At this point it's guaranteed that mSync is not under state transition,
75 // as we are holding its mutex.
76
77 Mutexed<CCodecBufferChannel::QueueSync::Counter>::Locked count(mSync.mCount);
78 if (count->value == -1) {
79 mRunning = false;
80 } else {
81 ++count->value;
82 mRunning = true;
83 }
84 }
85
~QueueGuard()86 CCodecBufferChannel::QueueGuard::~QueueGuard() {
87 if (mRunning) {
88 // We are not holding mGuardLock at this point so that QueueSync::stop() can
89 // keep holding the lock until mCount reaches zero.
90 Mutexed<CCodecBufferChannel::QueueSync::Counter>::Locked count(mSync.mCount);
91 --count->value;
92 count->cond.broadcast();
93 }
94 }
95
start()96 void CCodecBufferChannel::QueueSync::start() {
97 Mutex::Autolock l(mGuardLock);
98 // If stopped, it goes to running state; otherwise no-op.
99 Mutexed<Counter>::Locked count(mCount);
100 if (count->value == -1) {
101 count->value = 0;
102 }
103 }
104
stop()105 void CCodecBufferChannel::QueueSync::stop() {
106 Mutex::Autolock l(mGuardLock);
107 Mutexed<Counter>::Locked count(mCount);
108 if (count->value == -1) {
109 // no-op
110 return;
111 }
112 // Holding mGuardLock here blocks creation of additional QueueGuard objects, so
113 // mCount can only decrement. In other words, threads that acquired the lock
114 // are allowed to finish execution but additional threads trying to acquire
115 // the lock at this point will block, and then get QueueGuard at STOPPED
116 // state.
117 while (count->value != 0) {
118 count.waitForCondition(count->cond);
119 }
120 count->value = -1;
121 }
122
123 // CCodecBufferChannel::ReorderStash
124
ReorderStash()125 CCodecBufferChannel::ReorderStash::ReorderStash() {
126 clear();
127 }
128
clear()129 void CCodecBufferChannel::ReorderStash::clear() {
130 mPending.clear();
131 mStash.clear();
132 mDepth = 0;
133 mKey = C2Config::ORDINAL;
134 }
135
flush()136 void CCodecBufferChannel::ReorderStash::flush() {
137 mPending.clear();
138 mStash.clear();
139 }
140
setDepth(uint32_t depth)141 void CCodecBufferChannel::ReorderStash::setDepth(uint32_t depth) {
142 mPending.splice(mPending.end(), mStash);
143 mDepth = depth;
144 }
145
setKey(C2Config::ordinal_key_t key)146 void CCodecBufferChannel::ReorderStash::setKey(C2Config::ordinal_key_t key) {
147 mPending.splice(mPending.end(), mStash);
148 mKey = key;
149 }
150
pop(Entry * entry)151 bool CCodecBufferChannel::ReorderStash::pop(Entry *entry) {
152 if (mPending.empty()) {
153 return false;
154 }
155 entry->buffer = mPending.front().buffer;
156 entry->timestamp = mPending.front().timestamp;
157 entry->flags = mPending.front().flags;
158 entry->ordinal = mPending.front().ordinal;
159 mPending.pop_front();
160 return true;
161 }
162
emplace(const std::shared_ptr<C2Buffer> & buffer,int64_t timestamp,int32_t flags,const C2WorkOrdinalStruct & ordinal)163 void CCodecBufferChannel::ReorderStash::emplace(
164 const std::shared_ptr<C2Buffer> &buffer,
165 int64_t timestamp,
166 int32_t flags,
167 const C2WorkOrdinalStruct &ordinal) {
168 bool eos = flags & MediaCodec::BUFFER_FLAG_EOS;
169 if (!buffer && eos) {
170 // TRICKY: we may be violating ordering of the stash here. Because we
171 // don't expect any more emplace() calls after this, the ordering should
172 // not matter.
173 mStash.emplace_back(buffer, timestamp, flags, ordinal);
174 } else {
175 flags = flags & ~MediaCodec::BUFFER_FLAG_EOS;
176 auto it = mStash.begin();
177 for (; it != mStash.end(); ++it) {
178 if (less(ordinal, it->ordinal)) {
179 break;
180 }
181 }
182 mStash.emplace(it, buffer, timestamp, flags, ordinal);
183 if (eos) {
184 mStash.back().flags = mStash.back().flags | MediaCodec::BUFFER_FLAG_EOS;
185 }
186 }
187 while (!mStash.empty() && mStash.size() > mDepth) {
188 mPending.push_back(mStash.front());
189 mStash.pop_front();
190 }
191 }
192
defer(const CCodecBufferChannel::ReorderStash::Entry & entry)193 void CCodecBufferChannel::ReorderStash::defer(
194 const CCodecBufferChannel::ReorderStash::Entry &entry) {
195 mPending.push_front(entry);
196 }
197
hasPending() const198 bool CCodecBufferChannel::ReorderStash::hasPending() const {
199 return !mPending.empty();
200 }
201
less(const C2WorkOrdinalStruct & o1,const C2WorkOrdinalStruct & o2)202 bool CCodecBufferChannel::ReorderStash::less(
203 const C2WorkOrdinalStruct &o1, const C2WorkOrdinalStruct &o2) {
204 switch (mKey) {
205 case C2Config::ORDINAL: return o1.frameIndex < o2.frameIndex;
206 case C2Config::TIMESTAMP: return o1.timestamp < o2.timestamp;
207 case C2Config::CUSTOM: return o1.customOrdinal < o2.customOrdinal;
208 default:
209 ALOGD("Unrecognized key; default to timestamp");
210 return o1.frameIndex < o2.frameIndex;
211 }
212 }
213
214 // Input
215
Input()216 CCodecBufferChannel::Input::Input() : extraBuffers("extra") {}
217
218 // CCodecBufferChannel
219
CCodecBufferChannel(const std::shared_ptr<CCodecCallback> & callback)220 CCodecBufferChannel::CCodecBufferChannel(
221 const std::shared_ptr<CCodecCallback> &callback)
222 : mHeapSeqNum(-1),
223 mCCodecCallback(callback),
224 mFrameIndex(0u),
225 mFirstValidFrameIndex(0u),
226 mMetaMode(MODE_NONE),
227 mInputMetEos(false) {
228 mOutputSurface.lock()->maxDequeueBuffers = kSmoothnessFactor + kRenderingDepth;
229 {
230 Mutexed<Input>::Locked input(mInput);
231 input->buffers.reset(new DummyInputBuffers(""));
232 input->extraBuffers.flush();
233 input->inputDelay = 0u;
234 input->pipelineDelay = 0u;
235 input->numSlots = kSmoothnessFactor;
236 input->numExtraSlots = 0u;
237 }
238 {
239 Mutexed<Output>::Locked output(mOutput);
240 output->outputDelay = 0u;
241 output->numSlots = kSmoothnessFactor;
242 }
243 }
244
~CCodecBufferChannel()245 CCodecBufferChannel::~CCodecBufferChannel() {
246 if (mCrypto != nullptr && mDealer != nullptr && mHeapSeqNum >= 0) {
247 mCrypto->unsetHeap(mHeapSeqNum);
248 }
249 }
250
setComponent(const std::shared_ptr<Codec2Client::Component> & component)251 void CCodecBufferChannel::setComponent(
252 const std::shared_ptr<Codec2Client::Component> &component) {
253 mComponent = component;
254 mComponentName = component->getName() + StringPrintf("#%d", int(uintptr_t(component.get()) % 997));
255 mName = mComponentName.c_str();
256 }
257
setInputSurface(const std::shared_ptr<InputSurfaceWrapper> & surface)258 status_t CCodecBufferChannel::setInputSurface(
259 const std::shared_ptr<InputSurfaceWrapper> &surface) {
260 ALOGV("[%s] setInputSurface", mName);
261 mInputSurface = surface;
262 return mInputSurface->connect(mComponent);
263 }
264
signalEndOfInputStream()265 status_t CCodecBufferChannel::signalEndOfInputStream() {
266 if (mInputSurface == nullptr) {
267 return INVALID_OPERATION;
268 }
269 return mInputSurface->signalEndOfInputStream();
270 }
271
queueInputBufferInternal(sp<MediaCodecBuffer> buffer)272 status_t CCodecBufferChannel::queueInputBufferInternal(sp<MediaCodecBuffer> buffer) {
273 int64_t timeUs;
274 CHECK(buffer->meta()->findInt64("timeUs", &timeUs));
275
276 if (mInputMetEos) {
277 ALOGD("[%s] buffers after EOS ignored (%lld us)", mName, (long long)timeUs);
278 return OK;
279 }
280
281 int32_t flags = 0;
282 int32_t tmp = 0;
283 bool eos = false;
284 if (buffer->meta()->findInt32("eos", &tmp) && tmp) {
285 eos = true;
286 mInputMetEos = true;
287 ALOGV("[%s] input EOS", mName);
288 }
289 if (buffer->meta()->findInt32("csd", &tmp) && tmp) {
290 flags |= C2FrameData::FLAG_CODEC_CONFIG;
291 }
292 ALOGV("[%s] queueInputBuffer: buffer->size() = %zu", mName, buffer->size());
293 std::unique_ptr<C2Work> work(new C2Work);
294 work->input.ordinal.timestamp = timeUs;
295 work->input.ordinal.frameIndex = mFrameIndex++;
296 // WORKAROUND: until codecs support handling work after EOS and max output sizing, use timestamp
297 // manipulation to achieve image encoding via video codec, and to constrain encoded output.
298 // Keep client timestamp in customOrdinal
299 work->input.ordinal.customOrdinal = timeUs;
300 work->input.buffers.clear();
301
302 uint64_t queuedFrameIndex = work->input.ordinal.frameIndex.peeku();
303 std::vector<std::shared_ptr<C2Buffer>> queuedBuffers;
304 sp<Codec2Buffer> copy;
305
306 if (buffer->size() > 0u) {
307 Mutexed<Input>::Locked input(mInput);
308 std::shared_ptr<C2Buffer> c2buffer;
309 if (!input->buffers->releaseBuffer(buffer, &c2buffer, false)) {
310 return -ENOENT;
311 }
312 // TODO: we want to delay copying buffers.
313 if (input->extraBuffers.numComponentBuffers() < input->numExtraSlots) {
314 copy = input->buffers->cloneAndReleaseBuffer(buffer);
315 if (copy != nullptr) {
316 (void)input->extraBuffers.assignSlot(copy);
317 if (!input->extraBuffers.releaseSlot(copy, &c2buffer, false)) {
318 return UNKNOWN_ERROR;
319 }
320 bool released = input->buffers->releaseBuffer(buffer, nullptr, true);
321 ALOGV("[%s] queueInputBuffer: buffer copied; %sreleased",
322 mName, released ? "" : "not ");
323 buffer.clear();
324 } else {
325 ALOGW("[%s] queueInputBuffer: failed to copy a buffer; this may cause input "
326 "buffer starvation on component.", mName);
327 }
328 }
329 work->input.buffers.push_back(c2buffer);
330 queuedBuffers.push_back(c2buffer);
331 } else if (eos) {
332 flags |= C2FrameData::FLAG_END_OF_STREAM;
333 }
334 work->input.flags = (C2FrameData::flags_t)flags;
335 // TODO: fill info's
336
337 work->input.configUpdate = std::move(mParamsToBeSet);
338 work->worklets.clear();
339 work->worklets.emplace_back(new C2Worklet);
340
341 std::list<std::unique_ptr<C2Work>> items;
342 items.push_back(std::move(work));
343 mPipelineWatcher.lock()->onWorkQueued(
344 queuedFrameIndex,
345 std::move(queuedBuffers),
346 PipelineWatcher::Clock::now());
347 c2_status_t err = mComponent->queue(&items);
348 if (err != C2_OK) {
349 mPipelineWatcher.lock()->onWorkDone(queuedFrameIndex);
350 }
351
352 if (err == C2_OK && eos && buffer->size() > 0u) {
353 work.reset(new C2Work);
354 work->input.ordinal.timestamp = timeUs;
355 work->input.ordinal.frameIndex = mFrameIndex++;
356 // WORKAROUND: keep client timestamp in customOrdinal
357 work->input.ordinal.customOrdinal = timeUs;
358 work->input.buffers.clear();
359 work->input.flags = C2FrameData::FLAG_END_OF_STREAM;
360 work->worklets.emplace_back(new C2Worklet);
361
362 queuedFrameIndex = work->input.ordinal.frameIndex.peeku();
363 queuedBuffers.clear();
364
365 items.clear();
366 items.push_back(std::move(work));
367
368 mPipelineWatcher.lock()->onWorkQueued(
369 queuedFrameIndex,
370 std::move(queuedBuffers),
371 PipelineWatcher::Clock::now());
372 err = mComponent->queue(&items);
373 if (err != C2_OK) {
374 mPipelineWatcher.lock()->onWorkDone(queuedFrameIndex);
375 }
376 }
377 if (err == C2_OK) {
378 Mutexed<Input>::Locked input(mInput);
379 bool released = false;
380 if (buffer) {
381 released = input->buffers->releaseBuffer(buffer, nullptr, true);
382 } else if (copy) {
383 released = input->extraBuffers.releaseSlot(copy, nullptr, true);
384 }
385 ALOGV("[%s] queueInputBuffer: buffer%s %sreleased",
386 mName, (buffer == nullptr) ? "(copy)" : "", released ? "" : "not ");
387 }
388
389 feedInputBufferIfAvailableInternal();
390 return err;
391 }
392
setParameters(std::vector<std::unique_ptr<C2Param>> & params)393 status_t CCodecBufferChannel::setParameters(std::vector<std::unique_ptr<C2Param>> ¶ms) {
394 QueueGuard guard(mSync);
395 if (!guard.isRunning()) {
396 ALOGD("[%s] setParameters is only supported in the running state.", mName);
397 return -ENOSYS;
398 }
399 mParamsToBeSet.insert(mParamsToBeSet.end(),
400 std::make_move_iterator(params.begin()),
401 std::make_move_iterator(params.end()));
402 params.clear();
403 return OK;
404 }
405
queueInputBuffer(const sp<MediaCodecBuffer> & buffer)406 status_t CCodecBufferChannel::queueInputBuffer(const sp<MediaCodecBuffer> &buffer) {
407 QueueGuard guard(mSync);
408 if (!guard.isRunning()) {
409 ALOGD("[%s] No more buffers should be queued at current state.", mName);
410 return -ENOSYS;
411 }
412 return queueInputBufferInternal(buffer);
413 }
414
queueSecureInputBuffer(const sp<MediaCodecBuffer> & buffer,bool secure,const uint8_t * key,const uint8_t * iv,CryptoPlugin::Mode mode,CryptoPlugin::Pattern pattern,const CryptoPlugin::SubSample * subSamples,size_t numSubSamples,AString * errorDetailMsg)415 status_t CCodecBufferChannel::queueSecureInputBuffer(
416 const sp<MediaCodecBuffer> &buffer, bool secure, const uint8_t *key,
417 const uint8_t *iv, CryptoPlugin::Mode mode, CryptoPlugin::Pattern pattern,
418 const CryptoPlugin::SubSample *subSamples, size_t numSubSamples,
419 AString *errorDetailMsg) {
420 QueueGuard guard(mSync);
421 if (!guard.isRunning()) {
422 ALOGD("[%s] No more buffers should be queued at current state.", mName);
423 return -ENOSYS;
424 }
425
426 if (!hasCryptoOrDescrambler()) {
427 return -ENOSYS;
428 }
429 sp<EncryptedLinearBlockBuffer> encryptedBuffer((EncryptedLinearBlockBuffer *)buffer.get());
430
431 ssize_t result = -1;
432 ssize_t codecDataOffset = 0;
433 if (mCrypto != nullptr) {
434 ICrypto::DestinationBuffer destination;
435 if (secure) {
436 destination.mType = ICrypto::kDestinationTypeNativeHandle;
437 destination.mHandle = encryptedBuffer->handle();
438 } else {
439 destination.mType = ICrypto::kDestinationTypeSharedMemory;
440 destination.mSharedMemory = mDecryptDestination;
441 }
442 ICrypto::SourceBuffer source;
443 encryptedBuffer->fillSourceBuffer(&source);
444 result = mCrypto->decrypt(
445 key, iv, mode, pattern, source, buffer->offset(),
446 subSamples, numSubSamples, destination, errorDetailMsg);
447 if (result < 0) {
448 return result;
449 }
450 if (destination.mType == ICrypto::kDestinationTypeSharedMemory) {
451 encryptedBuffer->copyDecryptedContent(mDecryptDestination, result);
452 }
453 } else {
454 // Here we cast CryptoPlugin::SubSample to hardware::cas::native::V1_0::SubSample
455 // directly, the structure definitions should match as checked in DescramblerImpl.cpp.
456 hidl_vec<SubSample> hidlSubSamples;
457 hidlSubSamples.setToExternal((SubSample *)subSamples, numSubSamples, false /*own*/);
458
459 hardware::cas::native::V1_0::SharedBuffer srcBuffer;
460 encryptedBuffer->fillSourceBuffer(&srcBuffer);
461
462 DestinationBuffer dstBuffer;
463 if (secure) {
464 dstBuffer.type = BufferType::NATIVE_HANDLE;
465 dstBuffer.secureMemory = hidl_handle(encryptedBuffer->handle());
466 } else {
467 dstBuffer.type = BufferType::SHARED_MEMORY;
468 dstBuffer.nonsecureMemory = srcBuffer;
469 }
470
471 CasStatus status = CasStatus::OK;
472 hidl_string detailedError;
473 ScramblingControl sctrl = ScramblingControl::UNSCRAMBLED;
474
475 if (key != nullptr) {
476 sctrl = (ScramblingControl)key[0];
477 // Adjust for the PES offset
478 codecDataOffset = key[2] | (key[3] << 8);
479 }
480
481 auto returnVoid = mDescrambler->descramble(
482 sctrl,
483 hidlSubSamples,
484 srcBuffer,
485 0,
486 dstBuffer,
487 0,
488 [&status, &result, &detailedError] (
489 CasStatus _status, uint32_t _bytesWritten,
490 const hidl_string& _detailedError) {
491 status = _status;
492 result = (ssize_t)_bytesWritten;
493 detailedError = _detailedError;
494 });
495
496 if (!returnVoid.isOk() || status != CasStatus::OK || result < 0) {
497 ALOGI("[%s] descramble failed, trans=%s, status=%d, result=%zd",
498 mName, returnVoid.description().c_str(), status, result);
499 return UNKNOWN_ERROR;
500 }
501
502 if (result < codecDataOffset) {
503 ALOGD("invalid codec data offset: %zd, result %zd", codecDataOffset, result);
504 return BAD_VALUE;
505 }
506
507 ALOGV("[%s] descramble succeeded, %zd bytes", mName, result);
508
509 if (dstBuffer.type == BufferType::SHARED_MEMORY) {
510 encryptedBuffer->copyDecryptedContentFromMemory(result);
511 }
512 }
513
514 buffer->setRange(codecDataOffset, result - codecDataOffset);
515 return queueInputBufferInternal(buffer);
516 }
517
feedInputBufferIfAvailable()518 void CCodecBufferChannel::feedInputBufferIfAvailable() {
519 QueueGuard guard(mSync);
520 if (!guard.isRunning()) {
521 ALOGV("[%s] We're not running --- no input buffer reported", mName);
522 return;
523 }
524 feedInputBufferIfAvailableInternal();
525 }
526
feedInputBufferIfAvailableInternal()527 void CCodecBufferChannel::feedInputBufferIfAvailableInternal() {
528 if (mInputMetEos ||
529 mReorderStash.lock()->hasPending() ||
530 mPipelineWatcher.lock()->pipelineFull()) {
531 return;
532 } else {
533 Mutexed<Output>::Locked output(mOutput);
534 if (output->buffers->numClientBuffers() >= output->numSlots) {
535 return;
536 }
537 }
538 size_t numInputSlots = mInput.lock()->numSlots;
539 for (size_t i = 0; i < numInputSlots; ++i) {
540 sp<MediaCodecBuffer> inBuffer;
541 size_t index;
542 {
543 Mutexed<Input>::Locked input(mInput);
544 if (input->buffers->numClientBuffers() >= input->numSlots) {
545 return;
546 }
547 if (!input->buffers->requestNewBuffer(&index, &inBuffer)) {
548 ALOGV("[%s] no new buffer available", mName);
549 break;
550 }
551 }
552 ALOGV("[%s] new input index = %zu [%p]", mName, index, inBuffer.get());
553 mCallback->onInputBufferAvailable(index, inBuffer);
554 }
555 }
556
renderOutputBuffer(const sp<MediaCodecBuffer> & buffer,int64_t timestampNs)557 status_t CCodecBufferChannel::renderOutputBuffer(
558 const sp<MediaCodecBuffer> &buffer, int64_t timestampNs) {
559 ALOGV("[%s] renderOutputBuffer: %p", mName, buffer.get());
560 std::shared_ptr<C2Buffer> c2Buffer;
561 bool released = false;
562 {
563 Mutexed<Output>::Locked output(mOutput);
564 if (output->buffers) {
565 released = output->buffers->releaseBuffer(buffer, &c2Buffer);
566 }
567 }
568 // NOTE: some apps try to releaseOutputBuffer() with timestamp and/or render
569 // set to true.
570 sendOutputBuffers();
571 // input buffer feeding may have been gated by pending output buffers
572 feedInputBufferIfAvailable();
573 if (!c2Buffer) {
574 if (released) {
575 std::call_once(mRenderWarningFlag, [this] {
576 ALOGW("[%s] The app is calling releaseOutputBuffer() with "
577 "timestamp or render=true with non-video buffers. Apps should "
578 "call releaseOutputBuffer() with render=false for those.",
579 mName);
580 });
581 }
582 return INVALID_OPERATION;
583 }
584
585 #if 0
586 const std::vector<std::shared_ptr<const C2Info>> infoParams = c2Buffer->info();
587 ALOGV("[%s] queuing gfx buffer with %zu infos", mName, infoParams.size());
588 for (const std::shared_ptr<const C2Info> &info : infoParams) {
589 AString res;
590 for (size_t ix = 0; ix + 3 < info->size(); ix += 4) {
591 if (ix) res.append(", ");
592 res.append(*((int32_t*)info.get() + (ix / 4)));
593 }
594 ALOGV(" [%s]", res.c_str());
595 }
596 #endif
597 std::shared_ptr<const C2StreamRotationInfo::output> rotation =
598 std::static_pointer_cast<const C2StreamRotationInfo::output>(
599 c2Buffer->getInfo(C2StreamRotationInfo::output::PARAM_TYPE));
600 bool flip = rotation && (rotation->flip & 1);
601 uint32_t quarters = ((rotation ? rotation->value : 0) / 90) & 3;
602 uint32_t transform = 0;
603 switch (quarters) {
604 case 0: // no rotation
605 transform = flip ? HAL_TRANSFORM_FLIP_H : 0;
606 break;
607 case 1: // 90 degrees counter-clockwise
608 transform = flip ? (HAL_TRANSFORM_FLIP_V | HAL_TRANSFORM_ROT_90)
609 : HAL_TRANSFORM_ROT_270;
610 break;
611 case 2: // 180 degrees
612 transform = flip ? HAL_TRANSFORM_FLIP_V : HAL_TRANSFORM_ROT_180;
613 break;
614 case 3: // 90 degrees clockwise
615 transform = flip ? (HAL_TRANSFORM_FLIP_H | HAL_TRANSFORM_ROT_90)
616 : HAL_TRANSFORM_ROT_90;
617 break;
618 }
619
620 std::shared_ptr<const C2StreamSurfaceScalingInfo::output> surfaceScaling =
621 std::static_pointer_cast<const C2StreamSurfaceScalingInfo::output>(
622 c2Buffer->getInfo(C2StreamSurfaceScalingInfo::output::PARAM_TYPE));
623 uint32_t videoScalingMode = NATIVE_WINDOW_SCALING_MODE_SCALE_TO_WINDOW;
624 if (surfaceScaling) {
625 videoScalingMode = surfaceScaling->value;
626 }
627
628 // Use dataspace from format as it has the default aspects already applied
629 android_dataspace_t dataSpace = HAL_DATASPACE_UNKNOWN; // this is 0
630 (void)buffer->format()->findInt32("android._dataspace", (int32_t *)&dataSpace);
631
632 // HDR static info
633 std::shared_ptr<const C2StreamHdrStaticInfo::output> hdrStaticInfo =
634 std::static_pointer_cast<const C2StreamHdrStaticInfo::output>(
635 c2Buffer->getInfo(C2StreamHdrStaticInfo::output::PARAM_TYPE));
636
637 // HDR10 plus info
638 std::shared_ptr<const C2StreamHdr10PlusInfo::output> hdr10PlusInfo =
639 std::static_pointer_cast<const C2StreamHdr10PlusInfo::output>(
640 c2Buffer->getInfo(C2StreamHdr10PlusInfo::output::PARAM_TYPE));
641
642 {
643 Mutexed<OutputSurface>::Locked output(mOutputSurface);
644 if (output->surface == nullptr) {
645 ALOGI("[%s] cannot render buffer without surface", mName);
646 return OK;
647 }
648 }
649
650 std::vector<C2ConstGraphicBlock> blocks = c2Buffer->data().graphicBlocks();
651 if (blocks.size() != 1u) {
652 ALOGD("[%s] expected 1 graphic block, but got %zu", mName, blocks.size());
653 return UNKNOWN_ERROR;
654 }
655 const C2ConstGraphicBlock &block = blocks.front();
656
657 // TODO: revisit this after C2Fence implementation.
658 android::IGraphicBufferProducer::QueueBufferInput qbi(
659 timestampNs,
660 false, // droppable
661 dataSpace,
662 Rect(blocks.front().crop().left,
663 blocks.front().crop().top,
664 blocks.front().crop().right(),
665 blocks.front().crop().bottom()),
666 videoScalingMode,
667 transform,
668 Fence::NO_FENCE, 0);
669 if (hdrStaticInfo || hdr10PlusInfo) {
670 HdrMetadata hdr;
671 if (hdrStaticInfo) {
672 struct android_smpte2086_metadata smpte2086_meta = {
673 .displayPrimaryRed = {
674 hdrStaticInfo->mastering.red.x, hdrStaticInfo->mastering.red.y
675 },
676 .displayPrimaryGreen = {
677 hdrStaticInfo->mastering.green.x, hdrStaticInfo->mastering.green.y
678 },
679 .displayPrimaryBlue = {
680 hdrStaticInfo->mastering.blue.x, hdrStaticInfo->mastering.blue.y
681 },
682 .whitePoint = {
683 hdrStaticInfo->mastering.white.x, hdrStaticInfo->mastering.white.y
684 },
685 .maxLuminance = hdrStaticInfo->mastering.maxLuminance,
686 .minLuminance = hdrStaticInfo->mastering.minLuminance,
687 };
688
689 struct android_cta861_3_metadata cta861_meta = {
690 .maxContentLightLevel = hdrStaticInfo->maxCll,
691 .maxFrameAverageLightLevel = hdrStaticInfo->maxFall,
692 };
693
694 hdr.validTypes = HdrMetadata::SMPTE2086 | HdrMetadata::CTA861_3;
695 hdr.smpte2086 = smpte2086_meta;
696 hdr.cta8613 = cta861_meta;
697 }
698 if (hdr10PlusInfo) {
699 hdr.validTypes |= HdrMetadata::HDR10PLUS;
700 hdr.hdr10plus.assign(
701 hdr10PlusInfo->m.value,
702 hdr10PlusInfo->m.value + hdr10PlusInfo->flexCount());
703 }
704 qbi.setHdrMetadata(hdr);
705 }
706 // we don't have dirty regions
707 qbi.setSurfaceDamage(Region::INVALID_REGION);
708 android::IGraphicBufferProducer::QueueBufferOutput qbo;
709 status_t result = mComponent->queueToOutputSurface(block, qbi, &qbo);
710 if (result != OK) {
711 ALOGI("[%s] queueBuffer failed: %d", mName, result);
712 return result;
713 }
714 ALOGV("[%s] queue buffer successful", mName);
715
716 int64_t mediaTimeUs = 0;
717 (void)buffer->meta()->findInt64("timeUs", &mediaTimeUs);
718 mCCodecCallback->onOutputFramesRendered(mediaTimeUs, timestampNs);
719
720 return OK;
721 }
722
discardBuffer(const sp<MediaCodecBuffer> & buffer)723 status_t CCodecBufferChannel::discardBuffer(const sp<MediaCodecBuffer> &buffer) {
724 ALOGV("[%s] discardBuffer: %p", mName, buffer.get());
725 bool released = false;
726 {
727 Mutexed<Input>::Locked input(mInput);
728 if (input->buffers && input->buffers->releaseBuffer(buffer, nullptr, true)) {
729 released = true;
730 }
731 }
732 {
733 Mutexed<Output>::Locked output(mOutput);
734 if (output->buffers && output->buffers->releaseBuffer(buffer, nullptr)) {
735 released = true;
736 }
737 }
738 if (released) {
739 sendOutputBuffers();
740 feedInputBufferIfAvailable();
741 } else {
742 ALOGD("[%s] MediaCodec discarded an unknown buffer", mName);
743 }
744 return OK;
745 }
746
getInputBufferArray(Vector<sp<MediaCodecBuffer>> * array)747 void CCodecBufferChannel::getInputBufferArray(Vector<sp<MediaCodecBuffer>> *array) {
748 array->clear();
749 Mutexed<Input>::Locked input(mInput);
750
751 if (!input->buffers->isArrayMode()) {
752 input->buffers = input->buffers->toArrayMode(input->numSlots);
753 }
754
755 input->buffers->getArray(array);
756 }
757
getOutputBufferArray(Vector<sp<MediaCodecBuffer>> * array)758 void CCodecBufferChannel::getOutputBufferArray(Vector<sp<MediaCodecBuffer>> *array) {
759 array->clear();
760 Mutexed<Output>::Locked output(mOutput);
761
762 if (!output->buffers->isArrayMode()) {
763 output->buffers = output->buffers->toArrayMode(output->numSlots);
764 }
765
766 output->buffers->getArray(array);
767 }
768
start(const sp<AMessage> & inputFormat,const sp<AMessage> & outputFormat)769 status_t CCodecBufferChannel::start(
770 const sp<AMessage> &inputFormat, const sp<AMessage> &outputFormat) {
771 C2StreamBufferTypeSetting::input iStreamFormat(0u);
772 C2StreamBufferTypeSetting::output oStreamFormat(0u);
773 C2PortReorderBufferDepthTuning::output reorderDepth;
774 C2PortReorderKeySetting::output reorderKey;
775 C2PortActualDelayTuning::input inputDelay(0);
776 C2PortActualDelayTuning::output outputDelay(0);
777 C2ActualPipelineDelayTuning pipelineDelay(0);
778
779 c2_status_t err = mComponent->query(
780 {
781 &iStreamFormat,
782 &oStreamFormat,
783 &reorderDepth,
784 &reorderKey,
785 &inputDelay,
786 &pipelineDelay,
787 &outputDelay,
788 },
789 {},
790 C2_DONT_BLOCK,
791 nullptr);
792 if (err == C2_BAD_INDEX) {
793 if (!iStreamFormat || !oStreamFormat) {
794 return UNKNOWN_ERROR;
795 }
796 } else if (err != C2_OK) {
797 return UNKNOWN_ERROR;
798 }
799
800 {
801 Mutexed<ReorderStash>::Locked reorder(mReorderStash);
802 reorder->clear();
803 if (reorderDepth) {
804 reorder->setDepth(reorderDepth.value);
805 }
806 if (reorderKey) {
807 reorder->setKey(reorderKey.value);
808 }
809 }
810
811 uint32_t inputDelayValue = inputDelay ? inputDelay.value : 0;
812 uint32_t pipelineDelayValue = pipelineDelay ? pipelineDelay.value : 0;
813 uint32_t outputDelayValue = outputDelay ? outputDelay.value : 0;
814
815 size_t numInputSlots = inputDelayValue + pipelineDelayValue + kSmoothnessFactor;
816 size_t numOutputSlots = outputDelayValue + kSmoothnessFactor;
817
818 // TODO: get this from input format
819 bool secure = mComponent->getName().find(".secure") != std::string::npos;
820
821 std::shared_ptr<C2AllocatorStore> allocatorStore = GetCodec2PlatformAllocatorStore();
822 int poolMask = property_get_int32(
823 "debug.stagefright.c2-poolmask",
824 1 << C2PlatformAllocatorStore::ION |
825 1 << C2PlatformAllocatorStore::BUFFERQUEUE);
826
827 if (inputFormat != nullptr) {
828 bool graphic = (iStreamFormat.value == C2BufferData::GRAPHIC);
829 std::shared_ptr<C2BlockPool> pool;
830 {
831 Mutexed<BlockPools>::Locked pools(mBlockPools);
832
833 // set default allocator ID.
834 pools->inputAllocatorId = (graphic) ? C2PlatformAllocatorStore::GRALLOC
835 : C2PlatformAllocatorStore::ION;
836
837 // query C2PortAllocatorsTuning::input from component. If an allocator ID is obtained
838 // from component, create the input block pool with given ID. Otherwise, use default IDs.
839 std::vector<std::unique_ptr<C2Param>> params;
840 err = mComponent->query({ },
841 { C2PortAllocatorsTuning::input::PARAM_TYPE },
842 C2_DONT_BLOCK,
843 ¶ms);
844 if ((err != C2_OK && err != C2_BAD_INDEX) || params.size() != 1) {
845 ALOGD("[%s] Query input allocators returned %zu params => %s (%u)",
846 mName, params.size(), asString(err), err);
847 } else if (err == C2_OK && params.size() == 1) {
848 C2PortAllocatorsTuning::input *inputAllocators =
849 C2PortAllocatorsTuning::input::From(params[0].get());
850 if (inputAllocators && inputAllocators->flexCount() > 0) {
851 std::shared_ptr<C2Allocator> allocator;
852 // verify allocator IDs and resolve default allocator
853 allocatorStore->fetchAllocator(inputAllocators->m.values[0], &allocator);
854 if (allocator) {
855 pools->inputAllocatorId = allocator->getId();
856 } else {
857 ALOGD("[%s] component requested invalid input allocator ID %u",
858 mName, inputAllocators->m.values[0]);
859 }
860 }
861 }
862
863 // TODO: use C2Component wrapper to associate this pool with ourselves
864 if ((poolMask >> pools->inputAllocatorId) & 1) {
865 err = CreateCodec2BlockPool(pools->inputAllocatorId, nullptr, &pool);
866 ALOGD("[%s] Created input block pool with allocatorID %u => poolID %llu - %s (%d)",
867 mName, pools->inputAllocatorId,
868 (unsigned long long)(pool ? pool->getLocalId() : 111000111),
869 asString(err), err);
870 } else {
871 err = C2_NOT_FOUND;
872 }
873 if (err != C2_OK) {
874 C2BlockPool::local_id_t inputPoolId =
875 graphic ? C2BlockPool::BASIC_GRAPHIC : C2BlockPool::BASIC_LINEAR;
876 err = GetCodec2BlockPool(inputPoolId, nullptr, &pool);
877 ALOGD("[%s] Using basic input block pool with poolID %llu => got %llu - %s (%d)",
878 mName, (unsigned long long)inputPoolId,
879 (unsigned long long)(pool ? pool->getLocalId() : 111000111),
880 asString(err), err);
881 if (err != C2_OK) {
882 return NO_MEMORY;
883 }
884 }
885 pools->inputPool = pool;
886 }
887
888 bool forceArrayMode = false;
889 Mutexed<Input>::Locked input(mInput);
890 input->inputDelay = inputDelayValue;
891 input->pipelineDelay = pipelineDelayValue;
892 input->numSlots = numInputSlots;
893 input->extraBuffers.flush();
894 input->numExtraSlots = 0u;
895 if (graphic) {
896 if (mInputSurface) {
897 input->buffers.reset(new DummyInputBuffers(mName));
898 } else if (mMetaMode == MODE_ANW) {
899 input->buffers.reset(new GraphicMetadataInputBuffers(mName));
900 // This is to ensure buffers do not get released prematurely.
901 // TODO: handle this without going into array mode
902 forceArrayMode = true;
903 } else {
904 input->buffers.reset(new GraphicInputBuffers(numInputSlots, mName));
905 }
906 } else {
907 if (hasCryptoOrDescrambler()) {
908 int32_t capacity = kLinearBufferSize;
909 (void)inputFormat->findInt32(KEY_MAX_INPUT_SIZE, &capacity);
910 if ((size_t)capacity > kMaxLinearBufferSize) {
911 ALOGD("client requested %d, capped to %zu", capacity, kMaxLinearBufferSize);
912 capacity = kMaxLinearBufferSize;
913 }
914 if (mDealer == nullptr) {
915 mDealer = new MemoryDealer(
916 align(capacity, MemoryDealer::getAllocationAlignment())
917 * (numInputSlots + 1),
918 "EncryptedLinearInputBuffers");
919 mDecryptDestination = mDealer->allocate((size_t)capacity);
920 }
921 if (mCrypto != nullptr && mHeapSeqNum < 0) {
922 mHeapSeqNum = mCrypto->setHeap(mDealer->getMemoryHeap());
923 } else {
924 mHeapSeqNum = -1;
925 }
926 input->buffers.reset(new EncryptedLinearInputBuffers(
927 secure, mDealer, mCrypto, mHeapSeqNum, (size_t)capacity,
928 numInputSlots, mName));
929 forceArrayMode = true;
930 } else {
931 input->buffers.reset(new LinearInputBuffers(mName));
932 }
933 }
934 input->buffers->setFormat(inputFormat);
935
936 if (err == C2_OK) {
937 input->buffers->setPool(pool);
938 } else {
939 // TODO: error
940 }
941
942 if (forceArrayMode) {
943 input->buffers = input->buffers->toArrayMode(numInputSlots);
944 }
945 }
946
947 if (outputFormat != nullptr) {
948 sp<IGraphicBufferProducer> outputSurface;
949 uint32_t outputGeneration;
950 {
951 Mutexed<OutputSurface>::Locked output(mOutputSurface);
952 output->maxDequeueBuffers = numOutputSlots +
953 reorderDepth.value + kRenderingDepth;
954 if (!secure) {
955 output->maxDequeueBuffers += numInputSlots;
956 }
957 outputSurface = output->surface ?
958 output->surface->getIGraphicBufferProducer() : nullptr;
959 if (outputSurface) {
960 output->surface->setMaxDequeuedBufferCount(output->maxDequeueBuffers);
961 }
962 outputGeneration = output->generation;
963 }
964
965 bool graphic = (oStreamFormat.value == C2BufferData::GRAPHIC);
966 C2BlockPool::local_id_t outputPoolId_;
967
968 {
969 Mutexed<BlockPools>::Locked pools(mBlockPools);
970
971 // set default allocator ID.
972 pools->outputAllocatorId = (graphic) ? C2PlatformAllocatorStore::GRALLOC
973 : C2PlatformAllocatorStore::ION;
974
975 // query C2PortAllocatorsTuning::output from component, or use default allocator if
976 // unsuccessful.
977 std::vector<std::unique_ptr<C2Param>> params;
978 err = mComponent->query({ },
979 { C2PortAllocatorsTuning::output::PARAM_TYPE },
980 C2_DONT_BLOCK,
981 ¶ms);
982 if ((err != C2_OK && err != C2_BAD_INDEX) || params.size() != 1) {
983 ALOGD("[%s] Query output allocators returned %zu params => %s (%u)",
984 mName, params.size(), asString(err), err);
985 } else if (err == C2_OK && params.size() == 1) {
986 C2PortAllocatorsTuning::output *outputAllocators =
987 C2PortAllocatorsTuning::output::From(params[0].get());
988 if (outputAllocators && outputAllocators->flexCount() > 0) {
989 std::shared_ptr<C2Allocator> allocator;
990 // verify allocator IDs and resolve default allocator
991 allocatorStore->fetchAllocator(outputAllocators->m.values[0], &allocator);
992 if (allocator) {
993 pools->outputAllocatorId = allocator->getId();
994 } else {
995 ALOGD("[%s] component requested invalid output allocator ID %u",
996 mName, outputAllocators->m.values[0]);
997 }
998 }
999 }
1000
1001 // use bufferqueue if outputting to a surface.
1002 // query C2PortSurfaceAllocatorTuning::output from component, or use default allocator
1003 // if unsuccessful.
1004 if (outputSurface) {
1005 params.clear();
1006 err = mComponent->query({ },
1007 { C2PortSurfaceAllocatorTuning::output::PARAM_TYPE },
1008 C2_DONT_BLOCK,
1009 ¶ms);
1010 if ((err != C2_OK && err != C2_BAD_INDEX) || params.size() != 1) {
1011 ALOGD("[%s] Query output surface allocator returned %zu params => %s (%u)",
1012 mName, params.size(), asString(err), err);
1013 } else if (err == C2_OK && params.size() == 1) {
1014 C2PortSurfaceAllocatorTuning::output *surfaceAllocator =
1015 C2PortSurfaceAllocatorTuning::output::From(params[0].get());
1016 if (surfaceAllocator) {
1017 std::shared_ptr<C2Allocator> allocator;
1018 // verify allocator IDs and resolve default allocator
1019 allocatorStore->fetchAllocator(surfaceAllocator->value, &allocator);
1020 if (allocator) {
1021 pools->outputAllocatorId = allocator->getId();
1022 } else {
1023 ALOGD("[%s] component requested invalid surface output allocator ID %u",
1024 mName, surfaceAllocator->value);
1025 err = C2_BAD_VALUE;
1026 }
1027 }
1028 }
1029 if (pools->outputAllocatorId == C2PlatformAllocatorStore::GRALLOC
1030 && err != C2_OK
1031 && ((poolMask >> C2PlatformAllocatorStore::BUFFERQUEUE) & 1)) {
1032 pools->outputAllocatorId = C2PlatformAllocatorStore::BUFFERQUEUE;
1033 }
1034 }
1035
1036 if ((poolMask >> pools->outputAllocatorId) & 1) {
1037 err = mComponent->createBlockPool(
1038 pools->outputAllocatorId, &pools->outputPoolId, &pools->outputPoolIntf);
1039 ALOGI("[%s] Created output block pool with allocatorID %u => poolID %llu - %s",
1040 mName, pools->outputAllocatorId,
1041 (unsigned long long)pools->outputPoolId,
1042 asString(err));
1043 } else {
1044 err = C2_NOT_FOUND;
1045 }
1046 if (err != C2_OK) {
1047 // use basic pool instead
1048 pools->outputPoolId =
1049 graphic ? C2BlockPool::BASIC_GRAPHIC : C2BlockPool::BASIC_LINEAR;
1050 }
1051
1052 // Configure output block pool ID as parameter C2PortBlockPoolsTuning::output to
1053 // component.
1054 std::unique_ptr<C2PortBlockPoolsTuning::output> poolIdsTuning =
1055 C2PortBlockPoolsTuning::output::AllocUnique({ pools->outputPoolId });
1056
1057 std::vector<std::unique_ptr<C2SettingResult>> failures;
1058 err = mComponent->config({ poolIdsTuning.get() }, C2_MAY_BLOCK, &failures);
1059 ALOGD("[%s] Configured output block pool ids %llu => %s",
1060 mName, (unsigned long long)poolIdsTuning->m.values[0], asString(err));
1061 outputPoolId_ = pools->outputPoolId;
1062 }
1063
1064 Mutexed<Output>::Locked output(mOutput);
1065 output->outputDelay = outputDelayValue;
1066 output->numSlots = numOutputSlots;
1067 if (graphic) {
1068 if (outputSurface) {
1069 output->buffers.reset(new GraphicOutputBuffers(mName));
1070 } else {
1071 output->buffers.reset(new RawGraphicOutputBuffers(numOutputSlots, mName));
1072 }
1073 } else {
1074 output->buffers.reset(new LinearOutputBuffers(mName));
1075 }
1076 output->buffers->setFormat(outputFormat);
1077
1078
1079 // Try to set output surface to created block pool if given.
1080 if (outputSurface) {
1081 mComponent->setOutputSurface(
1082 outputPoolId_,
1083 outputSurface,
1084 outputGeneration);
1085 }
1086
1087 if (oStreamFormat.value == C2BufferData::LINEAR
1088 && mComponentName.find("c2.qti.") == std::string::npos) {
1089 // WORKAROUND: if we're using early CSD workaround we convert to
1090 // array mode, to appease apps assuming the output
1091 // buffers to be of the same size.
1092 output->buffers = output->buffers->toArrayMode(numOutputSlots);
1093
1094 int32_t channelCount;
1095 int32_t sampleRate;
1096 if (outputFormat->findInt32(KEY_CHANNEL_COUNT, &channelCount)
1097 && outputFormat->findInt32(KEY_SAMPLE_RATE, &sampleRate)) {
1098 int32_t delay = 0;
1099 int32_t padding = 0;;
1100 if (!outputFormat->findInt32("encoder-delay", &delay)) {
1101 delay = 0;
1102 }
1103 if (!outputFormat->findInt32("encoder-padding", &padding)) {
1104 padding = 0;
1105 }
1106 if (delay || padding) {
1107 // We need write access to the buffers, and we're already in
1108 // array mode.
1109 output->buffers->initSkipCutBuffer(delay, padding, sampleRate, channelCount);
1110 }
1111 }
1112 }
1113 }
1114
1115 // Set up pipeline control. This has to be done after mInputBuffers and
1116 // mOutputBuffers are initialized to make sure that lingering callbacks
1117 // about buffers from the previous generation do not interfere with the
1118 // newly initialized pipeline capacity.
1119
1120 {
1121 Mutexed<PipelineWatcher>::Locked watcher(mPipelineWatcher);
1122 watcher->inputDelay(inputDelayValue)
1123 .pipelineDelay(pipelineDelayValue)
1124 .outputDelay(outputDelayValue)
1125 .smoothnessFactor(kSmoothnessFactor);
1126 watcher->flush();
1127 }
1128
1129 mInputMetEos = false;
1130 mSync.start();
1131 return OK;
1132 }
1133
requestInitialInputBuffers()1134 status_t CCodecBufferChannel::requestInitialInputBuffers() {
1135 if (mInputSurface) {
1136 return OK;
1137 }
1138
1139 C2StreamBufferTypeSetting::output oStreamFormat(0u);
1140 c2_status_t err = mComponent->query({ &oStreamFormat }, {}, C2_DONT_BLOCK, nullptr);
1141 if (err != C2_OK) {
1142 return UNKNOWN_ERROR;
1143 }
1144 size_t numInputSlots = mInput.lock()->numSlots;
1145 std::vector<sp<MediaCodecBuffer>> toBeQueued;
1146 for (size_t i = 0; i < numInputSlots; ++i) {
1147 size_t index;
1148 sp<MediaCodecBuffer> buffer;
1149 {
1150 Mutexed<Input>::Locked input(mInput);
1151 if (!input->buffers->requestNewBuffer(&index, &buffer)) {
1152 if (i == 0) {
1153 ALOGW("[%s] start: cannot allocate memory at all", mName);
1154 return NO_MEMORY;
1155 } else {
1156 ALOGV("[%s] start: cannot allocate memory, only %zu buffers allocated",
1157 mName, i);
1158 }
1159 break;
1160 }
1161 }
1162 if (buffer) {
1163 Mutexed<std::list<sp<ABuffer>>>::Locked configs(mFlushedConfigs);
1164 ALOGV("[%s] input buffer %zu available", mName, index);
1165 bool post = true;
1166 if (!configs->empty()) {
1167 sp<ABuffer> config = configs->front();
1168 configs->pop_front();
1169 if (buffer->capacity() >= config->size()) {
1170 memcpy(buffer->base(), config->data(), config->size());
1171 buffer->setRange(0, config->size());
1172 buffer->meta()->clear();
1173 buffer->meta()->setInt64("timeUs", 0);
1174 buffer->meta()->setInt32("csd", 1);
1175 post = false;
1176 } else {
1177 ALOGD("[%s] buffer capacity too small for the config (%zu < %zu)",
1178 mName, buffer->capacity(), config->size());
1179 }
1180 } else if (oStreamFormat.value == C2BufferData::LINEAR && i == 0
1181 && mComponentName.find("c2.qti.") == std::string::npos) {
1182 // WORKAROUND: Some apps expect CSD available without queueing
1183 // any input. Queue an empty buffer to get the CSD.
1184 buffer->setRange(0, 0);
1185 buffer->meta()->clear();
1186 buffer->meta()->setInt64("timeUs", 0);
1187 post = false;
1188 }
1189 if (post) {
1190 mCallback->onInputBufferAvailable(index, buffer);
1191 } else {
1192 toBeQueued.emplace_back(buffer);
1193 }
1194 }
1195 }
1196 for (const sp<MediaCodecBuffer> &buffer : toBeQueued) {
1197 if (queueInputBufferInternal(buffer) != OK) {
1198 ALOGV("[%s] Error while queueing initial buffers", mName);
1199 }
1200 }
1201 return OK;
1202 }
1203
stop()1204 void CCodecBufferChannel::stop() {
1205 mSync.stop();
1206 mFirstValidFrameIndex = mFrameIndex.load(std::memory_order_relaxed);
1207 if (mInputSurface != nullptr) {
1208 mInputSurface.reset();
1209 }
1210 }
1211
flush(const std::list<std::unique_ptr<C2Work>> & flushedWork)1212 void CCodecBufferChannel::flush(const std::list<std::unique_ptr<C2Work>> &flushedWork) {
1213 ALOGV("[%s] flush", mName);
1214 {
1215 Mutexed<std::list<sp<ABuffer>>>::Locked configs(mFlushedConfigs);
1216 for (const std::unique_ptr<C2Work> &work : flushedWork) {
1217 if (!(work->input.flags & C2FrameData::FLAG_CODEC_CONFIG)) {
1218 continue;
1219 }
1220 if (work->input.buffers.empty()
1221 || work->input.buffers.front()->data().linearBlocks().empty()) {
1222 ALOGD("[%s] no linear codec config data found", mName);
1223 continue;
1224 }
1225 C2ReadView view =
1226 work->input.buffers.front()->data().linearBlocks().front().map().get();
1227 if (view.error() != C2_OK) {
1228 ALOGD("[%s] failed to map flushed codec config data: %d", mName, view.error());
1229 continue;
1230 }
1231 configs->push_back(ABuffer::CreateAsCopy(view.data(), view.capacity()));
1232 ALOGV("[%s] stashed flushed codec config data (size=%u)", mName, view.capacity());
1233 }
1234 }
1235 {
1236 Mutexed<Input>::Locked input(mInput);
1237 input->buffers->flush();
1238 input->extraBuffers.flush();
1239 }
1240 {
1241 Mutexed<Output>::Locked output(mOutput);
1242 output->buffers->flush(flushedWork);
1243 }
1244 mReorderStash.lock()->flush();
1245 mPipelineWatcher.lock()->flush();
1246 }
1247
onWorkDone(std::unique_ptr<C2Work> work,const sp<AMessage> & outputFormat,const C2StreamInitDataInfo::output * initData)1248 void CCodecBufferChannel::onWorkDone(
1249 std::unique_ptr<C2Work> work, const sp<AMessage> &outputFormat,
1250 const C2StreamInitDataInfo::output *initData) {
1251 if (handleWork(std::move(work), outputFormat, initData)) {
1252 feedInputBufferIfAvailable();
1253 }
1254 }
1255
onInputBufferDone(uint64_t frameIndex,size_t arrayIndex)1256 void CCodecBufferChannel::onInputBufferDone(
1257 uint64_t frameIndex, size_t arrayIndex) {
1258 if (mInputSurface) {
1259 return;
1260 }
1261 std::shared_ptr<C2Buffer> buffer =
1262 mPipelineWatcher.lock()->onInputBufferReleased(frameIndex, arrayIndex);
1263 bool newInputSlotAvailable;
1264 {
1265 Mutexed<Input>::Locked input(mInput);
1266 newInputSlotAvailable = input->buffers->expireComponentBuffer(buffer);
1267 if (!newInputSlotAvailable) {
1268 (void)input->extraBuffers.expireComponentBuffer(buffer);
1269 }
1270 }
1271 if (newInputSlotAvailable) {
1272 feedInputBufferIfAvailable();
1273 }
1274 }
1275
handleWork(std::unique_ptr<C2Work> work,const sp<AMessage> & outputFormat,const C2StreamInitDataInfo::output * initData)1276 bool CCodecBufferChannel::handleWork(
1277 std::unique_ptr<C2Work> work,
1278 const sp<AMessage> &outputFormat,
1279 const C2StreamInitDataInfo::output *initData) {
1280 if (outputFormat != nullptr) {
1281 Mutexed<Output>::Locked output(mOutput);
1282 ALOGD("[%s] onWorkDone: output format changed to %s",
1283 mName, outputFormat->debugString().c_str());
1284 output->buffers->setFormat(outputFormat);
1285
1286 AString mediaType;
1287 if (outputFormat->findString(KEY_MIME, &mediaType)
1288 && mediaType == MIMETYPE_AUDIO_RAW) {
1289 int32_t channelCount;
1290 int32_t sampleRate;
1291 if (outputFormat->findInt32(KEY_CHANNEL_COUNT, &channelCount)
1292 && outputFormat->findInt32(KEY_SAMPLE_RATE, &sampleRate)) {
1293 output->buffers->updateSkipCutBuffer(sampleRate, channelCount);
1294 }
1295 }
1296 }
1297
1298 if ((work->input.ordinal.frameIndex - mFirstValidFrameIndex.load()).peek() < 0) {
1299 // Discard frames from previous generation.
1300 ALOGD("[%s] Discard frames from previous generation.", mName);
1301 return false;
1302 }
1303
1304 if (mInputSurface == nullptr && (work->worklets.size() != 1u
1305 || !work->worklets.front()
1306 || !(work->worklets.front()->output.flags & C2FrameData::FLAG_INCOMPLETE))) {
1307 mPipelineWatcher.lock()->onWorkDone(work->input.ordinal.frameIndex.peeku());
1308 }
1309
1310 if (work->result == C2_NOT_FOUND) {
1311 ALOGD("[%s] flushed work; ignored.", mName);
1312 return true;
1313 }
1314
1315 if (work->result != C2_OK) {
1316 ALOGD("[%s] work failed to complete: %d", mName, work->result);
1317 mCCodecCallback->onError(work->result, ACTION_CODE_FATAL);
1318 return false;
1319 }
1320
1321 // NOTE: MediaCodec usage supposedly have only one worklet
1322 if (work->worklets.size() != 1u) {
1323 ALOGI("[%s] onWorkDone: incorrect number of worklets: %zu",
1324 mName, work->worklets.size());
1325 mCCodecCallback->onError(UNKNOWN_ERROR, ACTION_CODE_FATAL);
1326 return false;
1327 }
1328
1329 const std::unique_ptr<C2Worklet> &worklet = work->worklets.front();
1330
1331 std::shared_ptr<C2Buffer> buffer;
1332 // NOTE: MediaCodec usage supposedly have only one output stream.
1333 if (worklet->output.buffers.size() > 1u) {
1334 ALOGI("[%s] onWorkDone: incorrect number of output buffers: %zu",
1335 mName, worklet->output.buffers.size());
1336 mCCodecCallback->onError(UNKNOWN_ERROR, ACTION_CODE_FATAL);
1337 return false;
1338 } else if (worklet->output.buffers.size() == 1u) {
1339 buffer = worklet->output.buffers[0];
1340 if (!buffer) {
1341 ALOGD("[%s] onWorkDone: nullptr found in buffers; ignored.", mName);
1342 }
1343 }
1344
1345 std::optional<uint32_t> newInputDelay, newPipelineDelay;
1346 while (!worklet->output.configUpdate.empty()) {
1347 std::unique_ptr<C2Param> param;
1348 worklet->output.configUpdate.back().swap(param);
1349 worklet->output.configUpdate.pop_back();
1350 switch (param->coreIndex().coreIndex()) {
1351 case C2PortReorderBufferDepthTuning::CORE_INDEX: {
1352 C2PortReorderBufferDepthTuning::output reorderDepth;
1353 if (reorderDepth.updateFrom(*param)) {
1354 bool secure = mComponent->getName().find(".secure") != std::string::npos;
1355 mReorderStash.lock()->setDepth(reorderDepth.value);
1356 ALOGV("[%s] onWorkDone: updated reorder depth to %u",
1357 mName, reorderDepth.value);
1358 size_t numOutputSlots = mOutput.lock()->numSlots;
1359 size_t numInputSlots = mInput.lock()->numSlots;
1360 Mutexed<OutputSurface>::Locked output(mOutputSurface);
1361 output->maxDequeueBuffers = numOutputSlots +
1362 reorderDepth.value + kRenderingDepth;
1363 if (!secure) {
1364 output->maxDequeueBuffers += numInputSlots;
1365 }
1366 if (output->surface) {
1367 output->surface->setMaxDequeuedBufferCount(output->maxDequeueBuffers);
1368 }
1369 } else {
1370 ALOGD("[%s] onWorkDone: failed to read reorder depth", mName);
1371 }
1372 break;
1373 }
1374 case C2PortReorderKeySetting::CORE_INDEX: {
1375 C2PortReorderKeySetting::output reorderKey;
1376 if (reorderKey.updateFrom(*param)) {
1377 mReorderStash.lock()->setKey(reorderKey.value);
1378 ALOGV("[%s] onWorkDone: updated reorder key to %u",
1379 mName, reorderKey.value);
1380 } else {
1381 ALOGD("[%s] onWorkDone: failed to read reorder key", mName);
1382 }
1383 break;
1384 }
1385 case C2PortActualDelayTuning::CORE_INDEX: {
1386 if (param->isGlobal()) {
1387 C2ActualPipelineDelayTuning pipelineDelay;
1388 if (pipelineDelay.updateFrom(*param)) {
1389 ALOGV("[%s] onWorkDone: updating pipeline delay %u",
1390 mName, pipelineDelay.value);
1391 newPipelineDelay = pipelineDelay.value;
1392 (void)mPipelineWatcher.lock()->pipelineDelay(pipelineDelay.value);
1393 }
1394 }
1395 if (param->forInput()) {
1396 C2PortActualDelayTuning::input inputDelay;
1397 if (inputDelay.updateFrom(*param)) {
1398 ALOGV("[%s] onWorkDone: updating input delay %u",
1399 mName, inputDelay.value);
1400 newInputDelay = inputDelay.value;
1401 (void)mPipelineWatcher.lock()->inputDelay(inputDelay.value);
1402 }
1403 }
1404 if (param->forOutput()) {
1405 C2PortActualDelayTuning::output outputDelay;
1406 if (outputDelay.updateFrom(*param)) {
1407 ALOGV("[%s] onWorkDone: updating output delay %u",
1408 mName, outputDelay.value);
1409 bool secure = mComponent->getName().find(".secure") != std::string::npos;
1410 (void)mPipelineWatcher.lock()->outputDelay(outputDelay.value);
1411
1412 bool outputBuffersChanged = false;
1413 size_t numOutputSlots = 0;
1414 size_t numInputSlots = mInput.lock()->numSlots;
1415 {
1416 Mutexed<Output>::Locked output(mOutput);
1417 output->outputDelay = outputDelay.value;
1418 numOutputSlots = outputDelay.value + kSmoothnessFactor;
1419 if (output->numSlots < numOutputSlots) {
1420 output->numSlots = numOutputSlots;
1421 if (output->buffers->isArrayMode()) {
1422 OutputBuffersArray *array =
1423 (OutputBuffersArray *)output->buffers.get();
1424 ALOGV("[%s] onWorkDone: growing output buffer array to %zu",
1425 mName, numOutputSlots);
1426 array->grow(numOutputSlots);
1427 outputBuffersChanged = true;
1428 }
1429 }
1430 numOutputSlots = output->numSlots;
1431 }
1432
1433 if (outputBuffersChanged) {
1434 mCCodecCallback->onOutputBuffersChanged();
1435 }
1436
1437 uint32_t depth = mReorderStash.lock()->depth();
1438 Mutexed<OutputSurface>::Locked output(mOutputSurface);
1439 output->maxDequeueBuffers = numOutputSlots + depth + kRenderingDepth;
1440 if (!secure) {
1441 output->maxDequeueBuffers += numInputSlots;
1442 }
1443 if (output->surface) {
1444 output->surface->setMaxDequeuedBufferCount(output->maxDequeueBuffers);
1445 }
1446 }
1447 }
1448 break;
1449 }
1450 default:
1451 ALOGV("[%s] onWorkDone: unrecognized config update (%08X)",
1452 mName, param->index());
1453 break;
1454 }
1455 }
1456 if (newInputDelay || newPipelineDelay) {
1457 Mutexed<Input>::Locked input(mInput);
1458 size_t newNumSlots =
1459 newInputDelay.value_or(input->inputDelay) +
1460 newPipelineDelay.value_or(input->pipelineDelay) +
1461 kSmoothnessFactor;
1462 if (input->buffers->isArrayMode()) {
1463 if (input->numSlots >= newNumSlots) {
1464 input->numExtraSlots = 0;
1465 } else {
1466 input->numExtraSlots = newNumSlots - input->numSlots;
1467 }
1468 ALOGV("[%s] onWorkDone: updated number of extra slots to %zu (input array mode)",
1469 mName, input->numExtraSlots);
1470 } else {
1471 input->numSlots = newNumSlots;
1472 }
1473 }
1474
1475 int32_t flags = 0;
1476 if (worklet->output.flags & C2FrameData::FLAG_END_OF_STREAM) {
1477 flags |= MediaCodec::BUFFER_FLAG_EOS;
1478 ALOGV("[%s] onWorkDone: output EOS", mName);
1479 }
1480
1481 sp<MediaCodecBuffer> outBuffer;
1482 size_t index;
1483
1484 // WORKAROUND: adjust output timestamp based on client input timestamp and codec
1485 // input timestamp. Codec output timestamp (in the timestamp field) shall correspond to
1486 // the codec input timestamp, but client output timestamp should (reported in timeUs)
1487 // shall correspond to the client input timesamp (in customOrdinal). By using the
1488 // delta between the two, this allows for some timestamp deviation - e.g. if one input
1489 // produces multiple output.
1490 c2_cntr64_t timestamp =
1491 worklet->output.ordinal.timestamp + work->input.ordinal.customOrdinal
1492 - work->input.ordinal.timestamp;
1493 if (mInputSurface != nullptr) {
1494 // When using input surface we need to restore the original input timestamp.
1495 timestamp = work->input.ordinal.customOrdinal;
1496 }
1497 ALOGV("[%s] onWorkDone: input %lld, codec %lld => output %lld => %lld",
1498 mName,
1499 work->input.ordinal.customOrdinal.peekll(),
1500 work->input.ordinal.timestamp.peekll(),
1501 worklet->output.ordinal.timestamp.peekll(),
1502 timestamp.peekll());
1503
1504 if (initData != nullptr) {
1505 Mutexed<Output>::Locked output(mOutput);
1506 if (output->buffers->registerCsd(initData, &index, &outBuffer) == OK) {
1507 outBuffer->meta()->setInt64("timeUs", timestamp.peek());
1508 outBuffer->meta()->setInt32("flags", MediaCodec::BUFFER_FLAG_CODECCONFIG);
1509 ALOGV("[%s] onWorkDone: csd index = %zu [%p]", mName, index, outBuffer.get());
1510
1511 output.unlock();
1512 mCallback->onOutputBufferAvailable(index, outBuffer);
1513 } else {
1514 ALOGD("[%s] onWorkDone: unable to register csd", mName);
1515 output.unlock();
1516 mCCodecCallback->onError(UNKNOWN_ERROR, ACTION_CODE_FATAL);
1517 return false;
1518 }
1519 }
1520
1521 if (!buffer && !flags) {
1522 ALOGV("[%s] onWorkDone: Not reporting output buffer (%lld)",
1523 mName, work->input.ordinal.frameIndex.peekull());
1524 return true;
1525 }
1526
1527 if (buffer) {
1528 for (const std::shared_ptr<const C2Info> &info : buffer->info()) {
1529 // TODO: properly translate these to metadata
1530 switch (info->coreIndex().coreIndex()) {
1531 case C2StreamPictureTypeMaskInfo::CORE_INDEX:
1532 if (((C2StreamPictureTypeMaskInfo *)info.get())->value & C2Config::SYNC_FRAME) {
1533 flags |= MediaCodec::BUFFER_FLAG_SYNCFRAME;
1534 }
1535 break;
1536 default:
1537 break;
1538 }
1539 }
1540 }
1541
1542 {
1543 Mutexed<ReorderStash>::Locked reorder(mReorderStash);
1544 reorder->emplace(buffer, timestamp.peek(), flags, worklet->output.ordinal);
1545 if (flags & MediaCodec::BUFFER_FLAG_EOS) {
1546 // Flush reorder stash
1547 reorder->setDepth(0);
1548 }
1549 }
1550 sendOutputBuffers();
1551 return true;
1552 }
1553
sendOutputBuffers()1554 void CCodecBufferChannel::sendOutputBuffers() {
1555 ReorderStash::Entry entry;
1556 sp<MediaCodecBuffer> outBuffer;
1557 size_t index;
1558
1559 while (true) {
1560 Mutexed<ReorderStash>::Locked reorder(mReorderStash);
1561 if (!reorder->hasPending()) {
1562 break;
1563 }
1564 if (!reorder->pop(&entry)) {
1565 break;
1566 }
1567
1568 Mutexed<Output>::Locked output(mOutput);
1569 status_t err = output->buffers->registerBuffer(entry.buffer, &index, &outBuffer);
1570 if (err != OK) {
1571 bool outputBuffersChanged = false;
1572 if (err != WOULD_BLOCK) {
1573 if (!output->buffers->isArrayMode()) {
1574 output->buffers = output->buffers->toArrayMode(output->numSlots);
1575 }
1576 OutputBuffersArray *array = (OutputBuffersArray *)output->buffers.get();
1577 array->realloc(entry.buffer);
1578 outputBuffersChanged = true;
1579 }
1580 ALOGV("[%s] sendOutputBuffers: unable to register output buffer", mName);
1581 reorder->defer(entry);
1582
1583 output.unlock();
1584 reorder.unlock();
1585
1586 if (outputBuffersChanged) {
1587 mCCodecCallback->onOutputBuffersChanged();
1588 }
1589 return;
1590 }
1591 output.unlock();
1592 reorder.unlock();
1593
1594 outBuffer->meta()->setInt64("timeUs", entry.timestamp);
1595 outBuffer->meta()->setInt32("flags", entry.flags);
1596 ALOGV("[%s] sendOutputBuffers: out buffer index = %zu [%p] => %p + %zu (%lld)",
1597 mName, index, outBuffer.get(), outBuffer->data(), outBuffer->size(),
1598 (long long)entry.timestamp);
1599 mCallback->onOutputBufferAvailable(index, outBuffer);
1600 }
1601 }
1602
setSurface(const sp<Surface> & newSurface)1603 status_t CCodecBufferChannel::setSurface(const sp<Surface> &newSurface) {
1604 static std::atomic_uint32_t surfaceGeneration{0};
1605 uint32_t generation = (getpid() << 10) |
1606 ((surfaceGeneration.fetch_add(1, std::memory_order_relaxed) + 1)
1607 & ((1 << 10) - 1));
1608
1609 sp<IGraphicBufferProducer> producer;
1610 if (newSurface) {
1611 newSurface->setScalingMode(NATIVE_WINDOW_SCALING_MODE_SCALE_TO_WINDOW);
1612 newSurface->setDequeueTimeout(kDequeueTimeoutNs);
1613 newSurface->setMaxDequeuedBufferCount(mOutputSurface.lock()->maxDequeueBuffers);
1614 producer = newSurface->getIGraphicBufferProducer();
1615 producer->setGenerationNumber(generation);
1616 } else {
1617 ALOGE("[%s] setting output surface to null", mName);
1618 return INVALID_OPERATION;
1619 }
1620
1621 std::shared_ptr<Codec2Client::Configurable> outputPoolIntf;
1622 C2BlockPool::local_id_t outputPoolId;
1623 {
1624 Mutexed<BlockPools>::Locked pools(mBlockPools);
1625 outputPoolId = pools->outputPoolId;
1626 outputPoolIntf = pools->outputPoolIntf;
1627 }
1628
1629 if (outputPoolIntf) {
1630 if (mComponent->setOutputSurface(
1631 outputPoolId,
1632 producer,
1633 generation) != C2_OK) {
1634 ALOGI("[%s] setSurface: component setOutputSurface failed", mName);
1635 return INVALID_OPERATION;
1636 }
1637 }
1638
1639 {
1640 Mutexed<OutputSurface>::Locked output(mOutputSurface);
1641 output->surface = newSurface;
1642 output->generation = generation;
1643 }
1644
1645 return OK;
1646 }
1647
elapsed()1648 PipelineWatcher::Clock::duration CCodecBufferChannel::elapsed() {
1649 // When client pushed EOS, we want all the work to be done quickly.
1650 // Otherwise, component may have stalled work due to input starvation up to
1651 // the sum of the delay in the pipeline.
1652 size_t n = 0;
1653 if (!mInputMetEos) {
1654 size_t outputDelay = mOutput.lock()->outputDelay;
1655 Mutexed<Input>::Locked input(mInput);
1656 n = input->inputDelay + input->pipelineDelay + outputDelay;
1657 }
1658 return mPipelineWatcher.lock()->elapsed(PipelineWatcher::Clock::now(), n);
1659 }
1660
setMetaMode(MetaMode mode)1661 void CCodecBufferChannel::setMetaMode(MetaMode mode) {
1662 mMetaMode = mode;
1663 }
1664
toStatusT(c2_status_t c2s,c2_operation_t c2op)1665 status_t toStatusT(c2_status_t c2s, c2_operation_t c2op) {
1666 // C2_OK is always translated to OK.
1667 if (c2s == C2_OK) {
1668 return OK;
1669 }
1670
1671 // Operation-dependent translation
1672 // TODO: Add as necessary
1673 switch (c2op) {
1674 case C2_OPERATION_Component_start:
1675 switch (c2s) {
1676 case C2_NO_MEMORY:
1677 return NO_MEMORY;
1678 default:
1679 return UNKNOWN_ERROR;
1680 }
1681 default:
1682 break;
1683 }
1684
1685 // Backup operation-agnostic translation
1686 switch (c2s) {
1687 case C2_BAD_INDEX:
1688 return BAD_INDEX;
1689 case C2_BAD_VALUE:
1690 return BAD_VALUE;
1691 case C2_BLOCKING:
1692 return WOULD_BLOCK;
1693 case C2_DUPLICATE:
1694 return ALREADY_EXISTS;
1695 case C2_NO_INIT:
1696 return NO_INIT;
1697 case C2_NO_MEMORY:
1698 return NO_MEMORY;
1699 case C2_NOT_FOUND:
1700 return NAME_NOT_FOUND;
1701 case C2_TIMED_OUT:
1702 return TIMED_OUT;
1703 case C2_BAD_STATE:
1704 case C2_CANCELED:
1705 case C2_CANNOT_DO:
1706 case C2_CORRUPTED:
1707 case C2_OMITTED:
1708 case C2_REFUSED:
1709 return UNKNOWN_ERROR;
1710 default:
1711 return -static_cast<status_t>(c2s);
1712 }
1713 }
1714
1715 } // namespace android
1716