1 /*
2  * Copyright (C) 2019 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #define LOG_TAG "Camera3-HeicCompositeStream"
18 #define ATRACE_TAG ATRACE_TAG_CAMERA
19 //#define LOG_NDEBUG 0
20 
21 #include <linux/memfd.h>
22 #include <pthread.h>
23 #include <sys/syscall.h>
24 
25 #include <android/hardware/camera/device/3.5/types.h>
26 #include <libyuv.h>
27 #include <gui/Surface.h>
28 #include <utils/Log.h>
29 #include <utils/Trace.h>
30 
31 #include <mediadrm/ICrypto.h>
32 #include <media/MediaCodecBuffer.h>
33 #include <media/stagefright/foundation/ABuffer.h>
34 #include <media/stagefright/foundation/MediaDefs.h>
35 #include <media/stagefright/MediaCodecConstants.h>
36 
37 #include "common/CameraDeviceBase.h"
38 #include "utils/ExifUtils.h"
39 #include "HeicEncoderInfoManager.h"
40 #include "HeicCompositeStream.h"
41 
42 using android::hardware::camera::device::V3_5::CameraBlob;
43 using android::hardware::camera::device::V3_5::CameraBlobId;
44 
45 namespace android {
46 namespace camera3 {
47 
HeicCompositeStream(wp<CameraDeviceBase> device,wp<hardware::camera2::ICameraDeviceCallbacks> cb)48 HeicCompositeStream::HeicCompositeStream(wp<CameraDeviceBase> device,
49         wp<hardware::camera2::ICameraDeviceCallbacks> cb) :
50         CompositeStream(device, cb),
51         mUseHeic(false),
52         mNumOutputTiles(1),
53         mOutputWidth(0),
54         mOutputHeight(0),
55         mMaxHeicBufferSize(0),
56         mGridWidth(HeicEncoderInfoManager::kGridWidth),
57         mGridHeight(HeicEncoderInfoManager::kGridHeight),
58         mGridRows(1),
59         mGridCols(1),
60         mUseGrid(false),
61         mAppSegmentStreamId(-1),
62         mAppSegmentSurfaceId(-1),
63         mMainImageStreamId(-1),
64         mMainImageSurfaceId(-1),
65         mYuvBufferAcquired(false),
66         mProducerListener(new ProducerListener()),
67         mDequeuedOutputBufferCnt(0),
68         mLockedAppSegmentBufferCnt(0),
69         mCodecOutputCounter(0),
70         mGridTimestampUs(0) {
71 }
72 
~HeicCompositeStream()73 HeicCompositeStream::~HeicCompositeStream() {
74     // Call deinitCodec in case stream hasn't been deleted yet to avoid any
75     // memory/resource leak.
76     deinitCodec();
77 
78     mInputAppSegmentBuffers.clear();
79     mCodecOutputBuffers.clear();
80 
81     mAppSegmentStreamId = -1;
82     mAppSegmentSurfaceId = -1;
83     mAppSegmentConsumer.clear();
84     mAppSegmentSurface.clear();
85 
86     mMainImageStreamId = -1;
87     mMainImageSurfaceId = -1;
88     mMainImageConsumer.clear();
89     mMainImageSurface.clear();
90 }
91 
isHeicCompositeStream(const sp<Surface> & surface)92 bool HeicCompositeStream::isHeicCompositeStream(const sp<Surface> &surface) {
93     ANativeWindow *anw = surface.get();
94     status_t err;
95     int format;
96     if ((err = anw->query(anw, NATIVE_WINDOW_FORMAT, &format)) != OK) {
97         String8 msg = String8::format("Failed to query Surface format: %s (%d)", strerror(-err),
98                 err);
99         ALOGE("%s: %s", __FUNCTION__, msg.string());
100         return false;
101     }
102 
103     int dataspace;
104     if ((err = anw->query(anw, NATIVE_WINDOW_DEFAULT_DATASPACE, &dataspace)) != OK) {
105         String8 msg = String8::format("Failed to query Surface dataspace: %s (%d)", strerror(-err),
106                 err);
107         ALOGE("%s: %s", __FUNCTION__, msg.string());
108         return false;
109     }
110 
111     return ((format == HAL_PIXEL_FORMAT_BLOB) && (dataspace == HAL_DATASPACE_HEIF));
112 }
113 
createInternalStreams(const std::vector<sp<Surface>> & consumers,bool,uint32_t width,uint32_t height,int format,camera3_stream_rotation_t rotation,int * id,const String8 & physicalCameraId,std::vector<int> * surfaceIds,int,bool)114 status_t HeicCompositeStream::createInternalStreams(const std::vector<sp<Surface>>& consumers,
115         bool /*hasDeferredConsumer*/, uint32_t width, uint32_t height, int format,
116         camera3_stream_rotation_t rotation, int *id, const String8& physicalCameraId,
117         std::vector<int> *surfaceIds, int /*streamSetId*/, bool /*isShared*/) {
118 
119     sp<CameraDeviceBase> device = mDevice.promote();
120     if (!device.get()) {
121         ALOGE("%s: Invalid camera device!", __FUNCTION__);
122         return NO_INIT;
123     }
124 
125     status_t res = initializeCodec(width, height, device);
126     if (res != OK) {
127         ALOGE("%s: Failed to initialize HEIC/HEVC codec: %s (%d)",
128                 __FUNCTION__, strerror(-res), res);
129         return NO_INIT;
130     }
131 
132     sp<IGraphicBufferProducer> producer;
133     sp<IGraphicBufferConsumer> consumer;
134     BufferQueue::createBufferQueue(&producer, &consumer);
135     mAppSegmentConsumer = new CpuConsumer(consumer, kMaxAcquiredAppSegment);
136     mAppSegmentConsumer->setFrameAvailableListener(this);
137     mAppSegmentConsumer->setName(String8("Camera3-HeicComposite-AppSegmentStream"));
138     mAppSegmentSurface = new Surface(producer);
139 
140     mStaticInfo = device->info();
141 
142     res = device->createStream(mAppSegmentSurface, mAppSegmentMaxSize, 1, format,
143             kAppSegmentDataSpace, rotation, &mAppSegmentStreamId, physicalCameraId, surfaceIds);
144     if (res == OK) {
145         mAppSegmentSurfaceId = (*surfaceIds)[0];
146     } else {
147         ALOGE("%s: Failed to create JPEG App segment stream: %s (%d)", __FUNCTION__,
148                 strerror(-res), res);
149         return res;
150     }
151 
152     if (!mUseGrid) {
153         res = mCodec->createInputSurface(&producer);
154         if (res != OK) {
155             ALOGE("%s: Failed to create input surface for Heic codec: %s (%d)",
156                     __FUNCTION__, strerror(-res), res);
157             return res;
158         }
159     } else {
160         BufferQueue::createBufferQueue(&producer, &consumer);
161         mMainImageConsumer = new CpuConsumer(consumer, 1);
162         mMainImageConsumer->setFrameAvailableListener(this);
163         mMainImageConsumer->setName(String8("Camera3-HeicComposite-HevcInputYUVStream"));
164     }
165     mMainImageSurface = new Surface(producer);
166 
167     res = mCodec->start();
168     if (res != OK) {
169         ALOGE("%s: Failed to start codec: %s (%d)", __FUNCTION__,
170                 strerror(-res), res);
171         return res;
172     }
173 
174     std::vector<int> sourceSurfaceId;
175     //Use YUV_888 format if framework tiling is needed.
176     int srcStreamFmt = mUseGrid ? HAL_PIXEL_FORMAT_YCbCr_420_888 :
177             HAL_PIXEL_FORMAT_IMPLEMENTATION_DEFINED;
178     res = device->createStream(mMainImageSurface, width, height, srcStreamFmt, kHeifDataSpace,
179             rotation, id, physicalCameraId, &sourceSurfaceId);
180     if (res == OK) {
181         mMainImageSurfaceId = sourceSurfaceId[0];
182         mMainImageStreamId = *id;
183     } else {
184         ALOGE("%s: Failed to create main image stream: %s (%d)", __FUNCTION__,
185                 strerror(-res), res);
186         return res;
187     }
188 
189     mOutputSurface = consumers[0];
190     res = registerCompositeStreamListener(getStreamId());
191     if (res != OK) {
192         ALOGE("%s: Failed to register HAL main image stream", __FUNCTION__);
193         return res;
194     }
195 
196     initCopyRowFunction(width);
197     return res;
198 }
199 
deleteInternalStreams()200 status_t HeicCompositeStream::deleteInternalStreams() {
201     requestExit();
202     auto res = join();
203     if (res != OK) {
204         ALOGE("%s: Failed to join with the main processing thread: %s (%d)", __FUNCTION__,
205                 strerror(-res), res);
206     }
207 
208     deinitCodec();
209 
210     if (mAppSegmentStreamId >= 0) {
211         sp<CameraDeviceBase> device = mDevice.promote();
212         if (!device.get()) {
213             ALOGE("%s: Invalid camera device!", __FUNCTION__);
214             return NO_INIT;
215         }
216 
217         res = device->deleteStream(mAppSegmentStreamId);
218         mAppSegmentStreamId = -1;
219     }
220 
221     if (mOutputSurface != nullptr) {
222         mOutputSurface->disconnect(NATIVE_WINDOW_API_CAMERA);
223         mOutputSurface.clear();
224     }
225     return res;
226 }
227 
onBufferReleased(const BufferInfo & bufferInfo)228 void HeicCompositeStream::onBufferReleased(const BufferInfo& bufferInfo) {
229     Mutex::Autolock l(mMutex);
230 
231     if (bufferInfo.mError) return;
232 
233     mCodecOutputBufferTimestamps.push(bufferInfo.mTimestamp);
234     ALOGV("%s: [%" PRId64 "]: Adding codecOutputBufferTimestamp (%zu timestamps in total)",
235             __FUNCTION__, bufferInfo.mTimestamp, mCodecOutputBufferTimestamps.size());
236 }
237 
238 // We need to get the settings early to handle the case where the codec output
239 // arrives earlier than result metadata.
onBufferRequestForFrameNumber(uint64_t frameNumber,int streamId,const CameraMetadata & settings)240 void HeicCompositeStream::onBufferRequestForFrameNumber(uint64_t frameNumber, int streamId,
241         const CameraMetadata& settings) {
242     ATRACE_ASYNC_BEGIN("HEIC capture", frameNumber);
243 
244     Mutex::Autolock l(mMutex);
245     if (mErrorState || (streamId != getStreamId())) {
246         return;
247     }
248 
249     mPendingCaptureResults.emplace(frameNumber, CameraMetadata());
250 
251     camera_metadata_ro_entry entry;
252 
253     int32_t orientation = 0;
254     entry = settings.find(ANDROID_JPEG_ORIENTATION);
255     if (entry.count == 1) {
256         orientation = entry.data.i32[0];
257     }
258 
259     int32_t quality = kDefaultJpegQuality;
260     entry = settings.find(ANDROID_JPEG_QUALITY);
261     if (entry.count == 1) {
262         quality = entry.data.i32[0];
263     }
264 
265     mSettingsByFrameNumber[frameNumber] = std::make_pair(orientation, quality);
266 }
267 
onFrameAvailable(const BufferItem & item)268 void HeicCompositeStream::onFrameAvailable(const BufferItem& item) {
269     if (item.mDataSpace == static_cast<android_dataspace>(kAppSegmentDataSpace)) {
270         ALOGV("%s: JPEG APP segments buffer with ts: %" PRIu64 " ms. arrived!",
271                 __func__, ns2ms(item.mTimestamp));
272 
273         Mutex::Autolock l(mMutex);
274         if (!mErrorState) {
275             mInputAppSegmentBuffers.push_back(item.mTimestamp);
276             mInputReadyCondition.signal();
277         }
278     } else if (item.mDataSpace == kHeifDataSpace) {
279         ALOGV("%s: YUV_888 buffer with ts: %" PRIu64 " ms. arrived!",
280                 __func__, ns2ms(item.mTimestamp));
281 
282         Mutex::Autolock l(mMutex);
283         if (!mUseGrid) {
284             ALOGE("%s: YUV_888 internal stream is only supported for HEVC tiling",
285                     __FUNCTION__);
286             return;
287         }
288         if (!mErrorState) {
289             mInputYuvBuffers.push_back(item.mTimestamp);
290             mInputReadyCondition.signal();
291         }
292     } else {
293         ALOGE("%s: Unexpected data space: 0x%x", __FUNCTION__, item.mDataSpace);
294     }
295 }
296 
getCompositeStreamInfo(const OutputStreamInfo & streamInfo,const CameraMetadata & ch,std::vector<OutputStreamInfo> * compositeOutput)297 status_t HeicCompositeStream::getCompositeStreamInfo(const OutputStreamInfo &streamInfo,
298             const CameraMetadata& ch, std::vector<OutputStreamInfo>* compositeOutput /*out*/) {
299     if (compositeOutput == nullptr) {
300         return BAD_VALUE;
301     }
302 
303     compositeOutput->clear();
304 
305     bool useGrid, useHeic;
306     bool isSizeSupported = isSizeSupportedByHeifEncoder(
307             streamInfo.width, streamInfo.height, &useHeic, &useGrid, nullptr);
308     if (!isSizeSupported) {
309         // Size is not supported by either encoder.
310         return OK;
311     }
312 
313     compositeOutput->insert(compositeOutput->end(), 2, streamInfo);
314 
315     // JPEG APPS segments Blob stream info
316     (*compositeOutput)[0].width = calcAppSegmentMaxSize(ch);
317     (*compositeOutput)[0].height = 1;
318     (*compositeOutput)[0].format = HAL_PIXEL_FORMAT_BLOB;
319     (*compositeOutput)[0].dataSpace = kAppSegmentDataSpace;
320     (*compositeOutput)[0].consumerUsage = GRALLOC_USAGE_SW_READ_OFTEN;
321 
322     // YUV/IMPLEMENTATION_DEFINED stream info
323     (*compositeOutput)[1].width = streamInfo.width;
324     (*compositeOutput)[1].height = streamInfo.height;
325     (*compositeOutput)[1].format = useGrid ? HAL_PIXEL_FORMAT_YCbCr_420_888 :
326             HAL_PIXEL_FORMAT_IMPLEMENTATION_DEFINED;
327     (*compositeOutput)[1].dataSpace = kHeifDataSpace;
328     (*compositeOutput)[1].consumerUsage = useHeic ? GRALLOC_USAGE_HW_IMAGE_ENCODER :
329             useGrid ? GRALLOC_USAGE_SW_READ_OFTEN : GRALLOC_USAGE_HW_VIDEO_ENCODER;
330 
331     return NO_ERROR;
332 }
333 
isSizeSupportedByHeifEncoder(int32_t width,int32_t height,bool * useHeic,bool * useGrid,int64_t * stall,AString * hevcName)334 bool HeicCompositeStream::isSizeSupportedByHeifEncoder(int32_t width, int32_t height,
335         bool* useHeic, bool* useGrid, int64_t* stall, AString* hevcName) {
336     static HeicEncoderInfoManager& heicManager = HeicEncoderInfoManager::getInstance();
337     return heicManager.isSizeSupported(width, height, useHeic, useGrid, stall, hevcName);
338 }
339 
isInMemoryTempFileSupported()340 bool HeicCompositeStream::isInMemoryTempFileSupported() {
341     int memfd = syscall(__NR_memfd_create, "HEIF-try-memfd", MFD_CLOEXEC);
342     if (memfd == -1) {
343         if (errno != ENOSYS) {
344             ALOGE("%s: Failed to create tmpfs file. errno %d", __FUNCTION__, errno);
345         }
346         return false;
347     }
348     close(memfd);
349     return true;
350 }
351 
onHeicOutputFrameAvailable(const CodecOutputBufferInfo & outputBufferInfo)352 void HeicCompositeStream::onHeicOutputFrameAvailable(
353         const CodecOutputBufferInfo& outputBufferInfo) {
354     Mutex::Autolock l(mMutex);
355 
356     ALOGV("%s: index %d, offset %d, size %d, time %" PRId64 ", flags 0x%x",
357             __FUNCTION__, outputBufferInfo.index, outputBufferInfo.offset,
358             outputBufferInfo.size, outputBufferInfo.timeUs, outputBufferInfo.flags);
359 
360     if (!mErrorState) {
361         if ((outputBufferInfo.size > 0) &&
362                 ((outputBufferInfo.flags & MediaCodec::BUFFER_FLAG_CODECCONFIG) == 0)) {
363             mCodecOutputBuffers.push_back(outputBufferInfo);
364             mInputReadyCondition.signal();
365         } else {
366             ALOGV("%s: Releasing output buffer: size %d flags: 0x%x ", __FUNCTION__,
367                 outputBufferInfo.size, outputBufferInfo.flags);
368             mCodec->releaseOutputBuffer(outputBufferInfo.index);
369         }
370     } else {
371         mCodec->releaseOutputBuffer(outputBufferInfo.index);
372     }
373 }
374 
onHeicInputFrameAvailable(int32_t index)375 void HeicCompositeStream::onHeicInputFrameAvailable(int32_t index) {
376     Mutex::Autolock l(mMutex);
377 
378     if (!mUseGrid) {
379         ALOGE("%s: Codec YUV input mode must only be used for Hevc tiling mode", __FUNCTION__);
380         return;
381     }
382 
383     mCodecInputBuffers.push_back(index);
384     mInputReadyCondition.signal();
385 }
386 
onHeicFormatChanged(sp<AMessage> & newFormat)387 void HeicCompositeStream::onHeicFormatChanged(sp<AMessage>& newFormat) {
388     if (newFormat == nullptr) {
389         ALOGE("%s: newFormat must not be null!", __FUNCTION__);
390         return;
391     }
392 
393     Mutex::Autolock l(mMutex);
394 
395     AString mime;
396     AString mimeHeic(MIMETYPE_IMAGE_ANDROID_HEIC);
397     newFormat->findString(KEY_MIME, &mime);
398     if (mime != mimeHeic) {
399         // For HEVC codec, below keys need to be filled out or overwritten so that the
400         // muxer can handle them as HEIC output image.
401         newFormat->setString(KEY_MIME, mimeHeic);
402         newFormat->setInt32(KEY_WIDTH, mOutputWidth);
403         newFormat->setInt32(KEY_HEIGHT, mOutputHeight);
404         if (mUseGrid) {
405             newFormat->setInt32(KEY_TILE_WIDTH, mGridWidth);
406             newFormat->setInt32(KEY_TILE_HEIGHT, mGridHeight);
407             newFormat->setInt32(KEY_GRID_ROWS, mGridRows);
408             newFormat->setInt32(KEY_GRID_COLUMNS, mGridCols);
409         }
410     }
411     newFormat->setInt32(KEY_IS_DEFAULT, 1 /*isPrimary*/);
412 
413     int32_t gridRows, gridCols;
414     if (newFormat->findInt32(KEY_GRID_ROWS, &gridRows) &&
415             newFormat->findInt32(KEY_GRID_COLUMNS, &gridCols)) {
416         mNumOutputTiles = gridRows * gridCols;
417     } else {
418         mNumOutputTiles = 1;
419     }
420 
421     mFormat = newFormat;
422 
423     ALOGV("%s: mNumOutputTiles is %zu", __FUNCTION__, mNumOutputTiles);
424     mInputReadyCondition.signal();
425 }
426 
onHeicCodecError()427 void HeicCompositeStream::onHeicCodecError() {
428     Mutex::Autolock l(mMutex);
429     mErrorState = true;
430 }
431 
configureStream()432 status_t HeicCompositeStream::configureStream() {
433     if (isRunning()) {
434         // Processing thread is already running, nothing more to do.
435         return NO_ERROR;
436     }
437 
438     if (mOutputSurface.get() == nullptr) {
439         ALOGE("%s: No valid output surface set!", __FUNCTION__);
440         return NO_INIT;
441     }
442 
443     auto res = mOutputSurface->connect(NATIVE_WINDOW_API_CAMERA, mProducerListener);
444     if (res != OK) {
445         ALOGE("%s: Unable to connect to native window for stream %d",
446                 __FUNCTION__, mMainImageStreamId);
447         return res;
448     }
449 
450     if ((res = native_window_set_buffers_format(mOutputSurface.get(), HAL_PIXEL_FORMAT_BLOB))
451             != OK) {
452         ALOGE("%s: Unable to configure stream buffer format for stream %d", __FUNCTION__,
453                 mMainImageStreamId);
454         return res;
455     }
456 
457     ANativeWindow *anwConsumer = mOutputSurface.get();
458     int maxConsumerBuffers;
459     if ((res = anwConsumer->query(anwConsumer, NATIVE_WINDOW_MIN_UNDEQUEUED_BUFFERS,
460                     &maxConsumerBuffers)) != OK) {
461         ALOGE("%s: Unable to query consumer undequeued"
462                 " buffer count for stream %d", __FUNCTION__, mMainImageStreamId);
463         return res;
464     }
465 
466     // Cannot use SourceSurface buffer count since it could be codec's 512*512 tile
467     // buffer count.
468     if ((res = native_window_set_buffer_count(
469                     anwConsumer, kMaxOutputSurfaceProducerCount + maxConsumerBuffers)) != OK) {
470         ALOGE("%s: Unable to set buffer count for stream %d", __FUNCTION__, mMainImageStreamId);
471         return res;
472     }
473 
474     if ((res = native_window_set_buffers_dimensions(anwConsumer, mMaxHeicBufferSize, 1)) != OK) {
475         ALOGE("%s: Unable to set buffer dimension %zu x 1 for stream %d: %s (%d)",
476                 __FUNCTION__, mMaxHeicBufferSize, mMainImageStreamId, strerror(-res), res);
477         return res;
478     }
479 
480     run("HeicCompositeStreamProc");
481 
482     return NO_ERROR;
483 }
484 
insertGbp(SurfaceMap * outSurfaceMap,Vector<int32_t> * outputStreamIds,int32_t * currentStreamId)485 status_t HeicCompositeStream::insertGbp(SurfaceMap* /*out*/outSurfaceMap,
486         Vector<int32_t>* /*out*/outputStreamIds, int32_t* /*out*/currentStreamId) {
487     if (outSurfaceMap->find(mAppSegmentStreamId) == outSurfaceMap->end()) {
488         (*outSurfaceMap)[mAppSegmentStreamId] = std::vector<size_t>();
489         outputStreamIds->push_back(mAppSegmentStreamId);
490     }
491     (*outSurfaceMap)[mAppSegmentStreamId].push_back(mAppSegmentSurfaceId);
492 
493     if (outSurfaceMap->find(mMainImageStreamId) == outSurfaceMap->end()) {
494         (*outSurfaceMap)[mMainImageStreamId] = std::vector<size_t>();
495         outputStreamIds->push_back(mMainImageStreamId);
496     }
497     (*outSurfaceMap)[mMainImageStreamId].push_back(mMainImageSurfaceId);
498 
499     if (currentStreamId != nullptr) {
500         *currentStreamId = mMainImageStreamId;
501     }
502 
503     return NO_ERROR;
504 }
505 
onShutter(const CaptureResultExtras & resultExtras,nsecs_t timestamp)506 void HeicCompositeStream::onShutter(const CaptureResultExtras& resultExtras, nsecs_t timestamp) {
507     Mutex::Autolock l(mMutex);
508     if (mErrorState) {
509         return;
510     }
511 
512     if (mSettingsByFrameNumber.find(resultExtras.frameNumber) != mSettingsByFrameNumber.end()) {
513         ALOGV("%s: [%" PRId64 "]: frameNumber %" PRId64, __FUNCTION__,
514                 timestamp, resultExtras.frameNumber);
515         mFrameNumberMap.emplace(resultExtras.frameNumber, timestamp);
516         mSettingsByTimestamp[timestamp] = mSettingsByFrameNumber[resultExtras.frameNumber];
517         mSettingsByFrameNumber.erase(resultExtras.frameNumber);
518         mInputReadyCondition.signal();
519     }
520 }
521 
compilePendingInputLocked()522 void HeicCompositeStream::compilePendingInputLocked() {
523     while (!mSettingsByTimestamp.empty()) {
524         auto it = mSettingsByTimestamp.begin();
525         mPendingInputFrames[it->first].orientation = it->second.first;
526         mPendingInputFrames[it->first].quality = it->second.second;
527         mSettingsByTimestamp.erase(it);
528     }
529 
530     while (!mInputAppSegmentBuffers.empty()) {
531         CpuConsumer::LockedBuffer imgBuffer;
532         auto it = mInputAppSegmentBuffers.begin();
533         auto res = mAppSegmentConsumer->lockNextBuffer(&imgBuffer);
534         if (res == NOT_ENOUGH_DATA) {
535             // Can not lock any more buffers.
536             break;
537         } else if ((res != OK) || (*it != imgBuffer.timestamp)) {
538             if (res != OK) {
539                 ALOGE("%s: Error locking JPEG_APP_SEGMENTS image buffer: %s (%d)", __FUNCTION__,
540                         strerror(-res), res);
541             } else {
542                 ALOGE("%s: Expecting JPEG_APP_SEGMENTS buffer with time stamp: %" PRId64
543                         " received buffer with time stamp: %" PRId64, __FUNCTION__,
544                         *it, imgBuffer.timestamp);
545                 mAppSegmentConsumer->unlockBuffer(imgBuffer);
546             }
547             mPendingInputFrames[*it].error = true;
548             mInputAppSegmentBuffers.erase(it);
549             continue;
550         }
551 
552         if ((mPendingInputFrames.find(imgBuffer.timestamp) != mPendingInputFrames.end()) &&
553                 (mPendingInputFrames[imgBuffer.timestamp].error)) {
554             mAppSegmentConsumer->unlockBuffer(imgBuffer);
555         } else {
556             mPendingInputFrames[imgBuffer.timestamp].appSegmentBuffer = imgBuffer;
557             mLockedAppSegmentBufferCnt++;
558         }
559         mInputAppSegmentBuffers.erase(it);
560     }
561 
562     while (!mInputYuvBuffers.empty() && !mYuvBufferAcquired) {
563         CpuConsumer::LockedBuffer imgBuffer;
564         auto it = mInputYuvBuffers.begin();
565         auto res = mMainImageConsumer->lockNextBuffer(&imgBuffer);
566         if (res == NOT_ENOUGH_DATA) {
567             // Can not lock any more buffers.
568             break;
569         } else if (res != OK) {
570             ALOGE("%s: Error locking YUV_888 image buffer: %s (%d)", __FUNCTION__,
571                     strerror(-res), res);
572             mPendingInputFrames[*it].error = true;
573             mInputYuvBuffers.erase(it);
574             continue;
575         } else if (*it != imgBuffer.timestamp) {
576             ALOGW("%s: Expecting YUV_888 buffer with time stamp: %" PRId64 " received buffer with "
577                     "time stamp: %" PRId64, __FUNCTION__, *it, imgBuffer.timestamp);
578             mPendingInputFrames[*it].error = true;
579             mInputYuvBuffers.erase(it);
580             continue;
581         }
582 
583         if ((mPendingInputFrames.find(imgBuffer.timestamp) != mPendingInputFrames.end()) &&
584                 (mPendingInputFrames[imgBuffer.timestamp].error)) {
585             mMainImageConsumer->unlockBuffer(imgBuffer);
586         } else {
587             mPendingInputFrames[imgBuffer.timestamp].yuvBuffer = imgBuffer;
588             mYuvBufferAcquired = true;
589         }
590         mInputYuvBuffers.erase(it);
591     }
592 
593     while (!mCodecOutputBuffers.empty()) {
594         auto it = mCodecOutputBuffers.begin();
595         // Bitstream buffer timestamp doesn't necessarily directly correlate with input
596         // buffer timestamp. Assume encoder input to output is FIFO, use a queue
597         // to look up timestamp.
598         int64_t bufferTime = -1;
599         if (mCodecOutputBufferTimestamps.empty()) {
600             ALOGV("%s: Failed to find buffer timestamp for codec output buffer!", __FUNCTION__);
601             break;
602         } else {
603             // Direct mapping between camera timestamp (in ns) and codec timestamp (in us).
604             bufferTime = mCodecOutputBufferTimestamps.front();
605             mCodecOutputCounter++;
606             if (mCodecOutputCounter == mNumOutputTiles) {
607                 mCodecOutputBufferTimestamps.pop();
608                 mCodecOutputCounter = 0;
609             }
610 
611             mPendingInputFrames[bufferTime].codecOutputBuffers.push_back(*it);
612             ALOGV("%s: [%" PRId64 "]: Pushing codecOutputBuffers (time %" PRId64 " us)",
613                     __FUNCTION__, bufferTime, it->timeUs);
614         }
615         mCodecOutputBuffers.erase(it);
616     }
617 
618     while (!mFrameNumberMap.empty()) {
619         auto it = mFrameNumberMap.begin();
620         mPendingInputFrames[it->second].frameNumber = it->first;
621         ALOGV("%s: [%" PRId64 "]: frameNumber is %" PRId64, __FUNCTION__, it->second, it->first);
622         mFrameNumberMap.erase(it);
623     }
624 
625     while (!mCaptureResults.empty()) {
626         auto it = mCaptureResults.begin();
627         // Negative timestamp indicates that something went wrong during the capture result
628         // collection process.
629         if (it->first >= 0) {
630             if (mPendingInputFrames[it->first].frameNumber == std::get<0>(it->second)) {
631                 mPendingInputFrames[it->first].result =
632                         std::make_unique<CameraMetadata>(std::get<1>(it->second));
633             } else {
634                 ALOGE("%s: Capture result frameNumber/timestamp mapping changed between "
635                         "shutter and capture result!", __FUNCTION__);
636             }
637         }
638         mCaptureResults.erase(it);
639     }
640 
641     // mErrorFrameNumbers stores frame number of dropped buffers.
642     auto it = mErrorFrameNumbers.begin();
643     while (it != mErrorFrameNumbers.end()) {
644         bool frameFound = false;
645         for (auto &inputFrame : mPendingInputFrames) {
646             if (inputFrame.second.frameNumber == *it) {
647                 inputFrame.second.error = true;
648                 frameFound = true;
649                 break;
650             }
651         }
652 
653         if (frameFound) {
654             it = mErrorFrameNumbers.erase(it);
655         } else {
656             ALOGW("%s: Not able to find failing input with frame number: %" PRId64, __FUNCTION__,
657                     *it);
658             it++;
659         }
660     }
661 
662     // Distribute codec input buffers to be filled out from YUV output
663     for (auto it = mPendingInputFrames.begin();
664             it != mPendingInputFrames.end() && mCodecInputBuffers.size() > 0; it++) {
665         InputFrame& inputFrame(it->second);
666         if (inputFrame.codecInputCounter < mGridRows * mGridCols) {
667             // Available input tiles that are required for the current input
668             // image.
669             size_t newInputTiles = std::min(mCodecInputBuffers.size(),
670                     mGridRows * mGridCols - inputFrame.codecInputCounter);
671             for (size_t i = 0; i < newInputTiles; i++) {
672                 CodecInputBufferInfo inputInfo =
673                         { mCodecInputBuffers[0], mGridTimestampUs++, inputFrame.codecInputCounter };
674                 inputFrame.codecInputBuffers.push_back(inputInfo);
675 
676                 mCodecInputBuffers.erase(mCodecInputBuffers.begin());
677                 inputFrame.codecInputCounter++;
678             }
679             break;
680         }
681     }
682 }
683 
getNextReadyInputLocked(int64_t * currentTs)684 bool HeicCompositeStream::getNextReadyInputLocked(int64_t *currentTs /*out*/) {
685     if (currentTs == nullptr) {
686         return false;
687     }
688 
689     bool newInputAvailable = false;
690     for (auto& it : mPendingInputFrames) {
691         // New input is considered to be available only if:
692         // 1. input buffers are ready, or
693         // 2. App segment and muxer is created, or
694         // 3. A codec output tile is ready, and an output buffer is available.
695         // This makes sure that muxer gets created only when an output tile is
696         // generated, because right now we only handle 1 HEIC output buffer at a
697         // time (max dequeued buffer count is 1).
698         bool appSegmentReady = (it.second.appSegmentBuffer.data != nullptr) &&
699                 !it.second.appSegmentWritten && it.second.result != nullptr &&
700                 it.second.muxer != nullptr;
701         bool codecOutputReady = !it.second.codecOutputBuffers.empty();
702         bool codecInputReady = (it.second.yuvBuffer.data != nullptr) &&
703                 (!it.second.codecInputBuffers.empty());
704         bool hasOutputBuffer = it.second.muxer != nullptr ||
705                 (mDequeuedOutputBufferCnt < kMaxOutputSurfaceProducerCount);
706         if ((!it.second.error) &&
707                 (it.first < *currentTs) &&
708                 (appSegmentReady || (codecOutputReady && hasOutputBuffer) || codecInputReady)) {
709             *currentTs = it.first;
710             if (it.second.format == nullptr && mFormat != nullptr) {
711                 it.second.format = mFormat->dup();
712             }
713             newInputAvailable = true;
714             break;
715         }
716     }
717 
718     return newInputAvailable;
719 }
720 
getNextFailingInputLocked(int64_t * currentTs)721 int64_t HeicCompositeStream::getNextFailingInputLocked(int64_t *currentTs /*out*/) {
722     int64_t res = -1;
723     if (currentTs == nullptr) {
724         return res;
725     }
726 
727     for (const auto& it : mPendingInputFrames) {
728         if (it.second.error && !it.second.errorNotified && (it.first < *currentTs)) {
729             *currentTs = it.first;
730             res = it.second.frameNumber;
731             break;
732         }
733     }
734 
735     return res;
736 }
737 
processInputFrame(nsecs_t timestamp,InputFrame & inputFrame)738 status_t HeicCompositeStream::processInputFrame(nsecs_t timestamp,
739         InputFrame &inputFrame) {
740     ATRACE_CALL();
741     status_t res = OK;
742 
743     bool appSegmentReady = inputFrame.appSegmentBuffer.data != nullptr &&
744             !inputFrame.appSegmentWritten && inputFrame.result != nullptr &&
745             inputFrame.muxer != nullptr;
746     bool codecOutputReady = inputFrame.codecOutputBuffers.size() > 0;
747     bool codecInputReady = inputFrame.yuvBuffer.data != nullptr &&
748             !inputFrame.codecInputBuffers.empty();
749     bool hasOutputBuffer = inputFrame.muxer != nullptr ||
750             (mDequeuedOutputBufferCnt < kMaxOutputSurfaceProducerCount);
751 
752     ALOGV("%s: [%" PRId64 "]: appSegmentReady %d, codecOutputReady %d, codecInputReady %d,"
753             " dequeuedOutputBuffer %d", __FUNCTION__, timestamp, appSegmentReady,
754             codecOutputReady, codecInputReady, mDequeuedOutputBufferCnt);
755 
756     // Handle inputs for Hevc tiling
757     if (codecInputReady) {
758         res = processCodecInputFrame(inputFrame);
759         if (res != OK) {
760             ALOGE("%s: Failed to process codec input frame: %s (%d)", __FUNCTION__,
761                     strerror(-res), res);
762             return res;
763         }
764     }
765 
766     if (!(codecOutputReady && hasOutputBuffer) && !appSegmentReady) {
767         return OK;
768     }
769 
770     // Initialize and start muxer if not yet done so. In this case,
771     // codecOutputReady must be true. Otherwise, appSegmentReady is guaranteed
772     // to be false, and the function must have returned early.
773     if (inputFrame.muxer == nullptr) {
774         res = startMuxerForInputFrame(timestamp, inputFrame);
775         if (res != OK) {
776             ALOGE("%s: Failed to create and start muxer: %s (%d)", __FUNCTION__,
777                     strerror(-res), res);
778             return res;
779         }
780     }
781 
782     // Write JPEG APP segments data to the muxer.
783     if (appSegmentReady) {
784         res = processAppSegment(timestamp, inputFrame);
785         if (res != OK) {
786             ALOGE("%s: Failed to process JPEG APP segments: %s (%d)", __FUNCTION__,
787                     strerror(-res), res);
788             return res;
789         }
790     }
791 
792     // Write media codec bitstream buffers to muxer.
793     while (!inputFrame.codecOutputBuffers.empty()) {
794         res = processOneCodecOutputFrame(timestamp, inputFrame);
795         if (res != OK) {
796             ALOGE("%s: Failed to process codec output frame: %s (%d)", __FUNCTION__,
797                     strerror(-res), res);
798             return res;
799         }
800     }
801 
802     if (inputFrame.pendingOutputTiles == 0) {
803         if (inputFrame.appSegmentWritten) {
804             res = processCompletedInputFrame(timestamp, inputFrame);
805             if (res != OK) {
806                 ALOGE("%s: Failed to process completed input frame: %s (%d)", __FUNCTION__,
807                         strerror(-res), res);
808                 return res;
809             }
810         } else if (mLockedAppSegmentBufferCnt == kMaxAcquiredAppSegment) {
811             ALOGE("%s: Out-of-order app segment buffers reaches limit %u", __FUNCTION__,
812                     kMaxAcquiredAppSegment);
813             return INVALID_OPERATION;
814         }
815     }
816 
817     return res;
818 }
819 
startMuxerForInputFrame(nsecs_t timestamp,InputFrame & inputFrame)820 status_t HeicCompositeStream::startMuxerForInputFrame(nsecs_t timestamp, InputFrame &inputFrame) {
821     sp<ANativeWindow> outputANW = mOutputSurface;
822 
823     auto res = outputANW->dequeueBuffer(mOutputSurface.get(), &inputFrame.anb, &inputFrame.fenceFd);
824     if (res != OK) {
825         ALOGE("%s: Error retrieving output buffer: %s (%d)", __FUNCTION__, strerror(-res),
826                 res);
827         return res;
828     }
829     mDequeuedOutputBufferCnt++;
830 
831     // Combine current thread id, stream id and timestamp to uniquely identify image.
832     std::ostringstream tempOutputFile;
833     tempOutputFile << "HEIF-" << pthread_self() << "-"
834             << getStreamId() << "-" << timestamp;
835     inputFrame.fileFd = syscall(__NR_memfd_create, tempOutputFile.str().c_str(), MFD_CLOEXEC);
836     if (inputFrame.fileFd < 0) {
837         ALOGE("%s: Failed to create file %s. Error no is %d", __FUNCTION__,
838                 tempOutputFile.str().c_str(), errno);
839         return NO_INIT;
840     }
841     inputFrame.muxer = new MediaMuxer(inputFrame.fileFd, MediaMuxer::OUTPUT_FORMAT_HEIF);
842     if (inputFrame.muxer == nullptr) {
843         ALOGE("%s: Failed to create MediaMuxer for file fd %d",
844                 __FUNCTION__, inputFrame.fileFd);
845         return NO_INIT;
846     }
847 
848     res = inputFrame.muxer->setOrientationHint(inputFrame.orientation);
849     if (res != OK) {
850         ALOGE("%s: Failed to setOrientationHint: %s (%d)", __FUNCTION__,
851                 strerror(-res), res);
852         return res;
853     }
854     // Set encoder quality
855     {
856         sp<AMessage> qualityParams = new AMessage;
857         qualityParams->setInt32(PARAMETER_KEY_VIDEO_BITRATE, inputFrame.quality);
858         res = mCodec->setParameters(qualityParams);
859         if (res != OK) {
860             ALOGE("%s: Failed to set codec quality: %s (%d)",
861                     __FUNCTION__, strerror(-res), res);
862             return res;
863         }
864     }
865 
866     ssize_t trackId = inputFrame.muxer->addTrack(inputFrame.format);
867     if (trackId < 0) {
868         ALOGE("%s: Failed to addTrack to the muxer: %zd", __FUNCTION__, trackId);
869         return NO_INIT;
870     }
871 
872     inputFrame.trackIndex = trackId;
873     inputFrame.pendingOutputTiles = mNumOutputTiles;
874 
875     res = inputFrame.muxer->start();
876     if (res != OK) {
877         ALOGE("%s: Failed to start MediaMuxer: %s (%d)",
878                 __FUNCTION__, strerror(-res), res);
879         return res;
880     }
881 
882     ALOGV("%s: [%" PRId64 "]: Muxer started for inputFrame", __FUNCTION__,
883             timestamp);
884     return OK;
885 }
886 
processAppSegment(nsecs_t timestamp,InputFrame & inputFrame)887 status_t HeicCompositeStream::processAppSegment(nsecs_t timestamp, InputFrame &inputFrame) {
888     size_t app1Size = 0;
889     auto appSegmentSize = findAppSegmentsSize(inputFrame.appSegmentBuffer.data,
890             inputFrame.appSegmentBuffer.width * inputFrame.appSegmentBuffer.height,
891             &app1Size);
892     if (appSegmentSize == 0) {
893         ALOGE("%s: Failed to find JPEG APP segment size", __FUNCTION__);
894         return NO_INIT;
895     }
896 
897     std::unique_ptr<ExifUtils> exifUtils(ExifUtils::create());
898     auto exifRes = exifUtils->initialize(inputFrame.appSegmentBuffer.data, app1Size);
899     if (!exifRes) {
900         ALOGE("%s: Failed to initialize ExifUtils object!", __FUNCTION__);
901         return BAD_VALUE;
902     }
903     exifRes = exifUtils->setFromMetadata(*inputFrame.result, mStaticInfo,
904             mOutputWidth, mOutputHeight);
905     if (!exifRes) {
906         ALOGE("%s: Failed to set Exif tags using metadata and main image sizes", __FUNCTION__);
907         return BAD_VALUE;
908     }
909     exifRes = exifUtils->setOrientation(inputFrame.orientation);
910     if (!exifRes) {
911         ALOGE("%s: ExifUtils failed to set orientation", __FUNCTION__);
912         return BAD_VALUE;
913     }
914     exifRes = exifUtils->generateApp1();
915     if (!exifRes) {
916         ALOGE("%s: ExifUtils failed to generate APP1 segment", __FUNCTION__);
917         return BAD_VALUE;
918     }
919 
920     unsigned int newApp1Length = exifUtils->getApp1Length();
921     const uint8_t *newApp1Segment = exifUtils->getApp1Buffer();
922 
923     //Assemble the APP1 marker buffer required by MediaCodec
924     uint8_t kExifApp1Marker[] = {'E', 'x', 'i', 'f', 0xFF, 0xE1, 0x00, 0x00};
925     kExifApp1Marker[6] = static_cast<uint8_t>(newApp1Length >> 8);
926     kExifApp1Marker[7] = static_cast<uint8_t>(newApp1Length & 0xFF);
927     size_t appSegmentBufferSize = sizeof(kExifApp1Marker) +
928             appSegmentSize - app1Size + newApp1Length;
929     uint8_t* appSegmentBuffer = new uint8_t[appSegmentBufferSize];
930     memcpy(appSegmentBuffer, kExifApp1Marker, sizeof(kExifApp1Marker));
931     memcpy(appSegmentBuffer + sizeof(kExifApp1Marker), newApp1Segment, newApp1Length);
932     if (appSegmentSize - app1Size > 0) {
933         memcpy(appSegmentBuffer + sizeof(kExifApp1Marker) + newApp1Length,
934                 inputFrame.appSegmentBuffer.data + app1Size, appSegmentSize - app1Size);
935     }
936 
937     sp<ABuffer> aBuffer = new ABuffer(appSegmentBuffer, appSegmentBufferSize);
938     auto res = inputFrame.muxer->writeSampleData(aBuffer, inputFrame.trackIndex,
939             timestamp, MediaCodec::BUFFER_FLAG_MUXER_DATA);
940     delete[] appSegmentBuffer;
941 
942     if (res != OK) {
943         ALOGE("%s: Failed to write JPEG APP segments to muxer: %s (%d)",
944                 __FUNCTION__, strerror(-res), res);
945         return res;
946     }
947 
948     ALOGV("%s: [%" PRId64 "]: appSegmentSize is %zu, width %d, height %d, app1Size %zu",
949           __FUNCTION__, timestamp, appSegmentSize, inputFrame.appSegmentBuffer.width,
950           inputFrame.appSegmentBuffer.height, app1Size);
951 
952     inputFrame.appSegmentWritten = true;
953     // Release the buffer now so any pending input app segments can be processed
954     mAppSegmentConsumer->unlockBuffer(inputFrame.appSegmentBuffer);
955     inputFrame.appSegmentBuffer.data = nullptr;
956     mLockedAppSegmentBufferCnt--;
957 
958     return OK;
959 }
960 
processCodecInputFrame(InputFrame & inputFrame)961 status_t HeicCompositeStream::processCodecInputFrame(InputFrame &inputFrame) {
962     for (auto& inputBuffer : inputFrame.codecInputBuffers) {
963         sp<MediaCodecBuffer> buffer;
964         auto res = mCodec->getInputBuffer(inputBuffer.index, &buffer);
965         if (res != OK) {
966             ALOGE("%s: Error getting codec input buffer: %s (%d)", __FUNCTION__,
967                     strerror(-res), res);
968             return res;
969         }
970 
971         // Copy one tile from source to destination.
972         size_t tileX = inputBuffer.tileIndex % mGridCols;
973         size_t tileY = inputBuffer.tileIndex / mGridCols;
974         size_t top = mGridHeight * tileY;
975         size_t left = mGridWidth * tileX;
976         size_t width = (tileX == static_cast<size_t>(mGridCols) - 1) ?
977                 mOutputWidth - tileX * mGridWidth : mGridWidth;
978         size_t height = (tileY == static_cast<size_t>(mGridRows) - 1) ?
979                 mOutputHeight - tileY * mGridHeight : mGridHeight;
980         ALOGV("%s: inputBuffer tileIndex [%zu, %zu], top %zu, left %zu, width %zu, height %zu,"
981                 " timeUs %" PRId64, __FUNCTION__, tileX, tileY, top, left, width, height,
982                 inputBuffer.timeUs);
983 
984         res = copyOneYuvTile(buffer, inputFrame.yuvBuffer, top, left, width, height);
985         if (res != OK) {
986             ALOGE("%s: Failed to copy YUV tile %s (%d)", __FUNCTION__,
987                     strerror(-res), res);
988             return res;
989         }
990 
991         res = mCodec->queueInputBuffer(inputBuffer.index, 0, buffer->capacity(),
992                 inputBuffer.timeUs, 0, nullptr /*errorDetailMsg*/);
993         if (res != OK) {
994             ALOGE("%s: Failed to queueInputBuffer to Codec: %s (%d)",
995                     __FUNCTION__, strerror(-res), res);
996             return res;
997         }
998     }
999 
1000     inputFrame.codecInputBuffers.clear();
1001     return OK;
1002 }
1003 
processOneCodecOutputFrame(nsecs_t timestamp,InputFrame & inputFrame)1004 status_t HeicCompositeStream::processOneCodecOutputFrame(nsecs_t timestamp,
1005         InputFrame &inputFrame) {
1006     auto it = inputFrame.codecOutputBuffers.begin();
1007     sp<MediaCodecBuffer> buffer;
1008     status_t res = mCodec->getOutputBuffer(it->index, &buffer);
1009     if (res != OK) {
1010         ALOGE("%s: Error getting Heic codec output buffer at index %d: %s (%d)",
1011                 __FUNCTION__, it->index, strerror(-res), res);
1012         return res;
1013     }
1014     if (buffer == nullptr) {
1015         ALOGE("%s: Invalid Heic codec output buffer at index %d",
1016                 __FUNCTION__, it->index);
1017         return BAD_VALUE;
1018     }
1019 
1020     sp<ABuffer> aBuffer = new ABuffer(buffer->data(), buffer->size());
1021     res = inputFrame.muxer->writeSampleData(
1022             aBuffer, inputFrame.trackIndex, timestamp, 0 /*flags*/);
1023     if (res != OK) {
1024         ALOGE("%s: Failed to write buffer index %d to muxer: %s (%d)",
1025                 __FUNCTION__, it->index, strerror(-res), res);
1026         return res;
1027     }
1028 
1029     mCodec->releaseOutputBuffer(it->index);
1030     if (inputFrame.pendingOutputTiles == 0) {
1031         ALOGW("%s: Codec generated more tiles than expected!", __FUNCTION__);
1032     } else {
1033         inputFrame.pendingOutputTiles--;
1034     }
1035 
1036     inputFrame.codecOutputBuffers.erase(inputFrame.codecOutputBuffers.begin());
1037 
1038     ALOGV("%s: [%" PRId64 "]: Output buffer index %d",
1039         __FUNCTION__, timestamp, it->index);
1040     return OK;
1041 }
1042 
processCompletedInputFrame(nsecs_t timestamp,InputFrame & inputFrame)1043 status_t HeicCompositeStream::processCompletedInputFrame(nsecs_t timestamp,
1044         InputFrame &inputFrame) {
1045     sp<ANativeWindow> outputANW = mOutputSurface;
1046     inputFrame.muxer->stop();
1047 
1048     // Copy the content of the file to memory.
1049     sp<GraphicBuffer> gb = GraphicBuffer::from(inputFrame.anb);
1050     void* dstBuffer;
1051     auto res = gb->lockAsync(GRALLOC_USAGE_SW_WRITE_OFTEN, &dstBuffer, inputFrame.fenceFd);
1052     if (res != OK) {
1053         ALOGE("%s: Error trying to lock output buffer fence: %s (%d)", __FUNCTION__,
1054                 strerror(-res), res);
1055         return res;
1056     }
1057 
1058     off_t fSize = lseek(inputFrame.fileFd, 0, SEEK_END);
1059     if (static_cast<size_t>(fSize) > mMaxHeicBufferSize - sizeof(CameraBlob)) {
1060         ALOGE("%s: Error: MediaMuxer output size %ld is larger than buffer sizer %zu",
1061                 __FUNCTION__, fSize, mMaxHeicBufferSize - sizeof(CameraBlob));
1062         return BAD_VALUE;
1063     }
1064 
1065     lseek(inputFrame.fileFd, 0, SEEK_SET);
1066     ssize_t bytesRead = read(inputFrame.fileFd, dstBuffer, fSize);
1067     if (bytesRead < fSize) {
1068         ALOGE("%s: Only %zd of %ld bytes read", __FUNCTION__, bytesRead, fSize);
1069         return BAD_VALUE;
1070     }
1071 
1072     close(inputFrame.fileFd);
1073     inputFrame.fileFd = -1;
1074 
1075     // Fill in HEIC header
1076     uint8_t *header = static_cast<uint8_t*>(dstBuffer) + mMaxHeicBufferSize - sizeof(CameraBlob);
1077     struct CameraBlob *blobHeader = (struct CameraBlob *)header;
1078     // Must be in sync with CAMERA3_HEIC_BLOB_ID in android_media_Utils.cpp
1079     blobHeader->blobId = static_cast<CameraBlobId>(0x00FE);
1080     blobHeader->blobSize = fSize;
1081 
1082     res = native_window_set_buffers_timestamp(mOutputSurface.get(), timestamp);
1083     if (res != OK) {
1084         ALOGE("%s: Stream %d: Error setting timestamp: %s (%d)",
1085                __FUNCTION__, getStreamId(), strerror(-res), res);
1086         return res;
1087     }
1088 
1089     res = outputANW->queueBuffer(mOutputSurface.get(), inputFrame.anb, /*fence*/ -1);
1090     if (res != OK) {
1091         ALOGE("%s: Failed to queueBuffer to Heic stream: %s (%d)", __FUNCTION__,
1092                 strerror(-res), res);
1093         return res;
1094     }
1095     inputFrame.anb = nullptr;
1096     mDequeuedOutputBufferCnt--;
1097 
1098     ALOGV("%s: [%" PRId64 "]", __FUNCTION__, timestamp);
1099     ATRACE_ASYNC_END("HEIC capture", inputFrame.frameNumber);
1100     return OK;
1101 }
1102 
1103 
releaseInputFrameLocked(InputFrame * inputFrame)1104 void HeicCompositeStream::releaseInputFrameLocked(InputFrame *inputFrame /*out*/) {
1105     if (inputFrame == nullptr) {
1106         return;
1107     }
1108 
1109     if (inputFrame->appSegmentBuffer.data != nullptr) {
1110         mAppSegmentConsumer->unlockBuffer(inputFrame->appSegmentBuffer);
1111         inputFrame->appSegmentBuffer.data = nullptr;
1112     }
1113 
1114     while (!inputFrame->codecOutputBuffers.empty()) {
1115         auto it = inputFrame->codecOutputBuffers.begin();
1116         ALOGV("%s: releaseOutputBuffer index %d", __FUNCTION__, it->index);
1117         mCodec->releaseOutputBuffer(it->index);
1118         inputFrame->codecOutputBuffers.erase(it);
1119     }
1120 
1121     if (inputFrame->yuvBuffer.data != nullptr) {
1122         mMainImageConsumer->unlockBuffer(inputFrame->yuvBuffer);
1123         inputFrame->yuvBuffer.data = nullptr;
1124         mYuvBufferAcquired = false;
1125     }
1126 
1127     while (!inputFrame->codecInputBuffers.empty()) {
1128         auto it = inputFrame->codecInputBuffers.begin();
1129         inputFrame->codecInputBuffers.erase(it);
1130     }
1131 
1132     if ((inputFrame->error || mErrorState) && !inputFrame->errorNotified) {
1133         notifyError(inputFrame->frameNumber);
1134         inputFrame->errorNotified = true;
1135     }
1136 
1137     if (inputFrame->fileFd >= 0) {
1138         close(inputFrame->fileFd);
1139         inputFrame->fileFd = -1;
1140     }
1141 
1142     if (inputFrame->anb != nullptr) {
1143         sp<ANativeWindow> outputANW = mOutputSurface;
1144         outputANW->cancelBuffer(mOutputSurface.get(), inputFrame->anb, /*fence*/ -1);
1145         inputFrame->anb = nullptr;
1146     }
1147 }
1148 
releaseInputFramesLocked()1149 void HeicCompositeStream::releaseInputFramesLocked() {
1150     auto it = mPendingInputFrames.begin();
1151     while (it != mPendingInputFrames.end()) {
1152         auto& inputFrame = it->second;
1153         if (inputFrame.error ||
1154             (inputFrame.appSegmentWritten && inputFrame.pendingOutputTiles == 0)) {
1155             releaseInputFrameLocked(&inputFrame);
1156             it = mPendingInputFrames.erase(it);
1157         } else {
1158             it++;
1159         }
1160     }
1161 }
1162 
initializeCodec(uint32_t width,uint32_t height,const sp<CameraDeviceBase> & cameraDevice)1163 status_t HeicCompositeStream::initializeCodec(uint32_t width, uint32_t height,
1164         const sp<CameraDeviceBase>& cameraDevice) {
1165     ALOGV("%s", __FUNCTION__);
1166 
1167     bool useGrid = false;
1168     AString hevcName;
1169     bool isSizeSupported = isSizeSupportedByHeifEncoder(width, height,
1170             &mUseHeic, &useGrid, nullptr, &hevcName);
1171     if (!isSizeSupported) {
1172         ALOGE("%s: Encoder doesnt' support size %u x %u!",
1173                 __FUNCTION__, width, height);
1174         return BAD_VALUE;
1175     }
1176 
1177     // Create Looper for MediaCodec.
1178     auto desiredMime = mUseHeic ? MIMETYPE_IMAGE_ANDROID_HEIC : MIMETYPE_VIDEO_HEVC;
1179     mCodecLooper = new ALooper;
1180     mCodecLooper->setName("Camera3-HeicComposite-MediaCodecLooper");
1181     status_t res = mCodecLooper->start(
1182             false,   // runOnCallingThread
1183             false,    // canCallJava
1184             PRIORITY_AUDIO);
1185     if (res != OK) {
1186         ALOGE("%s: Failed to start codec looper: %s (%d)",
1187                 __FUNCTION__, strerror(-res), res);
1188         return NO_INIT;
1189     }
1190 
1191     // Create HEIC/HEVC codec.
1192     if (mUseHeic) {
1193         mCodec = MediaCodec::CreateByType(mCodecLooper, desiredMime, true /*encoder*/);
1194     } else {
1195         mCodec = MediaCodec::CreateByComponentName(mCodecLooper, hevcName);
1196     }
1197     if (mCodec == nullptr) {
1198         ALOGE("%s: Failed to create codec for %s", __FUNCTION__, desiredMime);
1199         return NO_INIT;
1200     }
1201 
1202     // Create Looper and handler for Codec callback.
1203     mCodecCallbackHandler = new CodecCallbackHandler(this);
1204     if (mCodecCallbackHandler == nullptr) {
1205         ALOGE("%s: Failed to create codec callback handler", __FUNCTION__);
1206         return NO_MEMORY;
1207     }
1208     mCallbackLooper = new ALooper;
1209     mCallbackLooper->setName("Camera3-HeicComposite-MediaCodecCallbackLooper");
1210     res = mCallbackLooper->start(
1211             false,   // runOnCallingThread
1212             false,    // canCallJava
1213             PRIORITY_AUDIO);
1214     if (res != OK) {
1215         ALOGE("%s: Failed to start media callback looper: %s (%d)",
1216                 __FUNCTION__, strerror(-res), res);
1217         return NO_INIT;
1218     }
1219     mCallbackLooper->registerHandler(mCodecCallbackHandler);
1220 
1221     mAsyncNotify = new AMessage(kWhatCallbackNotify, mCodecCallbackHandler);
1222     res = mCodec->setCallback(mAsyncNotify);
1223     if (res != OK) {
1224         ALOGE("%s: Failed to set MediaCodec callback: %s (%d)", __FUNCTION__,
1225                 strerror(-res), res);
1226         return res;
1227     }
1228 
1229     // Create output format and configure the Codec.
1230     sp<AMessage> outputFormat = new AMessage();
1231     outputFormat->setString(KEY_MIME, desiredMime);
1232     outputFormat->setInt32(KEY_BITRATE_MODE, BITRATE_MODE_CQ);
1233     outputFormat->setInt32(KEY_QUALITY, kDefaultJpegQuality);
1234     // Ask codec to skip timestamp check and encode all frames.
1235     outputFormat->setInt64(KEY_MAX_PTS_GAP_TO_ENCODER, kNoFrameDropMaxPtsGap);
1236 
1237     int32_t gridWidth, gridHeight, gridRows, gridCols;
1238     if (useGrid || mUseHeic) {
1239         gridWidth = HeicEncoderInfoManager::kGridWidth;
1240         gridHeight = HeicEncoderInfoManager::kGridHeight;
1241         gridRows = (height + gridHeight - 1)/gridHeight;
1242         gridCols = (width + gridWidth - 1)/gridWidth;
1243 
1244         if (mUseHeic) {
1245             outputFormat->setInt32(KEY_TILE_WIDTH, gridWidth);
1246             outputFormat->setInt32(KEY_TILE_HEIGHT, gridHeight);
1247             outputFormat->setInt32(KEY_GRID_COLUMNS, gridCols);
1248             outputFormat->setInt32(KEY_GRID_ROWS, gridRows);
1249         }
1250 
1251     } else {
1252         gridWidth = width;
1253         gridHeight = height;
1254         gridRows = 1;
1255         gridCols = 1;
1256     }
1257 
1258     outputFormat->setInt32(KEY_WIDTH, !useGrid ? width : gridWidth);
1259     outputFormat->setInt32(KEY_HEIGHT, !useGrid ? height : gridHeight);
1260     outputFormat->setInt32(KEY_I_FRAME_INTERVAL, 0);
1261     outputFormat->setInt32(KEY_COLOR_FORMAT,
1262             useGrid ? COLOR_FormatYUV420Flexible : COLOR_FormatSurface);
1263     outputFormat->setInt32(KEY_FRAME_RATE, gridRows * gridCols);
1264     // This only serves as a hint to encoder when encoding is not real-time.
1265     outputFormat->setInt32(KEY_OPERATING_RATE, useGrid ? kGridOpRate : kNoGridOpRate);
1266 
1267     res = mCodec->configure(outputFormat, nullptr /*nativeWindow*/,
1268             nullptr /*crypto*/, CONFIGURE_FLAG_ENCODE);
1269     if (res != OK) {
1270         ALOGE("%s: Failed to configure codec: %s (%d)", __FUNCTION__,
1271                 strerror(-res), res);
1272         return res;
1273     }
1274 
1275     mGridWidth = gridWidth;
1276     mGridHeight = gridHeight;
1277     mGridRows = gridRows;
1278     mGridCols = gridCols;
1279     mUseGrid = useGrid;
1280     mOutputWidth = width;
1281     mOutputHeight = height;
1282     mAppSegmentMaxSize = calcAppSegmentMaxSize(cameraDevice->info());
1283     mMaxHeicBufferSize = mOutputWidth * mOutputHeight * 3 / 2 + mAppSegmentMaxSize;
1284 
1285     return OK;
1286 }
1287 
deinitCodec()1288 void HeicCompositeStream::deinitCodec() {
1289     ALOGV("%s", __FUNCTION__);
1290     if (mCodec != nullptr) {
1291         mCodec->stop();
1292         mCodec->release();
1293         mCodec.clear();
1294     }
1295 
1296     if (mCodecLooper != nullptr) {
1297         mCodecLooper->stop();
1298         mCodecLooper.clear();
1299     }
1300 
1301     if (mCallbackLooper != nullptr) {
1302         mCallbackLooper->stop();
1303         mCallbackLooper.clear();
1304     }
1305 
1306     mAsyncNotify.clear();
1307     mFormat.clear();
1308 }
1309 
1310 // Return the size of the complete list of app segment, 0 indicates failure
findAppSegmentsSize(const uint8_t * appSegmentBuffer,size_t maxSize,size_t * app1SegmentSize)1311 size_t HeicCompositeStream::findAppSegmentsSize(const uint8_t* appSegmentBuffer,
1312         size_t maxSize, size_t *app1SegmentSize) {
1313     if (appSegmentBuffer == nullptr || app1SegmentSize == nullptr) {
1314         ALOGE("%s: Invalid input appSegmentBuffer %p, app1SegmentSize %p",
1315                 __FUNCTION__, appSegmentBuffer, app1SegmentSize);
1316         return 0;
1317     }
1318 
1319     size_t expectedSize = 0;
1320     // First check for EXIF transport header at the end of the buffer
1321     const uint8_t *header = appSegmentBuffer + (maxSize - sizeof(struct CameraBlob));
1322     const struct CameraBlob *blob = (const struct CameraBlob*)(header);
1323     if (blob->blobId != CameraBlobId::JPEG_APP_SEGMENTS) {
1324         ALOGE("%s: Invalid EXIF blobId %hu", __FUNCTION__, blob->blobId);
1325         return 0;
1326     }
1327 
1328     expectedSize = blob->blobSize;
1329     if (expectedSize == 0 || expectedSize > maxSize - sizeof(struct CameraBlob)) {
1330         ALOGE("%s: Invalid blobSize %zu.", __FUNCTION__, expectedSize);
1331         return 0;
1332     }
1333 
1334     uint32_t totalSize = 0;
1335 
1336     // Verify APP1 marker (mandatory)
1337     uint8_t app1Marker[] = {0xFF, 0xE1};
1338     if (memcmp(appSegmentBuffer, app1Marker, sizeof(app1Marker))) {
1339         ALOGE("%s: Invalid APP1 marker: %x, %x", __FUNCTION__,
1340                 appSegmentBuffer[0], appSegmentBuffer[1]);
1341         return 0;
1342     }
1343     totalSize += sizeof(app1Marker);
1344 
1345     uint16_t app1Size = (static_cast<uint16_t>(appSegmentBuffer[totalSize]) << 8) +
1346             appSegmentBuffer[totalSize+1];
1347     totalSize += app1Size;
1348 
1349     ALOGV("%s: Expected APP segments size %zu, APP1 segment size %u",
1350             __FUNCTION__, expectedSize, app1Size);
1351     while (totalSize < expectedSize) {
1352         if (appSegmentBuffer[totalSize] != 0xFF ||
1353                 appSegmentBuffer[totalSize+1] <= 0xE1 ||
1354                 appSegmentBuffer[totalSize+1] > 0xEF) {
1355             // Invalid APPn marker
1356             ALOGE("%s: Invalid APPn marker: %x, %x", __FUNCTION__,
1357                     appSegmentBuffer[totalSize], appSegmentBuffer[totalSize+1]);
1358             return 0;
1359         }
1360         totalSize += 2;
1361 
1362         uint16_t appnSize = (static_cast<uint16_t>(appSegmentBuffer[totalSize]) << 8) +
1363                 appSegmentBuffer[totalSize+1];
1364         totalSize += appnSize;
1365     }
1366 
1367     if (totalSize != expectedSize) {
1368         ALOGE("%s: Invalid JPEG APP segments: totalSize %u vs expected size %zu",
1369                 __FUNCTION__, totalSize, expectedSize);
1370         return 0;
1371     }
1372 
1373     *app1SegmentSize = app1Size + sizeof(app1Marker);
1374     return expectedSize;
1375 }
1376 
findTimestampInNsLocked(int64_t timeInUs)1377 int64_t HeicCompositeStream::findTimestampInNsLocked(int64_t timeInUs) {
1378     for (const auto& fn : mFrameNumberMap) {
1379         if (timeInUs == ns2us(fn.second)) {
1380             return fn.second;
1381         }
1382     }
1383     for (const auto& inputFrame : mPendingInputFrames) {
1384         if (timeInUs == ns2us(inputFrame.first)) {
1385             return inputFrame.first;
1386         }
1387     }
1388     return -1;
1389 }
1390 
copyOneYuvTile(sp<MediaCodecBuffer> & codecBuffer,const CpuConsumer::LockedBuffer & yuvBuffer,size_t top,size_t left,size_t width,size_t height)1391 status_t HeicCompositeStream::copyOneYuvTile(sp<MediaCodecBuffer>& codecBuffer,
1392         const CpuConsumer::LockedBuffer& yuvBuffer,
1393         size_t top, size_t left, size_t width, size_t height) {
1394     ATRACE_CALL();
1395 
1396     // Get stride information for codecBuffer
1397     sp<ABuffer> imageData;
1398     if (!codecBuffer->meta()->findBuffer("image-data", &imageData)) {
1399         ALOGE("%s: Codec input buffer is not for image data!", __FUNCTION__);
1400         return BAD_VALUE;
1401     }
1402     if (imageData->size() != sizeof(MediaImage2)) {
1403         ALOGE("%s: Invalid codec input image size %zu, expected %zu",
1404                 __FUNCTION__, imageData->size(), sizeof(MediaImage2));
1405         return BAD_VALUE;
1406     }
1407     MediaImage2* imageInfo = reinterpret_cast<MediaImage2*>(imageData->data());
1408     if (imageInfo->mType != MediaImage2::MEDIA_IMAGE_TYPE_YUV ||
1409             imageInfo->mBitDepth != 8 ||
1410             imageInfo->mBitDepthAllocated != 8 ||
1411             imageInfo->mNumPlanes != 3) {
1412         ALOGE("%s: Invalid codec input image info: mType %d, mBitDepth %d, "
1413                 "mBitDepthAllocated %d, mNumPlanes %d!", __FUNCTION__,
1414                 imageInfo->mType, imageInfo->mBitDepth,
1415                 imageInfo->mBitDepthAllocated, imageInfo->mNumPlanes);
1416         return BAD_VALUE;
1417     }
1418 
1419     ALOGV("%s: yuvBuffer chromaStep %d, chromaStride %d",
1420             __FUNCTION__, yuvBuffer.chromaStep, yuvBuffer.chromaStride);
1421     ALOGV("%s: U offset %u, V offset %u, U rowInc %d, V rowInc %d, U colInc %d, V colInc %d",
1422             __FUNCTION__, imageInfo->mPlane[MediaImage2::U].mOffset,
1423             imageInfo->mPlane[MediaImage2::V].mOffset,
1424             imageInfo->mPlane[MediaImage2::U].mRowInc,
1425             imageInfo->mPlane[MediaImage2::V].mRowInc,
1426             imageInfo->mPlane[MediaImage2::U].mColInc,
1427             imageInfo->mPlane[MediaImage2::V].mColInc);
1428 
1429     // Y
1430     for (auto row = top; row < top+height; row++) {
1431         uint8_t *dst = codecBuffer->data() + imageInfo->mPlane[MediaImage2::Y].mOffset +
1432                 imageInfo->mPlane[MediaImage2::Y].mRowInc * (row - top);
1433         mFnCopyRow(yuvBuffer.data+row*yuvBuffer.stride+left, dst, width);
1434     }
1435 
1436     // U is Cb, V is Cr
1437     bool codecUPlaneFirst = imageInfo->mPlane[MediaImage2::V].mOffset >
1438             imageInfo->mPlane[MediaImage2::U].mOffset;
1439     uint32_t codecUvOffsetDiff = codecUPlaneFirst ?
1440             imageInfo->mPlane[MediaImage2::V].mOffset - imageInfo->mPlane[MediaImage2::U].mOffset :
1441             imageInfo->mPlane[MediaImage2::U].mOffset - imageInfo->mPlane[MediaImage2::V].mOffset;
1442     bool isCodecUvSemiplannar = (codecUvOffsetDiff == 1) &&
1443             (imageInfo->mPlane[MediaImage2::U].mRowInc ==
1444             imageInfo->mPlane[MediaImage2::V].mRowInc) &&
1445             (imageInfo->mPlane[MediaImage2::U].mColInc == 2) &&
1446             (imageInfo->mPlane[MediaImage2::V].mColInc == 2);
1447     bool isCodecUvPlannar =
1448             ((codecUPlaneFirst && codecUvOffsetDiff >=
1449                     imageInfo->mPlane[MediaImage2::U].mRowInc * imageInfo->mHeight/2) ||
1450             ((!codecUPlaneFirst && codecUvOffsetDiff >=
1451                     imageInfo->mPlane[MediaImage2::V].mRowInc * imageInfo->mHeight/2))) &&
1452             imageInfo->mPlane[MediaImage2::U].mColInc == 1 &&
1453             imageInfo->mPlane[MediaImage2::V].mColInc == 1;
1454     bool cameraUPlaneFirst = yuvBuffer.dataCr > yuvBuffer.dataCb;
1455 
1456     if (isCodecUvSemiplannar && yuvBuffer.chromaStep == 2 &&
1457             (codecUPlaneFirst == cameraUPlaneFirst)) {
1458         // UV semiplannar
1459         // The chrome plane could be either Cb first, or Cr first. Take the
1460         // smaller address.
1461         uint8_t *src = std::min(yuvBuffer.dataCb, yuvBuffer.dataCr);
1462         MediaImage2::PlaneIndex dstPlane = codecUvOffsetDiff > 0 ? MediaImage2::U : MediaImage2::V;
1463         for (auto row = top/2; row < (top+height)/2; row++) {
1464             uint8_t *dst = codecBuffer->data() + imageInfo->mPlane[dstPlane].mOffset +
1465                     imageInfo->mPlane[dstPlane].mRowInc * (row - top/2);
1466             mFnCopyRow(src+row*yuvBuffer.chromaStride+left, dst, width);
1467         }
1468     } else if (isCodecUvPlannar && yuvBuffer.chromaStep == 1) {
1469         // U plane
1470         for (auto row = top/2; row < (top+height)/2; row++) {
1471             uint8_t *dst = codecBuffer->data() + imageInfo->mPlane[MediaImage2::U].mOffset +
1472                     imageInfo->mPlane[MediaImage2::U].mRowInc * (row - top/2);
1473             mFnCopyRow(yuvBuffer.dataCb+row*yuvBuffer.chromaStride+left/2, dst, width/2);
1474         }
1475 
1476         // V plane
1477         for (auto row = top/2; row < (top+height)/2; row++) {
1478             uint8_t *dst = codecBuffer->data() + imageInfo->mPlane[MediaImage2::V].mOffset +
1479                     imageInfo->mPlane[MediaImage2::V].mRowInc * (row - top/2);
1480             mFnCopyRow(yuvBuffer.dataCr+row*yuvBuffer.chromaStride+left/2, dst, width/2);
1481         }
1482     } else {
1483         // Convert between semiplannar and plannar, or when UV orders are
1484         // different.
1485         uint8_t *dst = codecBuffer->data();
1486         for (auto row = top/2; row < (top+height)/2; row++) {
1487             for (auto col = left/2; col < (left+width)/2; col++) {
1488                 // U/Cb
1489                 int32_t dstIndex = imageInfo->mPlane[MediaImage2::U].mOffset +
1490                         imageInfo->mPlane[MediaImage2::U].mRowInc * (row - top/2) +
1491                         imageInfo->mPlane[MediaImage2::U].mColInc * (col - left/2);
1492                 int32_t srcIndex = row * yuvBuffer.chromaStride + yuvBuffer.chromaStep * col;
1493                 dst[dstIndex] = yuvBuffer.dataCb[srcIndex];
1494 
1495                 // V/Cr
1496                 dstIndex = imageInfo->mPlane[MediaImage2::V].mOffset +
1497                         imageInfo->mPlane[MediaImage2::V].mRowInc * (row - top/2) +
1498                         imageInfo->mPlane[MediaImage2::V].mColInc * (col - left/2);
1499                 srcIndex = row * yuvBuffer.chromaStride + yuvBuffer.chromaStep * col;
1500                 dst[dstIndex] = yuvBuffer.dataCr[srcIndex];
1501             }
1502         }
1503     }
1504     return OK;
1505 }
1506 
initCopyRowFunction(int32_t width)1507 void HeicCompositeStream::initCopyRowFunction(int32_t width)
1508 {
1509     using namespace libyuv;
1510 
1511     mFnCopyRow = CopyRow_C;
1512 #if defined(HAS_COPYROW_SSE2)
1513     if (TestCpuFlag(kCpuHasSSE2)) {
1514         mFnCopyRow = IS_ALIGNED(width, 32) ? CopyRow_SSE2 : CopyRow_Any_SSE2;
1515     }
1516 #endif
1517 #if defined(HAS_COPYROW_AVX)
1518     if (TestCpuFlag(kCpuHasAVX)) {
1519         mFnCopyRow = IS_ALIGNED(width, 64) ? CopyRow_AVX : CopyRow_Any_AVX;
1520     }
1521 #endif
1522 #if defined(HAS_COPYROW_ERMS)
1523     if (TestCpuFlag(kCpuHasERMS)) {
1524         mFnCopyRow = CopyRow_ERMS;
1525     }
1526 #endif
1527 #if defined(HAS_COPYROW_NEON)
1528     if (TestCpuFlag(kCpuHasNEON)) {
1529         mFnCopyRow = IS_ALIGNED(width, 32) ? CopyRow_NEON : CopyRow_Any_NEON;
1530     }
1531 #endif
1532 #if defined(HAS_COPYROW_MIPS)
1533     if (TestCpuFlag(kCpuHasMIPS)) {
1534         mFnCopyRow = CopyRow_MIPS;
1535     }
1536 #endif
1537 }
1538 
calcAppSegmentMaxSize(const CameraMetadata & info)1539 size_t HeicCompositeStream::calcAppSegmentMaxSize(const CameraMetadata& info) {
1540     camera_metadata_ro_entry_t entry = info.find(ANDROID_HEIC_INFO_MAX_JPEG_APP_SEGMENTS_COUNT);
1541     size_t maxAppsSegment = 1;
1542     if (entry.count > 0) {
1543         maxAppsSegment = entry.data.u8[0] < 1 ? 1 :
1544                 entry.data.u8[0] > 16 ? 16 : entry.data.u8[0];
1545     }
1546     return maxAppsSegment * (2 + 0xFFFF) + sizeof(struct CameraBlob);
1547 }
1548 
threadLoop()1549 bool HeicCompositeStream::threadLoop() {
1550     int64_t currentTs = INT64_MAX;
1551     bool newInputAvailable = false;
1552 
1553     {
1554         Mutex::Autolock l(mMutex);
1555         if (mErrorState) {
1556             // In case we landed in error state, return any pending buffers and
1557             // halt all further processing.
1558             compilePendingInputLocked();
1559             releaseInputFramesLocked();
1560             return false;
1561         }
1562 
1563 
1564         while (!newInputAvailable) {
1565             compilePendingInputLocked();
1566             newInputAvailable = getNextReadyInputLocked(&currentTs);
1567 
1568             if (!newInputAvailable) {
1569                 auto failingFrameNumber = getNextFailingInputLocked(&currentTs);
1570                 if (failingFrameNumber >= 0) {
1571                     // We cannot erase 'mPendingInputFrames[currentTs]' at this point because it is
1572                     // possible for two internal stream buffers to fail. In such scenario the
1573                     // composite stream should notify the client about a stream buffer error only
1574                     // once and this information is kept within 'errorNotified'.
1575                     // Any present failed input frames will be removed on a subsequent call to
1576                     // 'releaseInputFramesLocked()'.
1577                     releaseInputFrameLocked(&mPendingInputFrames[currentTs]);
1578                     currentTs = INT64_MAX;
1579                 }
1580 
1581                 auto ret = mInputReadyCondition.waitRelative(mMutex, kWaitDuration);
1582                 if (ret == TIMED_OUT) {
1583                     return true;
1584                 } else if (ret != OK) {
1585                     ALOGE("%s: Timed wait on condition failed: %s (%d)", __FUNCTION__,
1586                             strerror(-ret), ret);
1587                     return false;
1588                 }
1589             }
1590         }
1591     }
1592 
1593     auto res = processInputFrame(currentTs, mPendingInputFrames[currentTs]);
1594     Mutex::Autolock l(mMutex);
1595     if (res != OK) {
1596         ALOGE("%s: Failed processing frame with timestamp: %" PRIu64 ": %s (%d)",
1597                 __FUNCTION__, currentTs, strerror(-res), res);
1598         mPendingInputFrames[currentTs].error = true;
1599     }
1600 
1601     releaseInputFramesLocked();
1602 
1603     return true;
1604 }
1605 
onStreamBufferError(const CaptureResultExtras & resultExtras)1606 bool HeicCompositeStream::onStreamBufferError(const CaptureResultExtras& resultExtras) {
1607     bool res = false;
1608     // Buffer errors concerning internal composite streams should not be directly visible to
1609     // camera clients. They must only receive a single buffer error with the public composite
1610     // stream id.
1611     if ((resultExtras.errorStreamId == mAppSegmentStreamId) ||
1612             (resultExtras.errorStreamId == mMainImageStreamId)) {
1613         flagAnErrorFrameNumber(resultExtras.frameNumber);
1614         res = true;
1615     }
1616 
1617     return res;
1618 }
1619 
onResultError(const CaptureResultExtras & resultExtras)1620 void HeicCompositeStream::onResultError(const CaptureResultExtras& resultExtras) {
1621     // For result error, since the APPS_SEGMENT buffer already contains EXIF,
1622     // simply skip using the capture result metadata to override EXIF.
1623     Mutex::Autolock l(mMutex);
1624 
1625     int64_t timestamp = -1;
1626     for (const auto& fn : mFrameNumberMap) {
1627         if (fn.first == resultExtras.frameNumber) {
1628             timestamp = fn.second;
1629             break;
1630         }
1631     }
1632     if (timestamp == -1) {
1633         for (const auto& inputFrame : mPendingInputFrames) {
1634             if (inputFrame.second.frameNumber == resultExtras.frameNumber) {
1635                 timestamp = inputFrame.first;
1636                 break;
1637             }
1638         }
1639     }
1640 
1641     if (timestamp == -1) {
1642         ALOGE("%s: Failed to find shutter timestamp for result error!", __FUNCTION__);
1643         return;
1644     }
1645 
1646     mCaptureResults.emplace(timestamp, std::make_tuple(resultExtras.frameNumber, CameraMetadata()));
1647     mInputReadyCondition.signal();
1648 }
1649 
onMessageReceived(const sp<AMessage> & msg)1650 void HeicCompositeStream::CodecCallbackHandler::onMessageReceived(const sp<AMessage> &msg) {
1651     sp<HeicCompositeStream> parent = mParent.promote();
1652     if (parent == nullptr) return;
1653 
1654     switch (msg->what()) {
1655         case kWhatCallbackNotify: {
1656              int32_t cbID;
1657              if (!msg->findInt32("callbackID", &cbID)) {
1658                  ALOGE("kWhatCallbackNotify: callbackID is expected.");
1659                  break;
1660              }
1661 
1662              ALOGV("kWhatCallbackNotify: cbID = %d", cbID);
1663 
1664              switch (cbID) {
1665                  case MediaCodec::CB_INPUT_AVAILABLE: {
1666                      int32_t index;
1667                      if (!msg->findInt32("index", &index)) {
1668                          ALOGE("CB_INPUT_AVAILABLE: index is expected.");
1669                          break;
1670                      }
1671                      parent->onHeicInputFrameAvailable(index);
1672                      break;
1673                  }
1674 
1675                  case MediaCodec::CB_OUTPUT_AVAILABLE: {
1676                      int32_t index;
1677                      size_t offset;
1678                      size_t size;
1679                      int64_t timeUs;
1680                      int32_t flags;
1681 
1682                      if (!msg->findInt32("index", &index)) {
1683                          ALOGE("CB_OUTPUT_AVAILABLE: index is expected.");
1684                          break;
1685                      }
1686                      if (!msg->findSize("offset", &offset)) {
1687                          ALOGE("CB_OUTPUT_AVAILABLE: offset is expected.");
1688                          break;
1689                      }
1690                      if (!msg->findSize("size", &size)) {
1691                          ALOGE("CB_OUTPUT_AVAILABLE: size is expected.");
1692                          break;
1693                      }
1694                      if (!msg->findInt64("timeUs", &timeUs)) {
1695                          ALOGE("CB_OUTPUT_AVAILABLE: timeUs is expected.");
1696                          break;
1697                      }
1698                      if (!msg->findInt32("flags", &flags)) {
1699                          ALOGE("CB_OUTPUT_AVAILABLE: flags is expected.");
1700                          break;
1701                      }
1702 
1703                      CodecOutputBufferInfo bufferInfo = {
1704                          index,
1705                          (int32_t)offset,
1706                          (int32_t)size,
1707                          timeUs,
1708                          (uint32_t)flags};
1709 
1710                      parent->onHeicOutputFrameAvailable(bufferInfo);
1711                      break;
1712                  }
1713 
1714                  case MediaCodec::CB_OUTPUT_FORMAT_CHANGED: {
1715                      sp<AMessage> format;
1716                      if (!msg->findMessage("format", &format)) {
1717                          ALOGE("CB_OUTPUT_FORMAT_CHANGED: format is expected.");
1718                          break;
1719                      }
1720                      // Here format is MediaCodec's internal copy of output format.
1721                      // Make a copy since onHeicFormatChanged() might modify it.
1722                      sp<AMessage> formatCopy;
1723                      if (format != nullptr) {
1724                          formatCopy = format->dup();
1725                      }
1726                      parent->onHeicFormatChanged(formatCopy);
1727                      break;
1728                  }
1729 
1730                  case MediaCodec::CB_ERROR: {
1731                      status_t err;
1732                      int32_t actionCode;
1733                      AString detail;
1734                      if (!msg->findInt32("err", &err)) {
1735                          ALOGE("CB_ERROR: err is expected.");
1736                          break;
1737                      }
1738                      if (!msg->findInt32("action", &actionCode)) {
1739                          ALOGE("CB_ERROR: action is expected.");
1740                          break;
1741                      }
1742                      msg->findString("detail", &detail);
1743                      ALOGE("Codec reported error(0x%x), actionCode(%d), detail(%s)",
1744                              err, actionCode, detail.c_str());
1745 
1746                      parent->onHeicCodecError();
1747                      break;
1748                  }
1749 
1750                  default: {
1751                      ALOGE("kWhatCallbackNotify: callbackID(%d) is unexpected.", cbID);
1752                      break;
1753                  }
1754              }
1755              break;
1756         }
1757 
1758         default:
1759             ALOGE("shouldn't be here");
1760             break;
1761     }
1762 }
1763 
1764 }; // namespace camera3
1765 }; // namespace android
1766