1 /* 2 * Copyright (C) 2019 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 #ifndef ANDROID_SERVERS_CAMERA_CAMERA3_HEIC_COMPOSITE_STREAM_H 18 #define ANDROID_SERVERS_CAMERA_CAMERA3_HEIC_COMPOSITE_STREAM_H 19 20 #include <queue> 21 22 #include <gui/IProducerListener.h> 23 #include <gui/CpuConsumer.h> 24 25 #include <media/hardware/VideoAPI.h> 26 #include <media/MediaCodecBuffer.h> 27 #include <media/stagefright/foundation/ALooper.h> 28 #include <media/stagefright/foundation/AMessage.h> 29 #include <media/stagefright/MediaCodec.h> 30 #include <media/stagefright/MediaMuxer.h> 31 32 #include "CompositeStream.h" 33 34 namespace android { 35 namespace camera3 { 36 37 class HeicCompositeStream : public CompositeStream, public Thread, 38 public CpuConsumer::FrameAvailableListener { 39 public: 40 HeicCompositeStream(wp<CameraDeviceBase> device, 41 wp<hardware::camera2::ICameraDeviceCallbacks> cb); 42 ~HeicCompositeStream() override; 43 44 static bool isHeicCompositeStream(const sp<Surface> &surface); 45 46 status_t createInternalStreams(const std::vector<sp<Surface>>& consumers, 47 bool hasDeferredConsumer, uint32_t width, uint32_t height, int format, 48 camera3_stream_rotation_t rotation, int *id, const String8& physicalCameraId, 49 std::vector<int> *surfaceIds, int streamSetId, bool isShared) override; 50 51 status_t deleteInternalStreams() override; 52 53 status_t configureStream() override; 54 55 status_t insertGbp(SurfaceMap* /*out*/outSurfaceMap, Vector<int32_t>* /*out*/outputStreamIds, 56 int32_t* /*out*/currentStreamId) override; 57 58 void onShutter(const CaptureResultExtras& resultExtras, nsecs_t timestamp) override; 59 getStreamId()60 int getStreamId() override { return mMainImageStreamId; } 61 62 // Use onShutter to keep track of frame number <-> timestamp mapping. 63 void onBufferReleased(const BufferInfo& bufferInfo) override; 64 void onBufferRequestForFrameNumber(uint64_t frameNumber, int streamId, 65 const CameraMetadata& settings) override; 66 67 // CpuConsumer listener implementation 68 void onFrameAvailable(const BufferItem& item) override; 69 70 // Return stream information about the internal camera streams 71 static status_t getCompositeStreamInfo(const OutputStreamInfo &streamInfo, 72 const CameraMetadata& ch, std::vector<OutputStreamInfo>* compositeOutput /*out*/); 73 74 static bool isSizeSupportedByHeifEncoder(int32_t width, int32_t height, 75 bool* useHeic, bool* useGrid, int64_t* stall, AString* hevcName = nullptr); 76 static bool isInMemoryTempFileSupported(); 77 protected: 78 79 bool threadLoop() override; 80 bool onStreamBufferError(const CaptureResultExtras& resultExtras) override; 81 void onResultError(const CaptureResultExtras& resultExtras) override; 82 83 private: 84 // 85 // HEIC/HEVC Codec related structures, utility functions, and callbacks 86 // 87 struct CodecOutputBufferInfo { 88 int32_t index; 89 int32_t offset; 90 int32_t size; 91 int64_t timeUs; 92 uint32_t flags; 93 }; 94 95 struct CodecInputBufferInfo { 96 int32_t index; 97 int64_t timeUs; 98 size_t tileIndex; 99 }; 100 101 class CodecCallbackHandler : public AHandler { 102 public: CodecCallbackHandler(wp<HeicCompositeStream> parent)103 explicit CodecCallbackHandler(wp<HeicCompositeStream> parent) { 104 mParent = parent; 105 } 106 virtual void onMessageReceived(const sp<AMessage> &msg); 107 private: 108 wp<HeicCompositeStream> mParent; 109 }; 110 111 enum { 112 kWhatCallbackNotify, 113 }; 114 115 bool mUseHeic; 116 sp<MediaCodec> mCodec; 117 sp<ALooper> mCodecLooper, mCallbackLooper; 118 sp<CodecCallbackHandler> mCodecCallbackHandler; 119 sp<AMessage> mAsyncNotify; 120 sp<AMessage> mFormat; 121 size_t mNumOutputTiles; 122 123 int32_t mOutputWidth, mOutputHeight; 124 size_t mMaxHeicBufferSize; 125 int32_t mGridWidth, mGridHeight; 126 size_t mGridRows, mGridCols; 127 bool mUseGrid; // Whether to use framework YUV frame tiling. 128 129 static const int64_t kNoFrameDropMaxPtsGap = -1000000; 130 static const int32_t kNoGridOpRate = 30; 131 static const int32_t kGridOpRate = 120; 132 133 void onHeicOutputFrameAvailable(const CodecOutputBufferInfo& bufferInfo); 134 void onHeicInputFrameAvailable(int32_t index); // Only called for YUV input mode. 135 void onHeicFormatChanged(sp<AMessage>& newFormat); 136 void onHeicCodecError(); 137 138 status_t initializeCodec(uint32_t width, uint32_t height, 139 const sp<CameraDeviceBase>& cameraDevice); 140 void deinitCodec(); 141 142 // 143 // Composite stream related structures, utility functions and callbacks. 144 // 145 struct InputFrame { 146 int32_t orientation; 147 int32_t quality; 148 149 CpuConsumer::LockedBuffer appSegmentBuffer; 150 std::vector<CodecOutputBufferInfo> codecOutputBuffers; 151 std::unique_ptr<CameraMetadata> result; 152 153 // Fields that are only applicable to HEVC tiling. 154 CpuConsumer::LockedBuffer yuvBuffer; 155 std::vector<CodecInputBufferInfo> codecInputBuffers; 156 157 bool error; 158 bool errorNotified; 159 int64_t frameNumber; 160 161 sp<AMessage> format; 162 sp<MediaMuxer> muxer; 163 int fenceFd; 164 int fileFd; 165 ssize_t trackIndex; 166 ANativeWindowBuffer *anb; 167 168 bool appSegmentWritten; 169 size_t pendingOutputTiles; 170 size_t codecInputCounter; 171 InputFrameInputFrame172 InputFrame() : orientation(0), quality(kDefaultJpegQuality), error(false), 173 errorNotified(false), frameNumber(-1), fenceFd(-1), fileFd(-1), 174 trackIndex(-1), anb(nullptr), appSegmentWritten(false), 175 pendingOutputTiles(0), codecInputCounter(0) { } 176 }; 177 178 void compilePendingInputLocked(); 179 // Find first complete and valid frame with smallest timestamp 180 bool getNextReadyInputLocked(int64_t *currentTs /*out*/); 181 // Find next failing frame number with smallest timestamp and return respective frame number 182 int64_t getNextFailingInputLocked(int64_t *currentTs /*out*/); 183 184 status_t processInputFrame(nsecs_t timestamp, InputFrame &inputFrame); 185 status_t processCodecInputFrame(InputFrame &inputFrame); 186 status_t startMuxerForInputFrame(nsecs_t timestamp, InputFrame &inputFrame); 187 status_t processAppSegment(nsecs_t timestamp, InputFrame &inputFrame); 188 status_t processOneCodecOutputFrame(nsecs_t timestamp, InputFrame &inputFrame); 189 status_t processCompletedInputFrame(nsecs_t timestamp, InputFrame &inputFrame); 190 191 void releaseInputFrameLocked(InputFrame *inputFrame /*out*/); 192 void releaseInputFramesLocked(); 193 194 size_t findAppSegmentsSize(const uint8_t* appSegmentBuffer, size_t maxSize, 195 size_t* app1SegmentSize); 196 int64_t findTimestampInNsLocked(int64_t timeInUs); 197 status_t copyOneYuvTile(sp<MediaCodecBuffer>& codecBuffer, 198 const CpuConsumer::LockedBuffer& yuvBuffer, 199 size_t top, size_t left, size_t width, size_t height); 200 void initCopyRowFunction(int32_t width); 201 static size_t calcAppSegmentMaxSize(const CameraMetadata& info); 202 203 static const nsecs_t kWaitDuration = 10000000; // 10 ms 204 static const int32_t kDefaultJpegQuality = 99; 205 static const auto kJpegDataSpace = HAL_DATASPACE_V0_JFIF; 206 static const android_dataspace kAppSegmentDataSpace = 207 static_cast<android_dataspace>(HAL_DATASPACE_JPEG_APP_SEGMENTS); 208 static const android_dataspace kHeifDataSpace = 209 static_cast<android_dataspace>(HAL_DATASPACE_HEIF); 210 // Use the limit of pipeline depth in the API sepc as maximum number of acquired 211 // app segment buffers. 212 static const uint32_t kMaxAcquiredAppSegment = 8; 213 214 int mAppSegmentStreamId, mAppSegmentSurfaceId; 215 sp<CpuConsumer> mAppSegmentConsumer; 216 sp<Surface> mAppSegmentSurface; 217 size_t mAppSegmentMaxSize; 218 CameraMetadata mStaticInfo; 219 220 int mMainImageStreamId, mMainImageSurfaceId; 221 sp<Surface> mMainImageSurface; 222 sp<CpuConsumer> mMainImageConsumer; // Only applicable for HEVC codec. 223 bool mYuvBufferAcquired; // Only applicable to HEVC codec 224 225 static const int32_t kMaxOutputSurfaceProducerCount = 1; 226 sp<Surface> mOutputSurface; 227 sp<ProducerListener> mProducerListener; 228 int32_t mDequeuedOutputBufferCnt; 229 230 // Map from frame number to JPEG setting of orientation+quality 231 std::map<int64_t, std::pair<int32_t, int32_t>> mSettingsByFrameNumber; 232 // Map from timestamp to JPEG setting of orientation+quality 233 std::map<int64_t, std::pair<int32_t, int32_t>> mSettingsByTimestamp; 234 235 // Keep all incoming APP segment Blob buffer pending further processing. 236 std::vector<int64_t> mInputAppSegmentBuffers; 237 int32_t mLockedAppSegmentBufferCnt; 238 239 // Keep all incoming HEIC blob buffer pending further processing. 240 std::vector<CodecOutputBufferInfo> mCodecOutputBuffers; 241 std::queue<int64_t> mCodecOutputBufferTimestamps; 242 size_t mCodecOutputCounter; 243 244 // Keep all incoming Yuv buffer pending tiling and encoding (for HEVC YUV tiling only) 245 std::vector<int64_t> mInputYuvBuffers; 246 // Keep all codec input buffers ready to be filled out (for HEVC YUV tiling only) 247 std::vector<int32_t> mCodecInputBuffers; 248 249 // Artificial strictly incremental YUV grid timestamp to make encoder happy. 250 int64_t mGridTimestampUs; 251 252 // In most common use case, entries are accessed in order. 253 std::map<int64_t, InputFrame> mPendingInputFrames; 254 255 // Function pointer of libyuv row copy. 256 void (*mFnCopyRow)(const uint8_t* src, uint8_t* dst, int width); 257 }; 258 259 }; // namespace camera3 260 }; // namespace android 261 262 #endif //ANDROID_SERVERS_CAMERA_CAMERA3_HEIC_COMPOSITE_STREAM_H 263