1 /*
2  * Copyright (C) 2018 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #ifndef ANDROID_HARDWARE_CAMERA_DEVICE_V3_4_EXTCAMERADEVICE3SESSION_H
18 #define ANDROID_HARDWARE_CAMERA_DEVICE_V3_4_EXTCAMERADEVICE3SESSION_H
19 
20 #include <android/hardware/camera/device/3.2/ICameraDevice.h>
21 #include <android/hardware/camera/device/3.4/ICameraDeviceSession.h>
22 #include <fmq/MessageQueue.h>
23 #include <hidl/MQDescriptor.h>
24 #include <hidl/Status.h>
25 #include <include/convert.h>
26 #include <chrono>
27 #include <condition_variable>
28 #include <list>
29 #include <unordered_map>
30 #include <unordered_set>
31 #include "CameraMetadata.h"
32 #include "HandleImporter.h"
33 #include "Exif.h"
34 #include "utils/KeyedVector.h"
35 #include "utils/Mutex.h"
36 #include "utils/Thread.h"
37 #include "android-base/unique_fd.h"
38 #include "ExternalCameraUtils.h"
39 
40 namespace android {
41 namespace hardware {
42 namespace camera {
43 namespace device {
44 namespace V3_4 {
45 namespace implementation {
46 
47 using ::android::hardware::camera::device::V3_2::BufferCache;
48 using ::android::hardware::camera::device::V3_2::BufferStatus;
49 using ::android::hardware::camera::device::V3_2::CameraMetadata;
50 using ::android::hardware::camera::device::V3_2::CaptureRequest;
51 using ::android::hardware::camera::device::V3_2::CaptureResult;
52 using ::android::hardware::camera::device::V3_2::ErrorCode;
53 using ::android::hardware::camera::device::V3_2::ICameraDeviceCallback;
54 using ::android::hardware::camera::device::V3_2::MsgType;
55 using ::android::hardware::camera::device::V3_2::NotifyMsg;
56 using ::android::hardware::camera::device::V3_2::RequestTemplate;
57 using ::android::hardware::camera::device::V3_2::Stream;
58 using ::android::hardware::camera::device::V3_4::StreamConfiguration;
59 using ::android::hardware::camera::device::V3_2::StreamConfigurationMode;
60 using ::android::hardware::camera::device::V3_2::StreamRotation;
61 using ::android::hardware::camera::device::V3_2::StreamType;
62 using ::android::hardware::camera::device::V3_2::DataspaceFlags;
63 using ::android::hardware::camera::device::V3_2::CameraBlob;
64 using ::android::hardware::camera::device::V3_2::CameraBlobId;
65 using ::android::hardware::camera::device::V3_4::HalStreamConfiguration;
66 using ::android::hardware::camera::device::V3_4::ICameraDeviceSession;
67 using ::android::hardware::camera::common::V1_0::Status;
68 using ::android::hardware::camera::common::V1_0::helper::HandleImporter;
69 using ::android::hardware::camera::common::V1_0::helper::ExifUtils;
70 using ::android::hardware::camera::external::common::ExternalCameraConfig;
71 using ::android::hardware::camera::external::common::Size;
72 using ::android::hardware::camera::external::common::SizeHasher;
73 using ::android::hardware::graphics::common::V1_0::BufferUsage;
74 using ::android::hardware::graphics::common::V1_0::Dataspace;
75 using ::android::hardware::graphics::common::V1_0::PixelFormat;
76 using ::android::hardware::kSynchronizedReadWrite;
77 using ::android::hardware::MessageQueue;
78 using ::android::hardware::MQDescriptorSync;
79 using ::android::hardware::Return;
80 using ::android::hardware::Void;
81 using ::android::hardware::hidl_vec;
82 using ::android::hardware::hidl_string;
83 using ::android::sp;
84 using ::android::Mutex;
85 using ::android::base::unique_fd;
86 
87 struct ExternalCameraDeviceSession : public virtual RefBase {
88 
89     ExternalCameraDeviceSession(const sp<ICameraDeviceCallback>&,
90             const ExternalCameraConfig& cfg,
91             const std::vector<SupportedV4L2Format>& sortedFormats,
92             const CroppingType& croppingType,
93             const common::V1_0::helper::CameraMetadata& chars,
94             const std::string& cameraId,
95             unique_fd v4l2Fd);
96     virtual ~ExternalCameraDeviceSession();
97     // Call by CameraDevice to dump active device states
98     void dumpState(const native_handle_t*);
99     // Caller must use this method to check if CameraDeviceSession ctor failed
100     bool isInitFailed();
101     bool isClosed();
102 
103     // Retrieve the HIDL interface, split into its own class to avoid inheritance issues when
104     // dealing with minor version revs and simultaneous implementation and interface inheritance
getInterfaceExternalCameraDeviceSession105     virtual sp<ICameraDeviceSession> getInterface() {
106         return new TrampolineSessionInterface_3_4(this);
107     }
108 
109     static const int kMaxProcessedStream = 2;
110     static const int kMaxStallStream = 1;
111     static const uint32_t kMaxBytesPerPixel = 2;
112 
113 protected:
114 
115     // Methods from ::android::hardware::camera::device::V3_2::ICameraDeviceSession follow
116 
117     Return<void> constructDefaultRequestSettings(
118             RequestTemplate,
119             ICameraDeviceSession::constructDefaultRequestSettings_cb _hidl_cb);
120 
121     Return<void> configureStreams(
122             const V3_2::StreamConfiguration&,
123             ICameraDeviceSession::configureStreams_cb);
124 
125     Return<void> getCaptureRequestMetadataQueue(
126         ICameraDeviceSession::getCaptureRequestMetadataQueue_cb);
127 
128     Return<void> getCaptureResultMetadataQueue(
129         ICameraDeviceSession::getCaptureResultMetadataQueue_cb);
130 
131     Return<void> processCaptureRequest(
132             const hidl_vec<CaptureRequest>&,
133             const hidl_vec<BufferCache>&,
134             ICameraDeviceSession::processCaptureRequest_cb);
135 
136     Return<Status> flush();
137     Return<void> close(bool callerIsDtor = false);
138 
139     Return<void> configureStreams_3_3(
140             const V3_2::StreamConfiguration&,
141             ICameraDeviceSession::configureStreams_3_3_cb);
142 
143     Return<void> configureStreams_3_4(
144             const V3_4::StreamConfiguration& requestedConfiguration,
145             ICameraDeviceSession::configureStreams_3_4_cb _hidl_cb);
146 
147     Return<void> processCaptureRequest_3_4(
148             const hidl_vec<V3_4::CaptureRequest>& requests,
149             const hidl_vec<V3_2::BufferCache>& cachesToRemove,
150             ICameraDeviceSession::processCaptureRequest_3_4_cb _hidl_cb);
151 
152 protected:
153     struct HalStreamBuffer {
154         int32_t streamId;
155         uint64_t bufferId;
156         uint32_t width;
157         uint32_t height;
158         PixelFormat format;
159         V3_2::BufferUsageFlags usage;
160         buffer_handle_t* bufPtr;
161         int acquireFence;
162         bool fenceTimeout;
163     };
164 
165     struct HalRequest {
166         uint32_t frameNumber;
167         common::V1_0::helper::CameraMetadata setting;
168         sp<V4L2Frame> frameIn;
169         nsecs_t shutterTs;
170         std::vector<HalStreamBuffer> buffers;
171     };
172 
173     static const uint64_t BUFFER_ID_NO_BUFFER = 0;
174 
175     Status constructDefaultRequestSettingsRaw(RequestTemplate type,
176             V3_2::CameraMetadata *outMetadata);
177 
178     bool initialize();
179     // To init/close different version of output thread
180     virtual void initOutputThread();
181     virtual void closeOutputThread();
182     void closeOutputThreadImpl();
183 
184     Status initStatus() const;
185     status_t initDefaultRequests();
186     status_t fillCaptureResult(common::V1_0::helper::CameraMetadata& md, nsecs_t timestamp);
187     Status configureStreams(const V3_2::StreamConfiguration&,
188             V3_3::HalStreamConfiguration* out,
189             // Only filled by configureStreams_3_4, and only one blob stream supported
190             uint32_t blobBufferSize = 0);
191     // fps = 0.0 means default, which is
192     // slowest fps that is at least 30, or fastest fps if 30 is not supported
193     int configureV4l2StreamLocked(const SupportedV4L2Format& fmt, double fps = 0.0);
194     int v4l2StreamOffLocked();
195     int setV4l2FpsLocked(double fps);
196     static Status isStreamCombinationSupported(const V3_2::StreamConfiguration& config,
197             const std::vector<SupportedV4L2Format>& supportedFormats,
198             const ExternalCameraConfig& devCfg);
199 
200     // TODO: change to unique_ptr for better tracking
201     sp<V4L2Frame> dequeueV4l2FrameLocked(/*out*/nsecs_t* shutterTs); // Called with mLock hold
202     void enqueueV4l2Frame(const sp<V4L2Frame>&);
203 
204     // Check if input Stream is one of supported stream setting on this device
205     static bool isSupported(const Stream& stream,
206             const std::vector<SupportedV4L2Format>& supportedFormats,
207             const ExternalCameraConfig& cfg);
208 
209     // Validate and import request's output buffers and acquire fence
210     virtual Status importRequestLocked(
211             const CaptureRequest& request,
212             hidl_vec<buffer_handle_t*>& allBufPtrs,
213             hidl_vec<int>& allFences);
214 
215     Status importRequestLockedImpl(
216             const CaptureRequest& request,
217             hidl_vec<buffer_handle_t*>& allBufPtrs,
218             hidl_vec<int>& allFences,
219             // Optional argument for ICameraDeviceSession@3.5 impl
220             bool allowEmptyBuf = false);
221 
222     Status importBuffer(int32_t streamId,
223             uint64_t bufId, buffer_handle_t buf,
224             /*out*/buffer_handle_t** outBufPtr,
225             bool allowEmptyBuf);
226 
227     Status importBufferLocked(int32_t streamId,
228             uint64_t bufId, buffer_handle_t buf,
229             /*out*/buffer_handle_t** outBufPtr,
230             bool allowEmptyBuf);
231 
232     static void cleanupInflightFences(
233             hidl_vec<int>& allFences, size_t numFences);
234     void cleanupBuffersLocked(int id);
235     void updateBufferCaches(const hidl_vec<BufferCache>& cachesToRemove);
236 
237     Status processOneCaptureRequest(const CaptureRequest& request);
238 
239     Status processCaptureResult(std::shared_ptr<HalRequest>&);
240     Status processCaptureRequestError(const std::shared_ptr<HalRequest>&);
241     void notifyShutter(uint32_t frameNumber, nsecs_t shutterTs);
242     void notifyError(uint32_t frameNumber, int32_t streamId, ErrorCode ec);
243     void invokeProcessCaptureResultCallback(
244             hidl_vec<CaptureResult> &results, bool tryWriteFmq);
245     static void freeReleaseFences(hidl_vec<CaptureResult>&);
246 
247     Size getMaxJpegResolution() const;
248     Size getMaxThumbResolution() const;
249 
250     ssize_t getJpegBufferSize(uint32_t width, uint32_t height) const;
251 
252     int waitForV4L2BufferReturnLocked(std::unique_lock<std::mutex>& lk);
253 
254     class OutputThread : public android::Thread {
255     public:
256         OutputThread(wp<ExternalCameraDeviceSession> parent, CroppingType);
257         virtual ~OutputThread();
258 
259         Status allocateIntermediateBuffers(
260                 const Size& v4lSize, const Size& thumbSize,
261                 const hidl_vec<Stream>& streams,
262                 uint32_t blobBufferSize);
263         Status submitRequest(const std::shared_ptr<HalRequest>&);
264         void flush();
265         void dump(int fd);
266         virtual bool threadLoop() override;
267 
268         void setExifMakeModel(const std::string& make, const std::string& model);
269 
270     protected:
271         // Methods to request output buffer in parallel
272         // No-op for device@3.4. Implemented in device@3.5
requestBufferStartExternalCameraDeviceSession273         virtual int requestBufferStart(const std::vector<HalStreamBuffer>&) { return 0; }
waitForBufferRequestDoneExternalCameraDeviceSession274         virtual int waitForBufferRequestDone(
275                 /*out*/std::vector<HalStreamBuffer>*) { return 0; }
276 
277         static const uint32_t FLEX_YUV_GENERIC = static_cast<uint32_t>('F') |
278                 static_cast<uint32_t>('L') << 8 | static_cast<uint32_t>('E') << 16 |
279                 static_cast<uint32_t>('X') << 24;
280         // returns FLEX_YUV_GENERIC for formats other than YV12/YU12/NV12/NV21
281         static uint32_t getFourCcFromLayout(const YCbCrLayout&);
282         static int getCropRect(
283                 CroppingType ct, const Size& inSize, const Size& outSize, IMapper::Rect* out);
284 
285         static const int kFlushWaitTimeoutSec = 3; // 3 sec
286         static const int kReqWaitTimeoutMs = 33;   // 33ms
287         static const int kReqWaitTimesMax = 90;    // 33ms * 90 ~= 3 sec
288 
289         void waitForNextRequest(std::shared_ptr<HalRequest>* out);
290         void signalRequestDone();
291 
292         int cropAndScaleLocked(
293                 sp<AllocatedFrame>& in, const Size& outSize,
294                 YCbCrLayout* out);
295 
296         int cropAndScaleThumbLocked(
297                 sp<AllocatedFrame>& in, const Size& outSize,
298                 YCbCrLayout* out);
299 
300         int formatConvertLocked(const YCbCrLayout& in, const YCbCrLayout& out,
301                 Size sz, uint32_t format);
302 
303         static int encodeJpegYU12(const Size &inSz,
304                 const YCbCrLayout& inLayout, int jpegQuality,
305                 const void *app1Buffer, size_t app1Size,
306                 void *out, size_t maxOutSize,
307                 size_t &actualCodeSize);
308 
309         int createJpegLocked(HalStreamBuffer &halBuf, const std::shared_ptr<HalRequest>& req);
310 
311         const wp<ExternalCameraDeviceSession> mParent;
312         const CroppingType mCroppingType;
313 
314         mutable std::mutex mRequestListLock;      // Protect acccess to mRequestList,
315                                                   // mProcessingRequest and mProcessingFrameNumer
316         std::condition_variable mRequestCond;     // signaled when a new request is submitted
317         std::condition_variable mRequestDoneCond; // signaled when a request is done processing
318         std::list<std::shared_ptr<HalRequest>> mRequestList;
319         bool mProcessingRequest = false;
320         uint32_t mProcessingFrameNumer = 0;
321 
322         // V4L2 frameIn
323         // (MJPG decode)-> mYu12Frame
324         // (Scale)-> mScaledYu12Frames
325         // (Format convert) -> output gralloc frames
326         mutable std::mutex mBufferLock; // Protect access to intermediate buffers
327         sp<AllocatedFrame> mYu12Frame;
328         sp<AllocatedFrame> mYu12ThumbFrame;
329         std::unordered_map<Size, sp<AllocatedFrame>, SizeHasher> mIntermediateBuffers;
330         std::unordered_map<Size, sp<AllocatedFrame>, SizeHasher> mScaledYu12Frames;
331         YCbCrLayout mYu12FrameLayout;
332         YCbCrLayout mYu12ThumbFrameLayout;
333         uint32_t mBlobBufferSize = 0; // 0 -> HAL derive buffer size, else: use given size
334 
335         std::string mExifMake;
336         std::string mExifModel;
337     };
338 
339     // Protect (most of) HIDL interface methods from synchronized-entering
340     mutable Mutex mInterfaceLock;
341 
342     mutable Mutex mLock; // Protect all private members except otherwise noted
343     const sp<ICameraDeviceCallback> mCallback;
344     const ExternalCameraConfig& mCfg;
345     const common::V1_0::helper::CameraMetadata mCameraCharacteristics;
346     const std::vector<SupportedV4L2Format> mSupportedFormats;
347     const CroppingType mCroppingType;
348     const std::string& mCameraId;
349 
350     // Not protected by mLock, this is almost a const.
351     // Setup in constructor, reset in close() after OutputThread is joined
352     unique_fd mV4l2Fd;
353 
354     // device is closed either
355     //    - closed by user
356     //    - init failed
357     //    - camera disconnected
358     bool mClosed = false;
359     bool mInitialized = false;
360     bool mInitFail = false;
361     bool mFirstRequest = false;
362     common::V1_0::helper::CameraMetadata mLatestReqSetting;
363 
364     bool mV4l2Streaming = false;
365     SupportedV4L2Format mV4l2StreamingFmt;
366     double mV4l2StreamingFps = 0.0;
367     size_t mV4L2BufferCount = 0;
368 
369     static const int kBufferWaitTimeoutSec = 3; // TODO: handle long exposure (or not allowing)
370     std::mutex mV4l2BufferLock; // protect the buffer count and condition below
371     std::condition_variable mV4L2BufferReturned;
372     size_t mNumDequeuedV4l2Buffers = 0;
373     uint32_t mMaxV4L2BufferSize = 0;
374 
375     // Not protected by mLock (but might be used when mLock is locked)
376     sp<OutputThread> mOutputThread;
377 
378     // Stream ID -> Camera3Stream cache
379     std::unordered_map<int, Stream> mStreamMap;
380 
381     std::mutex mInflightFramesLock; // protect mInflightFrames
382     std::unordered_set<uint32_t>  mInflightFrames;
383 
384     // buffers currently circulating between HAL and camera service
385     // key: bufferId sent via HIDL interface
386     // value: imported buffer_handle_t
387     // Buffer will be imported during processCaptureRequest and will be freed
388     // when the its stream is deleted or camera device session is closed
389     typedef std::unordered_map<uint64_t, buffer_handle_t> CirculatingBuffers;
390     // Stream ID -> circulating buffers map
391     std::map<int, CirculatingBuffers> mCirculatingBuffers;
392     // Protect mCirculatingBuffers, must not lock mLock after acquiring this lock
393     mutable Mutex mCbsLock;
394 
395     std::mutex mAfTriggerLock; // protect mAfTrigger
396     bool mAfTrigger = false;
397 
398     static HandleImporter sHandleImporter;
399 
400     /* Beginning of members not changed after initialize() */
401     using RequestMetadataQueue = MessageQueue<uint8_t, kSynchronizedReadWrite>;
402     std::unique_ptr<RequestMetadataQueue> mRequestMetadataQueue;
403     using ResultMetadataQueue = MessageQueue<uint8_t, kSynchronizedReadWrite>;
404     std::shared_ptr<ResultMetadataQueue> mResultMetadataQueue;
405 
406     // Protect against invokeProcessCaptureResultCallback()
407     Mutex mProcessCaptureResultLock;
408 
409     std::unordered_map<RequestTemplate, CameraMetadata> mDefaultRequests;
410 
411     const Size mMaxThumbResolution;
412     const Size mMaxJpegResolution;
413     /* End of members not changed after initialize() */
414 
415 private:
416 
417     struct TrampolineSessionInterface_3_4 : public ICameraDeviceSession {
TrampolineSessionInterface_3_4ExternalCameraDeviceSession::TrampolineSessionInterface_3_4418         TrampolineSessionInterface_3_4(sp<ExternalCameraDeviceSession> parent) :
419                 mParent(parent) {}
420 
constructDefaultRequestSettingsExternalCameraDeviceSession::TrampolineSessionInterface_3_4421         virtual Return<void> constructDefaultRequestSettings(
422                 RequestTemplate type,
423                 V3_3::ICameraDeviceSession::constructDefaultRequestSettings_cb _hidl_cb) override {
424             return mParent->constructDefaultRequestSettings(type, _hidl_cb);
425         }
426 
configureStreamsExternalCameraDeviceSession::TrampolineSessionInterface_3_4427         virtual Return<void> configureStreams(
428                 const V3_2::StreamConfiguration& requestedConfiguration,
429                 V3_3::ICameraDeviceSession::configureStreams_cb _hidl_cb) override {
430             return mParent->configureStreams(requestedConfiguration, _hidl_cb);
431         }
432 
processCaptureRequestExternalCameraDeviceSession::TrampolineSessionInterface_3_4433         virtual Return<void> processCaptureRequest(const hidl_vec<V3_2::CaptureRequest>& requests,
434                 const hidl_vec<V3_2::BufferCache>& cachesToRemove,
435                 V3_3::ICameraDeviceSession::processCaptureRequest_cb _hidl_cb) override {
436             return mParent->processCaptureRequest(requests, cachesToRemove, _hidl_cb);
437         }
438 
getCaptureRequestMetadataQueueExternalCameraDeviceSession::TrampolineSessionInterface_3_4439         virtual Return<void> getCaptureRequestMetadataQueue(
440                 V3_3::ICameraDeviceSession::getCaptureRequestMetadataQueue_cb _hidl_cb) override  {
441             return mParent->getCaptureRequestMetadataQueue(_hidl_cb);
442         }
443 
getCaptureResultMetadataQueueExternalCameraDeviceSession::TrampolineSessionInterface_3_4444         virtual Return<void> getCaptureResultMetadataQueue(
445                 V3_3::ICameraDeviceSession::getCaptureResultMetadataQueue_cb _hidl_cb) override  {
446             return mParent->getCaptureResultMetadataQueue(_hidl_cb);
447         }
448 
flushExternalCameraDeviceSession::TrampolineSessionInterface_3_4449         virtual Return<Status> flush() override {
450             return mParent->flush();
451         }
452 
closeExternalCameraDeviceSession::TrampolineSessionInterface_3_4453         virtual Return<void> close() override {
454             return mParent->close();
455         }
456 
configureStreams_3_3ExternalCameraDeviceSession::TrampolineSessionInterface_3_4457         virtual Return<void> configureStreams_3_3(
458                 const V3_2::StreamConfiguration& requestedConfiguration,
459                 configureStreams_3_3_cb _hidl_cb) override {
460             return mParent->configureStreams_3_3(requestedConfiguration, _hidl_cb);
461         }
462 
configureStreams_3_4ExternalCameraDeviceSession::TrampolineSessionInterface_3_4463         virtual Return<void> configureStreams_3_4(
464                 const V3_4::StreamConfiguration& requestedConfiguration,
465                 configureStreams_3_4_cb _hidl_cb) override {
466             return mParent->configureStreams_3_4(requestedConfiguration, _hidl_cb);
467         }
468 
processCaptureRequest_3_4ExternalCameraDeviceSession::TrampolineSessionInterface_3_4469         virtual Return<void> processCaptureRequest_3_4(const hidl_vec<V3_4::CaptureRequest>& requests,
470                 const hidl_vec<V3_2::BufferCache>& cachesToRemove,
471                 ICameraDeviceSession::processCaptureRequest_3_4_cb _hidl_cb) override {
472             return mParent->processCaptureRequest_3_4(requests, cachesToRemove, _hidl_cb);
473         }
474 
475     private:
476         sp<ExternalCameraDeviceSession> mParent;
477     };
478 };
479 
480 }  // namespace implementation
481 }  // namespace V3_4
482 }  // namespace device
483 }  // namespace camera
484 }  // namespace hardware
485 }  // namespace android
486 
487 #endif  // ANDROID_HARDWARE_CAMERA_DEVICE_V3_4_EXTCAMERADEVICE3SESSION_H
488