1 /*
2  * Copyright (C) 2018 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 #ifndef _ACAMERA_DEVICE_H
17 #define _ACAMERA_DEVICE_H
18 
19 #include <memory>
20 #include <map>
21 #include <set>
22 #include <atomic>
23 #include <utility>
24 #include <vector>
25 #include <utils/StrongPointer.h>
26 #include <utils/Mutex.h>
27 #include <utils/List.h>
28 #include <utils/Vector.h>
29 #include <android/frameworks/cameraservice/device/2.0/ICameraDeviceUser.h>
30 #include <android/frameworks/cameraservice/device/2.0/ICameraDeviceCallback.h>
31 #include <android/frameworks/cameraservice/device/2.0/types.h>
32 #include <fmq/MessageQueue.h>
33 #include <media/stagefright/foundation/ALooper.h>
34 #include <media/stagefright/foundation/AHandler.h>
35 #include <media/stagefright/foundation/AMessage.h>
36 
37 #include <camera/NdkCameraManager.h>
38 #include <camera/NdkCameraCaptureSession.h>
39 
40 #include "ACameraMetadata.h"
41 #include "utils.h"
42 
43 namespace android {
44 namespace acam {
45 
46 using ICameraDeviceCallback = frameworks::cameraservice::device::V2_0::ICameraDeviceCallback;
47 using ICameraDeviceUser = frameworks::cameraservice::device::V2_0::ICameraDeviceUser;
48 using CaptureResultExtras = frameworks::cameraservice::device::V2_0::CaptureResultExtras;
49 using PhysicalCaptureResultInfo = frameworks::cameraservice::device::V2_0::PhysicalCaptureResultInfo;
50 using PhysicalCameraSettings = frameworks::cameraservice::device::V2_0::PhysicalCameraSettings;
51 using SubmitInfo = frameworks::cameraservice::device::V2_0::SubmitInfo;
52 using CaptureResultExtras = frameworks::cameraservice::device::V2_0::CaptureResultExtras;
53 using ErrorCode = frameworks::cameraservice::device::V2_0::ErrorCode;
54 using FmqSizeOrMetadata = frameworks::cameraservice::device::V2_0::FmqSizeOrMetadata;
55 using StreamConfigurationMode = frameworks::cameraservice::device::V2_0::StreamConfigurationMode;
56 using Status = frameworks::cameraservice::common::V2_0::Status;
57 using ResultMetadataQueue = hardware::MessageQueue<uint8_t, hardware::kSynchronizedReadWrite>;
58 using RequestMetadataQueue = hardware::MessageQueue<uint8_t, hardware::kSynchronizedReadWrite>;
59 using CameraStatusAndId = frameworks::cameraservice::service::V2_0::CameraStatusAndId;
60 
61 using hardware::hidl_vec;
62 using hardware::hidl_string;
63 using utils::native_handle_ptr_wrapper;
64 using utils::CaptureRequest;
65 using utils::OutputConfigurationWrapper;
66 
67 // Wrap ACameraCaptureFailure so it can be ref-counted
68 struct CameraCaptureFailure : public RefBase, public ACameraCaptureFailure { };
69 
70 // Wrap PhysicalCaptureResultInfo so that it can be ref-counted
71 struct PhysicalCaptureResultInfoLocal {
72     std::string physicalCameraId;
73     CameraMetadata physicalMetadata;
74 };
75 
76 struct ACameraPhysicalCaptureResultInfo: public RefBase {
ACameraPhysicalCaptureResultInfoACameraPhysicalCaptureResultInfo77     ACameraPhysicalCaptureResultInfo(const std::vector<PhysicalCaptureResultInfoLocal>& info,
78             int64_t frameNumber) :
79         mPhysicalResultInfo(info), mFrameNumber(frameNumber) {}
80 
81     std::vector<PhysicalCaptureResultInfoLocal> mPhysicalResultInfo;
82     int64_t mFrameNumber;
83 };
84 
85 class CameraDevice final : public RefBase {
86   public:
87     CameraDevice(const char* id, ACameraDevice_StateCallbacks* cb,
88                   sp<ACameraMetadata> chars,
89                   ACameraDevice* wrapper);
90     ~CameraDevice();
91 
getId()92     inline const char* getId() const { return mCameraId.c_str(); }
93 
94     camera_status_t createCaptureRequest(
95             ACameraDevice_request_template templateId,
96             const ACameraIdList* physicalCameraIdList,
97             ACaptureRequest** request) const;
98 
99     camera_status_t createCaptureSession(
100             const ACaptureSessionOutputContainer*       outputs,
101             const ACaptureRequest* sessionParameters,
102             const ACameraCaptureSession_stateCallbacks* callbacks,
103             /*out*/ACameraCaptureSession** session);
104 
105     camera_status_t isSessionConfigurationSupported(
106             const ACaptureSessionOutputContainer* sessionOutputContainer) const;
107 
108     // Callbacks from camera service
109     class ServiceCallback : public ICameraDeviceCallback {
110       public:
ServiceCallback(CameraDevice * device)111         explicit ServiceCallback(CameraDevice* device) : mDevice(device) {}
112         android::hardware::Return<void> onDeviceError(ErrorCode errorCode,
113                            const CaptureResultExtras& resultExtras) override;
114         android::hardware::Return<void> onDeviceIdle() override;
115         android::hardware::Return<void> onCaptureStarted(const CaptureResultExtras& resultExtras,
116                               uint64_t timestamp) override;
117         android::hardware::Return<void> onResultReceived(const FmqSizeOrMetadata& result,
118                               const CaptureResultExtras& resultExtras,
119                               const hidl_vec<PhysicalCaptureResultInfo>& physicalResultInfos) override;
120         android::hardware::Return<void> onRepeatingRequestError(uint64_t lastFrameNumber,
121                 int32_t stoppedSequenceId) override;
122       private:
123         camera_status_t readOneResultMetadata(const FmqSizeOrMetadata& fmqSizeOrMetadata,
124                 ResultMetadataQueue* metadataQueue, CameraMetadata* metadata);
125         const wp<CameraDevice> mDevice;
126     };
getServiceCallback()127     inline sp<ICameraDeviceCallback> getServiceCallback() {
128         return mServiceCallback;
129     };
130 
131     // Camera device is only functional after remote being set
132     void setRemoteDevice(sp<ICameraDeviceUser> remote);
133 
134     bool setDeviceMetadataQueues();
getWrapper()135     inline ACameraDevice* getWrapper() const { return mWrapper; };
136 
137     // Stop the looper thread and unregister the handler
138     void stopLooperAndDisconnect();
139 
140   private:
141     friend ACameraCaptureSession;
142     friend ACameraDevice;
143 
144     camera_status_t checkCameraClosedOrErrorLocked() const;
145 
146     // device goes into fatal error state after this
147     void setCameraDeviceErrorLocked(camera_status_t error);
148 
149     void disconnectLocked(sp<ACameraCaptureSession>& session); // disconnect from camera service
150 
151     camera_status_t stopRepeatingLocked();
152 
153     camera_status_t flushLocked(ACameraCaptureSession*);
154 
155     camera_status_t waitUntilIdleLocked();
156 
157     template<class T>
158     camera_status_t captureLocked(sp<ACameraCaptureSession> session,
159             /*optional*/T* cbs,
160             int numRequests, ACaptureRequest** requests,
161             /*optional*/int* captureSequenceId);
162 
163     template<class T>
164     camera_status_t setRepeatingRequestsLocked(sp<ACameraCaptureSession> session,
165             /*optional*/T* cbs,
166             int numRequests, ACaptureRequest** requests,
167             /*optional*/int* captureSequenceId);
168 
169     template<class T>
170     camera_status_t submitRequestsLocked(
171             sp<ACameraCaptureSession> session,
172             /*optional*/T* cbs,
173             int numRequests, ACaptureRequest** requests,
174             /*out*/int* captureSequenceId,
175             bool isRepeating);
176 
177     void addRequestSettingsMetadata(ACaptureRequest *aCaptureRequest, sp<CaptureRequest> &req);
178 
179     camera_status_t updateOutputConfigurationLocked(ACaptureSessionOutput *output);
180 
181     // Since this writes to ICameraDeviceUser's fmq, clients must take care that:
182     //   a) This function is called serially.
183     //   b) This function is called in accordance with ICameraDeviceUser.submitRequestList,
184     //      otherwise, the wrong capture request might have the wrong settings
185     //      metadata associated with it.
186     camera_status_t allocateCaptureRequestLocked(
187             const ACaptureRequest* request, sp<CaptureRequest>& outReq);
188     void allocateOneCaptureRequestMetadata(
189             PhysicalCameraSettings& cameraSettings,
190             const std::string& id, const sp<ACameraMetadata>& metadata);
191 
192     static ACaptureRequest* allocateACaptureRequest(sp<CaptureRequest>& req, const char* deviceId);
193     static void freeACaptureRequest(ACaptureRequest*);
194 
195     // only For session to hold device lock
196     // Always grab device lock before grabbing session lock
lockDeviceForSessionOps()197     void lockDeviceForSessionOps() const { mDeviceLock.lock(); };
unlockDevice()198     void unlockDevice() const { mDeviceLock.unlock(); };
199 
200     // For capture session to notify its end of life
201     void notifySessionEndOfLifeLocked(ACameraCaptureSession* session);
202 
203     camera_status_t configureStreamsLocked(const ACaptureSessionOutputContainer* outputs,
204            const ACaptureRequest* sessionParameters);
205 
206     // Input message will be posted and cleared after this returns
207     void postSessionMsgAndCleanup(sp<AMessage>& msg);
208 
209     mutable Mutex mDeviceLock;
210     const hidl_string mCameraId;                          // Camera ID
211     const ACameraDevice_StateCallbacks mAppCallbacks; // Callback to app
212     const sp<ACameraMetadata> mChars;    // Camera characteristics
213     const sp<ServiceCallback> mServiceCallback;
214     ACameraDevice* mWrapper;
215 
216     // stream id -> pair of (ACameraWindowType* from application, OutputConfiguration used for
217     // camera service)
218     std::map<int, std::pair<native_handle_ptr_wrapper, OutputConfigurationWrapper>> mConfiguredOutputs;
219 
220     // TODO: maybe a bool will suffice for synchronous implementation?
221     std::atomic_bool mClosing;
isClosed()222     inline bool isClosed() { return mClosing; }
223 
224     bool mInError = false;
225     camera_status_t mError = ACAMERA_OK;
226     void onCaptureErrorLocked(
227             ErrorCode errorCode,
228             const CaptureResultExtras& resultExtras);
229 
230     bool mIdle = true;
231     // This will avoid a busy session being deleted before it's back to idle state
232     sp<ACameraCaptureSession> mBusySession;
233 
234     sp<ICameraDeviceUser> mRemote;
235 
236     // Looper thread to handle callback to app
237     sp<ALooper> mCbLooper;
238     // definition of handler and message
239     enum {
240         // Device state callbacks
241         kWhatOnDisconnected,   // onDisconnected
242         kWhatOnError,          // onError
243         // Session state callbacks
244         kWhatSessionStateCb,   // onReady, onActive
245         // Capture callbacks
246         kWhatCaptureStart,     // onCaptureStarted
247         kWhatCaptureResult,    // onCaptureProgressed, onCaptureCompleted
248         kWhatLogicalCaptureResult, // onLogicalCameraCaptureCompleted
249         kWhatCaptureFail,      // onCaptureFailed
250         kWhatLogicalCaptureFail, // onLogicalCameraCaptureFailed
251         kWhatCaptureSeqEnd,    // onCaptureSequenceCompleted
252         kWhatCaptureSeqAbort,  // onCaptureSequenceAborted
253         kWhatCaptureBufferLost,// onCaptureBufferLost
254         // Internal cleanup
255         kWhatCleanUpSessions   // Cleanup cached sp<ACameraCaptureSession>
256     };
257     static const char* kContextKey;
258     static const char* kDeviceKey;
259     static const char* kErrorCodeKey;
260     static const char* kCallbackFpKey;
261     static const char* kSessionSpKey;
262     static const char* kCaptureRequestKey;
263     static const char* kTimeStampKey;
264     static const char* kCaptureResultKey;
265     static const char* kPhysicalCaptureResultKey;
266     static const char* kCaptureFailureKey;
267     static const char* kSequenceIdKey;
268     static const char* kFrameNumberKey;
269     static const char* kAnwKey;
270     static const char* kFailingPhysicalCameraId;
271 
272     class CallbackHandler : public AHandler {
273       public:
274         explicit CallbackHandler(const char *id);
275         void onMessageReceived(const sp<AMessage> &msg) override;
276 
277       private:
278         std::string mId;
279         // This handler will cache all capture session sp until kWhatCleanUpSessions
280         // is processed. This is used to guarantee the last session reference is always
281         // being removed in callback thread without holding camera device lock
282         Vector<sp<ACameraCaptureSession>> mCachedSessions;
283     };
284     sp<CallbackHandler> mHandler;
285 
286     /***********************************
287      * Capture session related members *
288      ***********************************/
289     // The current active session
290     wp<ACameraCaptureSession> mCurrentSession;
291     bool mFlushing = false;
292 
293     int mNextSessionId = 0;
294     // TODO: might need another looper/handler to handle callbacks from service
295 
296     static const int REQUEST_ID_NONE = -1;
297     int mRepeatingSequenceId = REQUEST_ID_NONE;
298 
299     // sequence id -> last frame number map
300     std::map<int32_t, int64_t> mSequenceLastFrameNumberMap;
301 
302     struct CallbackHolder {
303         CallbackHolder(sp<ACameraCaptureSession>          session,
304                        const Vector<sp<CaptureRequest>>&  requests,
305                        bool                               isRepeating,
306                        ACameraCaptureSession_captureCallbacks* cbs);
307         CallbackHolder(sp<ACameraCaptureSession>          session,
308                        const Vector<sp<CaptureRequest>>&  requests,
309                        bool                               isRepeating,
310                        ACameraCaptureSession_logicalCamera_captureCallbacks* lcbs);
311 
312         template <class T>
initCaptureCallbacksCallbackHolder313         void initCaptureCallbacks(T* cbs) {
314             mContext = nullptr;
315             mOnCaptureStarted = nullptr;
316             mOnCaptureProgressed = nullptr;
317             mOnCaptureCompleted = nullptr;
318             mOnLogicalCameraCaptureCompleted = nullptr;
319             mOnLogicalCameraCaptureFailed = nullptr;
320             mOnCaptureFailed = nullptr;
321             mOnCaptureSequenceCompleted = nullptr;
322             mOnCaptureSequenceAborted = nullptr;
323             mOnCaptureBufferLost = nullptr;
324             if (cbs != nullptr) {
325                 mContext = cbs->context;
326                 mOnCaptureStarted = cbs->onCaptureStarted;
327                 mOnCaptureProgressed = cbs->onCaptureProgressed;
328                 mOnCaptureSequenceCompleted = cbs->onCaptureSequenceCompleted;
329                 mOnCaptureSequenceAborted = cbs->onCaptureSequenceAborted;
330                 mOnCaptureBufferLost = cbs->onCaptureBufferLost;
331             }
332         }
333 
334         sp<ACameraCaptureSession>   mSession;
335         Vector<sp<CaptureRequest>>  mRequests;
336         const bool                  mIsRepeating;
337         const bool                  mIsLogicalCameraCallback;
338 
339         void*                       mContext;
340         ACameraCaptureSession_captureCallback_start mOnCaptureStarted;
341         ACameraCaptureSession_captureCallback_result mOnCaptureProgressed;
342         ACameraCaptureSession_captureCallback_result mOnCaptureCompleted;
343         ACameraCaptureSession_logicalCamera_captureCallback_result mOnLogicalCameraCaptureCompleted;
344         ACameraCaptureSession_logicalCamera_captureCallback_failed mOnLogicalCameraCaptureFailed;
345         ACameraCaptureSession_captureCallback_failed mOnCaptureFailed;
346         ACameraCaptureSession_captureCallback_sequenceEnd mOnCaptureSequenceCompleted;
347         ACameraCaptureSession_captureCallback_sequenceAbort mOnCaptureSequenceAborted;
348         ACameraCaptureSession_captureCallback_bufferLost mOnCaptureBufferLost;
349     };
350     // sequence id -> callbacks map
351     std::map<int, CallbackHolder> mSequenceCallbackMap;
352 
353     static const int64_t NO_FRAMES_CAPTURED = -1;
354     class FrameNumberTracker {
355       public:
356         // TODO: Called in onResultReceived and onCaptureErrorLocked
357         void updateTracker(int64_t frameNumber, bool isError);
getCompletedFrameNumber()358         inline int64_t getCompletedFrameNumber() { return mCompletedFrameNumber; }
359       private:
360         void update();
361         void updateCompletedFrameNumber(int64_t frameNumber);
362 
363         int64_t mCompletedFrameNumber = NO_FRAMES_CAPTURED;
364         List<int64_t> mSkippedFrameNumbers;
365         std::set<int64_t> mFutureErrorSet;
366     };
367     FrameNumberTracker mFrameNumberTracker;
368 
369     void checkRepeatingSequenceCompleteLocked(const int sequenceId, const int64_t lastFrameNumber);
370     void checkAndFireSequenceCompleteLocked();
371 
372     // Misc variables
373     int32_t mShadingMapSize[2];   // const after constructor
374     int32_t mPartialResultCount;  // const after constructor
375     std::shared_ptr<ResultMetadataQueue> mCaptureRequestMetadataQueue = nullptr;
376     std::shared_ptr<ResultMetadataQueue> mCaptureResultMetadataQueue = nullptr;
377 };
378 
379 } // namespace acam;
380 } // namespace android;
381 
382 /**
383  * ACameraDevice opaque struct definition
384  * Leave outside of android namespace because it's NDK struct
385  */
386 struct ACameraDevice {
ACameraDeviceACameraDevice387     ACameraDevice(const char* id, ACameraDevice_StateCallbacks* cb,
388                   sp<ACameraMetadata> chars) :
389             mDevice(new android::acam::CameraDevice(id, cb, std::move(chars), this)) {}
390 
391     ~ACameraDevice();
392     /*******************
393      * NDK public APIs *
394      *******************/
getIdACameraDevice395     inline const char* getId() const { return mDevice->getId(); }
396 
createCaptureRequestACameraDevice397     camera_status_t createCaptureRequest(
398             ACameraDevice_request_template templateId,
399             const ACameraIdList* physicalCameraIdList,
400             ACaptureRequest** request) const {
401         return mDevice->createCaptureRequest(templateId, physicalCameraIdList, request);
402     }
403 
createCaptureSessionACameraDevice404     camera_status_t createCaptureSession(
405             const ACaptureSessionOutputContainer*       outputs,
406             const ACaptureRequest* sessionParameters,
407             const ACameraCaptureSession_stateCallbacks* callbacks,
408             /*out*/ACameraCaptureSession** session) {
409         return mDevice->createCaptureSession(outputs, sessionParameters, callbacks, session);
410     }
411 
isSessionConfigurationSupportedACameraDevice412     camera_status_t isSessionConfigurationSupported(
413             const ACaptureSessionOutputContainer* sessionOutputContainer) const {
414         return mDevice->isSessionConfigurationSupported(sessionOutputContainer);
415     }
416 
417     /***********************
418      * Device interal APIs *
419      ***********************/
getServiceCallbackACameraDevice420     inline android::sp<android::acam::ICameraDeviceCallback> getServiceCallback() {
421         return mDevice->getServiceCallback();
422     };
423 
424     // Camera device is only functional after remote being set
setRemoteDeviceACameraDevice425     inline void setRemoteDevice(android::sp<android::acam::ICameraDeviceUser> remote) {
426         mDevice->setRemoteDevice(remote);
427     }
setDeviceMetadataQueuesACameraDevice428     inline bool setDeviceMetadataQueues() {
429         return mDevice->setDeviceMetadataQueues();
430     }
431   private:
432     android::sp<android::acam::CameraDevice> mDevice;
433 };
434 
435 #endif // _ACAMERA_DEVICE_H
436