1 /*
2  * Copyright (C) 2015 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 #ifndef _ACAMERA_DEVICE_H
17 #define _ACAMERA_DEVICE_H
18 
19 #include <memory>
20 #include <map>
21 #include <set>
22 #include <atomic>
23 #include <utility>
24 #include <vector>
25 #include <utils/StrongPointer.h>
26 #include <utils/Mutex.h>
27 #include <utils/String8.h>
28 #include <utils/List.h>
29 #include <utils/Vector.h>
30 
31 #include <android/hardware/camera2/BnCameraDeviceCallbacks.h>
32 #include <android/hardware/camera2/ICameraDeviceUser.h>
33 #include <media/stagefright/foundation/ALooper.h>
34 #include <media/stagefright/foundation/AHandler.h>
35 #include <media/stagefright/foundation/AMessage.h>
36 #include <camera/CaptureResult.h>
37 #include <camera/camera2/OutputConfiguration.h>
38 #include <camera/camera2/SessionConfiguration.h>
39 #include <camera/camera2/CaptureRequest.h>
40 
41 #include <camera/NdkCameraManager.h>
42 #include <camera/NdkCameraCaptureSession.h>
43 
44 #include "ACameraMetadata.h"
45 
46 namespace android {
47 namespace acam {
48 
49 // Wrap ACameraCaptureFailure so it can be ref-counted
50 struct CameraCaptureFailure : public RefBase, public ACameraCaptureFailure {};
51 
52 // Wrap PhysicalCaptureResultInfo so that it can be ref-counted
53 struct ACameraPhysicalCaptureResultInfo: public RefBase {
ACameraPhysicalCaptureResultInfoACameraPhysicalCaptureResultInfo54     ACameraPhysicalCaptureResultInfo(const std::vector<PhysicalCaptureResultInfo>& info,
55             int64_t frameNumber) :
56         mPhysicalResultInfo(info), mFrameNumber(frameNumber) {}
57 
58     std::vector<PhysicalCaptureResultInfo> mPhysicalResultInfo;
59     int64_t mFrameNumber;
60 };
61 
62 class CameraDevice final : public RefBase {
63   public:
64     CameraDevice(const char* id, ACameraDevice_StateCallbacks* cb,
65                   sp<ACameraMetadata> chars,
66                   ACameraDevice* wrapper);
67     ~CameraDevice();
68 
getId()69     inline const char* getId() const { return mCameraId.string(); }
70 
71     camera_status_t createCaptureRequest(
72             ACameraDevice_request_template templateId,
73             const ACameraIdList* physicalIdList,
74             ACaptureRequest** request) const;
75 
76     camera_status_t createCaptureSession(
77             const ACaptureSessionOutputContainer*       outputs,
78             const ACaptureRequest* sessionParameters,
79             const ACameraCaptureSession_stateCallbacks* callbacks,
80             /*out*/ACameraCaptureSession** session);
81 
82     camera_status_t isSessionConfigurationSupported(
83             const ACaptureSessionOutputContainer* sessionOutputContainer) const;
84 
85     // Callbacks from camera service
86     class ServiceCallback : public hardware::camera2::BnCameraDeviceCallbacks {
87       public:
ServiceCallback(CameraDevice * device)88         explicit ServiceCallback(CameraDevice* device) : mDevice(device) {}
89         binder::Status onDeviceError(int32_t errorCode,
90                            const CaptureResultExtras& resultExtras) override;
91         binder::Status onDeviceIdle() override;
92         binder::Status onCaptureStarted(const CaptureResultExtras& resultExtras,
93                               int64_t timestamp) override;
94         binder::Status onResultReceived(const CameraMetadata& metadata,
95                               const CaptureResultExtras& resultExtras,
96                               const std::vector<PhysicalCaptureResultInfo>& physicalResultInfos) override;
97         binder::Status onPrepared(int streamId) override;
98         binder::Status onRequestQueueEmpty() override;
99         binder::Status onRepeatingRequestError(int64_t lastFrameNumber,
100                 int32_t stoppedSequenceId) override;
101       private:
102         const wp<CameraDevice> mDevice;
103     };
getServiceCallback()104     inline sp<hardware::camera2::ICameraDeviceCallbacks> getServiceCallback() {
105         return mServiceCallback;
106     };
107 
108     // Camera device is only functional after remote being set
109     void setRemoteDevice(sp<hardware::camera2::ICameraDeviceUser> remote);
110 
getWrapper()111     inline ACameraDevice* getWrapper() const { return mWrapper; };
112 
113     // Stop the looper thread and unregister the handler
114     void stopLooperAndDisconnect();
115 
116   private:
117     friend ACameraCaptureSession;
118     camera_status_t checkCameraClosedOrErrorLocked() const;
119 
120     // device goes into fatal error state after this
121     void setCameraDeviceErrorLocked(camera_status_t error);
122 
123     void disconnectLocked(sp<ACameraCaptureSession>& session); // disconnect from camera service
124 
125     camera_status_t stopRepeatingLocked();
126 
127     camera_status_t flushLocked(ACameraCaptureSession*);
128 
129     camera_status_t waitUntilIdleLocked();
130 
131 
132     template<class T>
133     camera_status_t captureLocked(sp<ACameraCaptureSession> session,
134             /*optional*/T* cbs,
135             int numRequests, ACaptureRequest** requests,
136             /*optional*/int* captureSequenceId);
137 
138     template<class T>
139     camera_status_t setRepeatingRequestsLocked(sp<ACameraCaptureSession> session,
140             /*optional*/T* cbs,
141             int numRequests, ACaptureRequest** requests,
142             /*optional*/int* captureSequenceId);
143 
144     template<class T>
145     camera_status_t submitRequestsLocked(
146             sp<ACameraCaptureSession> session,
147             /*optional*/T* cbs,
148             int numRequests, ACaptureRequest** requests,
149             /*out*/int* captureSequenceId,
150             bool isRepeating);
151 
152     camera_status_t updateOutputConfigurationLocked(ACaptureSessionOutput *output);
153 
154     camera_status_t allocateCaptureRequest(
155             const ACaptureRequest* request, sp<CaptureRequest>& outReq);
156 
157     static ACaptureRequest* allocateACaptureRequest(sp<CaptureRequest>& req,
158             const std::string& deviceId);
159     static void freeACaptureRequest(ACaptureRequest*);
160 
161     // only For session to hold device lock
162     // Always grab device lock before grabbing session lock
lockDeviceForSessionOps()163     void lockDeviceForSessionOps() const { mDeviceLock.lock(); };
unlockDevice()164     void unlockDevice() const { mDeviceLock.unlock(); };
165 
166     // For capture session to notify its end of life
167     void notifySessionEndOfLifeLocked(ACameraCaptureSession* session);
168 
169     camera_status_t configureStreamsLocked(const ACaptureSessionOutputContainer* outputs,
170            const ACaptureRequest* sessionParameters);
171 
172     // Input message will be posted and cleared after this returns
173     void postSessionMsgAndCleanup(sp<AMessage>& msg);
174 
175     static camera_status_t getIGBPfromAnw(
176             ANativeWindow* anw, sp<IGraphicBufferProducer>& out);
177 
178     static camera_status_t getSurfaceFromANativeWindow(
179             ANativeWindow* anw, sp<Surface>& out);
180 
181     mutable Mutex mDeviceLock;
182     const String8 mCameraId;                          // Camera ID
183     const ACameraDevice_StateCallbacks mAppCallbacks; // Callback to app
184     const sp<ACameraMetadata> mChars;                 // Camera characteristics
185     const sp<ServiceCallback> mServiceCallback;
186     ACameraDevice* mWrapper;
187 
188     // stream id -> pair of (ANW* from application, OutputConfiguration used for camera service)
189     std::map<int, std::pair<ANativeWindow*, OutputConfiguration>> mConfiguredOutputs;
190 
191     // TODO: maybe a bool will suffice for synchronous implementation?
192     std::atomic_bool mClosing;
isClosed()193     inline bool isClosed() { return mClosing; }
194 
195     bool mInError = false;
196     camera_status_t mError = ACAMERA_OK;
197     void onCaptureErrorLocked(
198             int32_t errorCode,
199             const CaptureResultExtras& resultExtras);
200 
201     bool mIdle = true;
202     // This will avoid a busy session being deleted before it's back to idle state
203     sp<ACameraCaptureSession> mBusySession;
204 
205     sp<hardware::camera2::ICameraDeviceUser> mRemote;
206 
207     // Looper thread to handle callback to app
208     sp<ALooper> mCbLooper;
209     // definition of handler and message
210     enum {
211         // Device state callbacks
212         kWhatOnDisconnected,   // onDisconnected
213         kWhatOnError,          // onError
214         // Session state callbacks
215         kWhatSessionStateCb,   // onReady, onActive
216         // Capture callbacks
217         kWhatCaptureStart,     // onCaptureStarted
218         kWhatCaptureResult,    // onCaptureProgressed, onCaptureCompleted
219         kWhatLogicalCaptureResult, // onLogicalCameraCaptureCompleted
220         kWhatCaptureFail,      // onCaptureFailed
221         kWhatLogicalCaptureFail, // onLogicalCameraCaptureFailed
222         kWhatCaptureSeqEnd,    // onCaptureSequenceCompleted
223         kWhatCaptureSeqAbort,  // onCaptureSequenceAborted
224         kWhatCaptureBufferLost,// onCaptureBufferLost
225         // Internal cleanup
226         kWhatCleanUpSessions   // Cleanup cached sp<ACameraCaptureSession>
227     };
228     static const char* kContextKey;
229     static const char* kDeviceKey;
230     static const char* kErrorCodeKey;
231     static const char* kCallbackFpKey;
232     static const char* kSessionSpKey;
233     static const char* kCaptureRequestKey;
234     static const char* kTimeStampKey;
235     static const char* kCaptureResultKey;
236     static const char* kPhysicalCaptureResultKey;
237     static const char* kCaptureFailureKey;
238     static const char* kSequenceIdKey;
239     static const char* kFrameNumberKey;
240     static const char* kAnwKey;
241     static const char* kFailingPhysicalCameraId;
242 
243     class CallbackHandler : public AHandler {
244       public:
245         explicit CallbackHandler(const char* id);
246         void onMessageReceived(const sp<AMessage> &msg) override;
247 
248       private:
249         std::string mId;
250         // This handler will cache all capture session sp until kWhatCleanUpSessions
251         // is processed. This is used to guarantee the last session reference is always
252         // being removed in callback thread without holding camera device lock
253         Vector<sp<ACameraCaptureSession>> mCachedSessions;
254     };
255     sp<CallbackHandler> mHandler;
256 
257     /***********************************
258      * Capture session related members *
259      ***********************************/
260     // The current active session
261     wp<ACameraCaptureSession> mCurrentSession;
262     bool mFlushing = false;
263 
264     int mNextSessionId = 0;
265     // TODO: might need another looper/handler to handle callbacks from service
266 
267     static const int REQUEST_ID_NONE = -1;
268     int mRepeatingSequenceId = REQUEST_ID_NONE;
269 
270     // sequence id -> last frame number map
271     std::map<int, int64_t> mSequenceLastFrameNumberMap;
272 
273     struct CallbackHolder {
274         CallbackHolder(sp<ACameraCaptureSession>          session,
275                        const Vector<sp<CaptureRequest> >& requests,
276                        bool                               isRepeating,
277                        ACameraCaptureSession_captureCallbacks* cbs);
278         CallbackHolder(sp<ACameraCaptureSession>          session,
279                        const Vector<sp<CaptureRequest> >& requests,
280                        bool                               isRepeating,
281                        ACameraCaptureSession_logicalCamera_captureCallbacks* lcbs);
282 
283         template <class T>
initCaptureCallbacksCallbackHolder284         void initCaptureCallbacks(T* cbs) {
285             mContext = nullptr;
286             mOnCaptureStarted = nullptr;
287             mOnCaptureProgressed = nullptr;
288             mOnCaptureCompleted = nullptr;
289             mOnLogicalCameraCaptureCompleted = nullptr;
290             mOnLogicalCameraCaptureFailed = nullptr;
291             mOnCaptureFailed = nullptr;
292             mOnCaptureSequenceCompleted = nullptr;
293             mOnCaptureSequenceAborted = nullptr;
294             mOnCaptureBufferLost = nullptr;
295             if (cbs != nullptr) {
296                 mContext = cbs->context;
297                 mOnCaptureStarted = cbs->onCaptureStarted;
298                 mOnCaptureProgressed = cbs->onCaptureProgressed;
299                 mOnCaptureSequenceCompleted = cbs->onCaptureSequenceCompleted;
300                 mOnCaptureSequenceAborted = cbs->onCaptureSequenceAborted;
301                 mOnCaptureBufferLost = cbs->onCaptureBufferLost;
302             }
303         }
304         sp<ACameraCaptureSession>   mSession;
305         Vector<sp<CaptureRequest> > mRequests;
306         const bool                  mIsRepeating;
307         const bool                  mIsLogicalCameraCallback;
308 
309         void*                       mContext;
310         ACameraCaptureSession_captureCallback_start mOnCaptureStarted;
311         ACameraCaptureSession_captureCallback_result mOnCaptureProgressed;
312         ACameraCaptureSession_captureCallback_result mOnCaptureCompleted;
313         ACameraCaptureSession_logicalCamera_captureCallback_result mOnLogicalCameraCaptureCompleted;
314         ACameraCaptureSession_logicalCamera_captureCallback_failed mOnLogicalCameraCaptureFailed;
315         ACameraCaptureSession_captureCallback_failed mOnCaptureFailed;
316         ACameraCaptureSession_captureCallback_sequenceEnd mOnCaptureSequenceCompleted;
317         ACameraCaptureSession_captureCallback_sequenceAbort mOnCaptureSequenceAborted;
318         ACameraCaptureSession_captureCallback_bufferLost mOnCaptureBufferLost;
319     };
320     // sequence id -> callbacks map
321     std::map<int, CallbackHolder> mSequenceCallbackMap;
322 
323     static const int64_t NO_FRAMES_CAPTURED = -1;
324     class FrameNumberTracker {
325       public:
326         // TODO: Called in onResultReceived and onCaptureErrorLocked
327         void updateTracker(int64_t frameNumber, bool isError);
getCompletedFrameNumber()328         inline int64_t getCompletedFrameNumber() { return mCompletedFrameNumber; }
329       private:
330         void update();
331         void updateCompletedFrameNumber(int64_t frameNumber);
332 
333         int64_t mCompletedFrameNumber = NO_FRAMES_CAPTURED;
334         List<int64_t> mSkippedFrameNumbers;
335         std::set<int64_t> mFutureErrorSet;
336     };
337     FrameNumberTracker mFrameNumberTracker;
338 
339     void checkRepeatingSequenceCompleteLocked(const int sequenceId, const int64_t lastFrameNumber);
340     void checkAndFireSequenceCompleteLocked();
341 
342     // Misc variables
343     int32_t mShadingMapSize[2];   // const after constructor
344     int32_t mPartialResultCount;  // const after constructor
345     std::vector<std::string> mPhysicalIds; // const after constructor
346 
347 };
348 
349 } // namespace acam;
350 } // namespace android;
351 
352 /**
353  * ACameraDevice opaque struct definition
354  * Leave outside of android namespace because it's NDK struct
355  */
356 struct ACameraDevice {
ACameraDeviceACameraDevice357     ACameraDevice(const char* id, ACameraDevice_StateCallbacks* cb,
358                   sp<ACameraMetadata> chars) :
359             mDevice(new android::acam::CameraDevice(id, cb, chars, this)) {}
360 
361     ~ACameraDevice();
362 
363     /*******************
364      * NDK public APIs *
365      *******************/
getIdACameraDevice366     inline const char* getId() const { return mDevice->getId(); }
367 
createCaptureRequestACameraDevice368     camera_status_t createCaptureRequest(
369             ACameraDevice_request_template templateId,
370             const ACameraIdList* physicalCameraIdList,
371             ACaptureRequest** request) const {
372         return mDevice->createCaptureRequest(templateId, physicalCameraIdList, request);
373     }
374 
createCaptureSessionACameraDevice375     camera_status_t createCaptureSession(
376             const ACaptureSessionOutputContainer*       outputs,
377             const ACaptureRequest* sessionParameters,
378             const ACameraCaptureSession_stateCallbacks* callbacks,
379             /*out*/ACameraCaptureSession** session) {
380         return mDevice->createCaptureSession(outputs, sessionParameters, callbacks, session);
381     }
382 
isSessionConfigurationSupportedACameraDevice383     camera_status_t isSessionConfigurationSupported(
384             const ACaptureSessionOutputContainer* sessionOutputContainer) const {
385         return mDevice->isSessionConfigurationSupported(sessionOutputContainer);
386     }
387 
388     /***********************
389      * Device interal APIs *
390      ***********************/
getServiceCallbackACameraDevice391     inline android::sp<android::hardware::camera2::ICameraDeviceCallbacks> getServiceCallback() {
392         return mDevice->getServiceCallback();
393     };
394 
395     // Camera device is only functional after remote being set
setRemoteDeviceACameraDevice396     inline void setRemoteDevice(android::sp<android::hardware::camera2::ICameraDeviceUser> remote) {
397         mDevice->setRemoteDevice(remote);
398     }
399 
400   private:
401     android::sp<android::acam::CameraDevice> mDevice;
402 };
403 
404 #endif // _ACAMERA_DEVICE_H
405