1 /*
2 * Copyright (C) 2012-2018 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #define LOG_TAG "Camera2-CallbackProcessor"
18 #define ATRACE_TAG ATRACE_TAG_CAMERA
19 //#define LOG_NDEBUG 0
20
21 #include <utils/Log.h>
22 #include <utils/Trace.h>
23 #include <gui/Surface.h>
24
25 #include "common/CameraDeviceBase.h"
26 #include "api1/Camera2Client.h"
27 #include "api1/client2/CallbackProcessor.h"
28
29 #define ALIGN(x, mask) ( ((x) + (mask) - 1) & ~((mask) - 1) )
30
31 namespace android {
32 namespace camera2 {
33
CallbackProcessor(sp<Camera2Client> client)34 CallbackProcessor::CallbackProcessor(sp<Camera2Client> client):
35 Thread(false),
36 mClient(client),
37 mDevice(client->getCameraDevice()),
38 mId(client->getCameraId()),
39 mCallbackAvailable(false),
40 mCallbackToApp(false),
41 mCallbackStreamId(NO_STREAM) {
42 }
43
~CallbackProcessor()44 CallbackProcessor::~CallbackProcessor() {
45 ALOGV("%s: Exit", __FUNCTION__);
46 deleteStream();
47 }
48
onFrameAvailable(const BufferItem &)49 void CallbackProcessor::onFrameAvailable(const BufferItem& /*item*/) {
50 Mutex::Autolock l(mInputMutex);
51 if (!mCallbackAvailable) {
52 mCallbackAvailable = true;
53 mCallbackAvailableSignal.signal();
54 }
55 }
56
setCallbackWindow(const sp<Surface> & callbackWindow)57 status_t CallbackProcessor::setCallbackWindow(
58 const sp<Surface>& callbackWindow) {
59 ATRACE_CALL();
60 status_t res;
61
62 Mutex::Autolock l(mInputMutex);
63
64 sp<Camera2Client> client = mClient.promote();
65 if (client == 0) return OK;
66 sp<CameraDeviceBase> device = client->getCameraDevice();
67
68 // If the window is changing, clear out stream if it already exists
69 if (mCallbackWindow != callbackWindow && mCallbackStreamId != NO_STREAM) {
70 res = device->deleteStream(mCallbackStreamId);
71 if (res != OK) {
72 ALOGE("%s: Camera %d: Unable to delete old stream "
73 "for callbacks: %s (%d)", __FUNCTION__,
74 client->getCameraId(), strerror(-res), res);
75 return res;
76 }
77 mCallbackStreamId = NO_STREAM;
78 mCallbackConsumer.clear();
79 }
80 mCallbackWindow = callbackWindow;
81 mCallbackToApp = (mCallbackWindow != NULL);
82
83 return OK;
84 }
85
updateStream(const Parameters & params)86 status_t CallbackProcessor::updateStream(const Parameters ¶ms) {
87 ATRACE_CALL();
88 status_t res;
89
90 Mutex::Autolock l(mInputMutex);
91
92 sp<CameraDeviceBase> device = mDevice.promote();
93 if (device == 0) {
94 ALOGE("%s: Camera %d: Device does not exist", __FUNCTION__, mId);
95 return INVALID_OPERATION;
96 }
97
98 // If possible, use the flexible YUV format
99 int32_t callbackFormat = params.previewFormat;
100 if (mCallbackToApp) {
101 // TODO: etalvala: This should use the flexible YUV format as well, but
102 // need to reconcile HAL2/HAL3 requirements.
103 callbackFormat = HAL_PIXEL_FORMAT_YV12;
104 } else if(params.fastInfo.useFlexibleYuv &&
105 (params.previewFormat == HAL_PIXEL_FORMAT_YCrCb_420_SP ||
106 params.previewFormat == HAL_PIXEL_FORMAT_YV12) ) {
107 callbackFormat = HAL_PIXEL_FORMAT_YCbCr_420_888;
108 }
109
110 if (!mCallbackToApp && mCallbackConsumer == 0) {
111 // Create CPU buffer queue endpoint, since app hasn't given us one
112 // Make it async to avoid disconnect deadlocks
113 sp<IGraphicBufferProducer> producer;
114 sp<IGraphicBufferConsumer> consumer;
115 BufferQueue::createBufferQueue(&producer, &consumer);
116 mCallbackConsumer = new CpuConsumer(consumer, kCallbackHeapCount);
117 mCallbackConsumer->setFrameAvailableListener(this);
118 mCallbackConsumer->setName(String8("Camera2-CallbackConsumer"));
119 mCallbackWindow = new Surface(producer);
120 }
121
122 if (mCallbackStreamId != NO_STREAM) {
123 // Check if stream parameters have to change
124 CameraDeviceBase::StreamInfo streamInfo;
125 res = device->getStreamInfo(mCallbackStreamId, &streamInfo);
126 if (res != OK) {
127 ALOGE("%s: Camera %d: Error querying callback output stream info: "
128 "%s (%d)", __FUNCTION__, mId,
129 strerror(-res), res);
130 return res;
131 }
132 if (streamInfo.width != (uint32_t)params.previewWidth ||
133 streamInfo.height != (uint32_t)params.previewHeight ||
134 !streamInfo.matchFormat((uint32_t)callbackFormat)) {
135 // Since size should only change while preview is not running,
136 // assuming that all existing use of old callback stream is
137 // completed.
138 ALOGV("%s: Camera %d: Deleting stream %d since the buffer "
139 "parameters changed", __FUNCTION__, mId, mCallbackStreamId);
140 res = device->deleteStream(mCallbackStreamId);
141 if (res != OK) {
142 ALOGE("%s: Camera %d: Unable to delete old output stream "
143 "for callbacks: %s (%d)", __FUNCTION__,
144 mId, strerror(-res), res);
145 return res;
146 }
147 mCallbackStreamId = NO_STREAM;
148 }
149 }
150
151 if (mCallbackStreamId == NO_STREAM) {
152 ALOGV("Creating callback stream: %d x %d, format 0x%x, API format 0x%x",
153 params.previewWidth, params.previewHeight,
154 callbackFormat, params.previewFormat);
155 res = device->createStream(mCallbackWindow,
156 params.previewWidth, params.previewHeight, callbackFormat,
157 HAL_DATASPACE_V0_JFIF, CAMERA3_STREAM_ROTATION_0, &mCallbackStreamId,
158 String8());
159 if (res != OK) {
160 ALOGE("%s: Camera %d: Can't create output stream for callbacks: "
161 "%s (%d)", __FUNCTION__, mId,
162 strerror(-res), res);
163 return res;
164 }
165 }
166
167 return OK;
168 }
169
deleteStream()170 status_t CallbackProcessor::deleteStream() {
171 ATRACE_CALL();
172 sp<CameraDeviceBase> device;
173 status_t res;
174 {
175 Mutex::Autolock l(mInputMutex);
176
177 if (mCallbackStreamId == NO_STREAM) {
178 return OK;
179 }
180 device = mDevice.promote();
181 if (device == 0) {
182 ALOGE("%s: Camera %d: Device does not exist", __FUNCTION__, mId);
183 return INVALID_OPERATION;
184 }
185 }
186 res = device->waitUntilDrained();
187 if (res != OK) {
188 ALOGE("%s: Error waiting for HAL to drain: %s (%d)",
189 __FUNCTION__, strerror(-res), res);
190 return res;
191 }
192
193 res = device->deleteStream(mCallbackStreamId);
194 if (res != OK) {
195 ALOGE("%s: Unable to delete callback stream: %s (%d)",
196 __FUNCTION__, strerror(-res), res);
197 return res;
198 }
199
200 {
201 Mutex::Autolock l(mInputMutex);
202
203 mCallbackHeap.clear();
204 mCallbackWindow.clear();
205 mCallbackConsumer.clear();
206
207 mCallbackStreamId = NO_STREAM;
208 }
209 return OK;
210 }
211
getStreamId() const212 int CallbackProcessor::getStreamId() const {
213 Mutex::Autolock l(mInputMutex);
214 return mCallbackStreamId;
215 }
216
dump(int,const Vector<String16> &) const217 void CallbackProcessor::dump(int /*fd*/, const Vector<String16>& /*args*/) const {
218 }
219
threadLoop()220 bool CallbackProcessor::threadLoop() {
221 status_t res;
222
223 {
224 Mutex::Autolock l(mInputMutex);
225 while (!mCallbackAvailable) {
226 res = mCallbackAvailableSignal.waitRelative(mInputMutex,
227 kWaitDuration);
228 if (res == TIMED_OUT) return true;
229 }
230 mCallbackAvailable = false;
231 }
232
233 do {
234 sp<Camera2Client> client = mClient.promote();
235 if (client == 0) {
236 res = discardNewCallback();
237 } else {
238 res = processNewCallback(client);
239 }
240 } while (res == OK);
241
242 return true;
243 }
244
discardNewCallback()245 status_t CallbackProcessor::discardNewCallback() {
246 ATRACE_CALL();
247 status_t res;
248 CpuConsumer::LockedBuffer imgBuffer;
249 res = mCallbackConsumer->lockNextBuffer(&imgBuffer);
250 if (res != OK) {
251 if (res != BAD_VALUE) {
252 ALOGE("%s: Camera %d: Error receiving next callback buffer: "
253 "%s (%d)", __FUNCTION__, mId, strerror(-res), res);
254 }
255 return res;
256 }
257 mCallbackConsumer->unlockBuffer(imgBuffer);
258 return OK;
259 }
260
processNewCallback(sp<Camera2Client> & client)261 status_t CallbackProcessor::processNewCallback(sp<Camera2Client> &client) {
262 ATRACE_CALL();
263 status_t res;
264
265 sp<Camera2Heap> callbackHeap;
266 bool useFlexibleYuv = false;
267 int32_t previewFormat = 0;
268 size_t heapIdx;
269
270 {
271 /* acquire SharedParameters before mMutex so we don't dead lock
272 with Camera2Client code calling into StreamingProcessor */
273 SharedParameters::Lock l(client->getParameters());
274 Mutex::Autolock m(mInputMutex);
275 CpuConsumer::LockedBuffer imgBuffer;
276 if (mCallbackStreamId == NO_STREAM) {
277 ALOGV("%s: Camera %d:No stream is available"
278 , __FUNCTION__, mId);
279 return INVALID_OPERATION;
280 }
281
282 ALOGV("%s: Getting buffer", __FUNCTION__);
283 res = mCallbackConsumer->lockNextBuffer(&imgBuffer);
284 if (res != OK) {
285 if (res != BAD_VALUE) {
286 ALOGE("%s: Camera %d: Error receiving next callback buffer: "
287 "%s (%d)", __FUNCTION__, mId, strerror(-res), res);
288 }
289 return res;
290 }
291 ALOGV("%s: Camera %d: Preview callback available", __FUNCTION__,
292 mId);
293
294 if ( l.mParameters.state != Parameters::PREVIEW
295 && l.mParameters.state != Parameters::RECORD
296 && l.mParameters.state != Parameters::VIDEO_SNAPSHOT) {
297 ALOGV("%s: Camera %d: No longer streaming",
298 __FUNCTION__, mId);
299 mCallbackConsumer->unlockBuffer(imgBuffer);
300 return OK;
301 }
302
303 if (! (l.mParameters.previewCallbackFlags &
304 CAMERA_FRAME_CALLBACK_FLAG_ENABLE_MASK) ) {
305 ALOGV("%s: No longer enabled, dropping", __FUNCTION__);
306 mCallbackConsumer->unlockBuffer(imgBuffer);
307 return OK;
308 }
309 if ((l.mParameters.previewCallbackFlags &
310 CAMERA_FRAME_CALLBACK_FLAG_ONE_SHOT_MASK) &&
311 !l.mParameters.previewCallbackOneShot) {
312 ALOGV("%s: One shot mode, already sent, dropping", __FUNCTION__);
313 mCallbackConsumer->unlockBuffer(imgBuffer);
314 return OK;
315 }
316
317 if (imgBuffer.width != static_cast<uint32_t>(l.mParameters.previewWidth) ||
318 imgBuffer.height != static_cast<uint32_t>(l.mParameters.previewHeight)) {
319 ALOGW("%s: The preview size has changed to %d x %d from %d x %d, this buffer is"
320 " no longer valid, dropping",__FUNCTION__,
321 l.mParameters.previewWidth, l.mParameters.previewHeight,
322 imgBuffer.width, imgBuffer.height);
323 mCallbackConsumer->unlockBuffer(imgBuffer);
324 return OK;
325 }
326
327 previewFormat = l.mParameters.previewFormat;
328 useFlexibleYuv = l.mParameters.fastInfo.useFlexibleYuv &&
329 (previewFormat == HAL_PIXEL_FORMAT_YCrCb_420_SP ||
330 previewFormat == HAL_PIXEL_FORMAT_YV12);
331
332 int32_t expectedFormat = useFlexibleYuv ?
333 HAL_PIXEL_FORMAT_YCbCr_420_888 : previewFormat;
334
335 if (imgBuffer.format != expectedFormat) {
336 ALOGE("%s: Camera %d: Unexpected format for callback: "
337 "0x%x, expected 0x%x", __FUNCTION__, mId,
338 imgBuffer.format, expectedFormat);
339 mCallbackConsumer->unlockBuffer(imgBuffer);
340 return INVALID_OPERATION;
341 }
342
343 // In one-shot mode, stop sending callbacks after the first one
344 if (l.mParameters.previewCallbackFlags &
345 CAMERA_FRAME_CALLBACK_FLAG_ONE_SHOT_MASK) {
346 ALOGV("%s: clearing oneshot", __FUNCTION__);
347 l.mParameters.previewCallbackOneShot = false;
348 }
349
350 uint32_t destYStride = 0;
351 uint32_t destCStride = 0;
352 if (useFlexibleYuv) {
353 if (previewFormat == HAL_PIXEL_FORMAT_YV12) {
354 // Strides must align to 16 for YV12
355 destYStride = ALIGN(imgBuffer.width, 16);
356 destCStride = ALIGN(destYStride / 2, 16);
357 } else {
358 // No padding for NV21
359 ALOG_ASSERT(previewFormat == HAL_PIXEL_FORMAT_YCrCb_420_SP,
360 "Unexpected preview format 0x%x", previewFormat);
361 destYStride = imgBuffer.width;
362 destCStride = destYStride / 2;
363 }
364 } else {
365 destYStride = imgBuffer.stride;
366 // don't care about cStride
367 }
368
369 size_t bufferSize = Camera2Client::calculateBufferSize(
370 imgBuffer.width, imgBuffer.height,
371 previewFormat, destYStride);
372 size_t currentBufferSize = (mCallbackHeap == 0) ?
373 0 : (mCallbackHeap->mHeap->getSize() / kCallbackHeapCount);
374 if (bufferSize != currentBufferSize) {
375 mCallbackHeap.clear();
376 mCallbackHeap = new Camera2Heap(bufferSize, kCallbackHeapCount,
377 "Camera2Client::CallbackHeap");
378 if (mCallbackHeap->mHeap->getSize() == 0) {
379 ALOGE("%s: Camera %d: Unable to allocate memory for callbacks",
380 __FUNCTION__, mId);
381 mCallbackConsumer->unlockBuffer(imgBuffer);
382 return INVALID_OPERATION;
383 }
384
385 mCallbackHeapHead = 0;
386 mCallbackHeapFree = kCallbackHeapCount;
387 }
388
389 if (mCallbackHeapFree == 0) {
390 ALOGE("%s: Camera %d: No free callback buffers, dropping frame",
391 __FUNCTION__, mId);
392 mCallbackConsumer->unlockBuffer(imgBuffer);
393 return OK;
394 }
395
396 heapIdx = mCallbackHeapHead;
397
398 mCallbackHeapHead = (mCallbackHeapHead + 1) % kCallbackHeapCount;
399 mCallbackHeapFree--;
400
401 // TODO: Get rid of this copy by passing the gralloc queue all the way
402 // to app
403
404 ssize_t offset;
405 size_t size;
406 sp<IMemoryHeap> heap =
407 mCallbackHeap->mBuffers[heapIdx]->getMemory(&offset,
408 &size);
409 uint8_t *data = (uint8_t*)heap->getBase() + offset;
410
411 if (!useFlexibleYuv) {
412 // Can just memcpy when HAL format matches API format
413 memcpy(data, imgBuffer.data, bufferSize);
414 } else {
415 res = convertFromFlexibleYuv(previewFormat, data, imgBuffer,
416 destYStride, destCStride);
417 if (res != OK) {
418 ALOGE("%s: Camera %d: Can't convert between 0x%x and 0x%x formats!",
419 __FUNCTION__, mId, imgBuffer.format, previewFormat);
420 mCallbackConsumer->unlockBuffer(imgBuffer);
421 return BAD_VALUE;
422 }
423 }
424
425 ALOGV("%s: Freeing buffer", __FUNCTION__);
426 mCallbackConsumer->unlockBuffer(imgBuffer);
427
428 // mCallbackHeap may get freed up once input mutex is released
429 callbackHeap = mCallbackHeap;
430 }
431
432 // Call outside parameter lock to allow re-entrancy from notification
433 {
434 Camera2Client::SharedCameraCallbacks::Lock
435 l(client->mSharedCameraCallbacks);
436 if (l.mRemoteCallback != 0) {
437 ALOGV("%s: Camera %d: Invoking client data callback",
438 __FUNCTION__, mId);
439 l.mRemoteCallback->dataCallback(CAMERA_MSG_PREVIEW_FRAME,
440 callbackHeap->mBuffers[heapIdx], NULL);
441 }
442 }
443
444 // Only increment free if we're still using the same heap
445 mCallbackHeapFree++;
446
447 ALOGV("%s: exit", __FUNCTION__);
448
449 return OK;
450 }
451
convertFromFlexibleYuv(int32_t previewFormat,uint8_t * dst,const CpuConsumer::LockedBuffer & src,uint32_t dstYStride,uint32_t dstCStride) const452 status_t CallbackProcessor::convertFromFlexibleYuv(int32_t previewFormat,
453 uint8_t *dst,
454 const CpuConsumer::LockedBuffer &src,
455 uint32_t dstYStride,
456 uint32_t dstCStride) const {
457
458 if (previewFormat != HAL_PIXEL_FORMAT_YCrCb_420_SP &&
459 previewFormat != HAL_PIXEL_FORMAT_YV12) {
460 ALOGE("%s: Camera %d: Unexpected preview format when using "
461 "flexible YUV: 0x%x", __FUNCTION__, mId, previewFormat);
462 return INVALID_OPERATION;
463 }
464
465 // Copy Y plane, adjusting for stride
466 const uint8_t *ySrc = src.data;
467 uint8_t *yDst = dst;
468 for (size_t row = 0; row < src.height; row++) {
469 memcpy(yDst, ySrc, src.width);
470 ySrc += src.stride;
471 yDst += dstYStride;
472 }
473
474 // Copy/swizzle chroma planes, 4:2:0 subsampling
475 const uint8_t *cbSrc = src.dataCb;
476 const uint8_t *crSrc = src.dataCr;
477 size_t chromaHeight = src.height / 2;
478 size_t chromaWidth = src.width / 2;
479 ssize_t chromaGap = src.chromaStride -
480 (chromaWidth * src.chromaStep);
481 size_t dstChromaGap = dstCStride - chromaWidth;
482
483 if (previewFormat == HAL_PIXEL_FORMAT_YCrCb_420_SP) {
484 // Flexible YUV chroma to NV21 chroma
485 uint8_t *crcbDst = yDst;
486 // Check for shortcuts
487 if (cbSrc == crSrc + 1 && src.chromaStep == 2) {
488 ALOGV("%s: Fast NV21->NV21", __FUNCTION__);
489 // Source has semiplanar CrCb chroma layout, can copy by rows
490 for (size_t row = 0; row < chromaHeight; row++) {
491 memcpy(crcbDst, crSrc, src.width);
492 crcbDst += src.width;
493 crSrc += src.chromaStride;
494 }
495 } else {
496 ALOGV("%s: Generic->NV21", __FUNCTION__);
497 // Generic copy, always works but not very efficient
498 for (size_t row = 0; row < chromaHeight; row++) {
499 for (size_t col = 0; col < chromaWidth; col++) {
500 *(crcbDst++) = *crSrc;
501 *(crcbDst++) = *cbSrc;
502 crSrc += src.chromaStep;
503 cbSrc += src.chromaStep;
504 }
505 crSrc += chromaGap;
506 cbSrc += chromaGap;
507 }
508 }
509 } else {
510 // flexible YUV chroma to YV12 chroma
511 ALOG_ASSERT(previewFormat == HAL_PIXEL_FORMAT_YV12,
512 "Unexpected preview format 0x%x", previewFormat);
513 uint8_t *crDst = yDst;
514 uint8_t *cbDst = yDst + chromaHeight * dstCStride;
515 if (src.chromaStep == 1) {
516 ALOGV("%s: Fast YV12->YV12", __FUNCTION__);
517 // Source has planar chroma layout, can copy by row
518 for (size_t row = 0; row < chromaHeight; row++) {
519 memcpy(crDst, crSrc, chromaWidth);
520 crDst += dstCStride;
521 crSrc += src.chromaStride;
522 }
523 for (size_t row = 0; row < chromaHeight; row++) {
524 memcpy(cbDst, cbSrc, chromaWidth);
525 cbDst += dstCStride;
526 cbSrc += src.chromaStride;
527 }
528 } else {
529 ALOGV("%s: Generic->YV12", __FUNCTION__);
530 // Generic copy, always works but not very efficient
531 for (size_t row = 0; row < chromaHeight; row++) {
532 for (size_t col = 0; col < chromaWidth; col++) {
533 *(crDst++) = *crSrc;
534 *(cbDst++) = *cbSrc;
535 crSrc += src.chromaStep;
536 cbSrc += src.chromaStep;
537 }
538 crSrc += chromaGap;
539 cbSrc += chromaGap;
540 crDst += dstChromaGap;
541 cbDst += dstChromaGap;
542 }
543 }
544 }
545
546 return OK;
547 }
548
549 }; // namespace camera2
550 }; // namespace android
551