1 /* 2 * Copyright (C) 2014 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package android.hardware.camera2.legacy; 18 19 import android.graphics.SurfaceTexture; 20 import android.hardware.Camera; 21 import android.hardware.camera2.CameraCharacteristics; 22 import android.hardware.camera2.CaptureRequest; 23 import android.hardware.camera2.impl.CameraDeviceImpl; 24 import android.hardware.camera2.utils.SubmitInfo; 25 import android.hardware.camera2.utils.SizeAreaComparator; 26 import android.hardware.camera2.impl.CameraMetadataNative; 27 import android.os.ConditionVariable; 28 import android.os.Handler; 29 import android.os.Message; 30 import android.os.SystemClock; 31 import android.util.Log; 32 import android.util.MutableLong; 33 import android.util.Pair; 34 import android.util.Size; 35 import android.view.Surface; 36 37 import java.io.IOException; 38 import java.util.ArrayList; 39 import java.util.Collection; 40 import java.util.Collections; 41 import java.util.Iterator; 42 import java.util.List; 43 import java.util.concurrent.TimeUnit; 44 import java.util.concurrent.atomic.AtomicBoolean; 45 46 import static com.android.internal.util.Preconditions.*; 47 48 /** 49 * This class executes requests to the {@link Camera}. 50 * 51 * <p> 52 * The main components of this class are: 53 * - A message queue of requests to the {@link Camera}. 54 * - A thread that consumes requests to the {@link Camera} and executes them. 55 * - A {@link GLThreadManager} that draws to the configured output {@link Surface}s. 56 * - An {@link CameraDeviceState} state machine that manages the callbacks for various operations. 57 * </p> 58 */ 59 @SuppressWarnings("deprecation") 60 public class RequestThreadManager { 61 private final String TAG; 62 private final int mCameraId; 63 private final RequestHandlerThread mRequestThread; 64 65 private static final boolean DEBUG = false; 66 // For slightly more spammy messages that will get repeated every frame 67 private static final boolean VERBOSE = false; 68 private Camera mCamera; 69 private final CameraCharacteristics mCharacteristics; 70 71 private final CameraDeviceState mDeviceState; 72 private final CaptureCollector mCaptureCollector; 73 private final LegacyFocusStateMapper mFocusStateMapper; 74 private final LegacyFaceDetectMapper mFaceDetectMapper; 75 76 private static final int MSG_CONFIGURE_OUTPUTS = 1; 77 private static final int MSG_SUBMIT_CAPTURE_REQUEST = 2; 78 private static final int MSG_CLEANUP = 3; 79 80 private static final int MAX_IN_FLIGHT_REQUESTS = 2; 81 82 private static final int PREVIEW_FRAME_TIMEOUT = 1000; // ms 83 private static final int JPEG_FRAME_TIMEOUT = 4000; // ms (same as CTS for API2) 84 private static final int REQUEST_COMPLETE_TIMEOUT = JPEG_FRAME_TIMEOUT; 85 86 private static final float ASPECT_RATIO_TOLERANCE = 0.01f; 87 private boolean mPreviewRunning = false; 88 89 private final List<Surface> mPreviewOutputs = new ArrayList<>(); 90 private final List<Surface> mCallbackOutputs = new ArrayList<>(); 91 private GLThreadManager mGLThreadManager; 92 private SurfaceTexture mPreviewTexture; 93 private Camera.Parameters mParams; 94 95 private final List<Long> mJpegSurfaceIds = new ArrayList<>(); 96 97 private Size mIntermediateBufferSize; 98 99 private final RequestQueue mRequestQueue = new RequestQueue(mJpegSurfaceIds); 100 private LegacyRequest mLastRequest = null; 101 private SurfaceTexture mDummyTexture; 102 private Surface mDummySurface; 103 104 private final Object mIdleLock = new Object(); 105 private final FpsCounter mPrevCounter = new FpsCounter("Incoming Preview"); 106 private final FpsCounter mRequestCounter = new FpsCounter("Incoming Requests"); 107 108 private final AtomicBoolean mQuit = new AtomicBoolean(false); 109 110 // Stuff JPEGs into HAL_PIXEL_FORMAT_RGBA_8888 gralloc buffers to get around SW write 111 // limitations for (b/17379185). 112 private static final boolean USE_BLOB_FORMAT_OVERRIDE = true; 113 114 /** 115 * Container object for Configure messages. 116 */ 117 private static class ConfigureHolder { 118 public final ConditionVariable condition; 119 public final Collection<Pair<Surface, Size>> surfaces; 120 ConfigureHolder(ConditionVariable condition, Collection<Pair<Surface, Size>> surfaces)121 public ConfigureHolder(ConditionVariable condition, Collection<Pair<Surface, 122 Size>> surfaces) { 123 this.condition = condition; 124 this.surfaces = surfaces; 125 } 126 } 127 128 /** 129 * Counter class used to calculate and log the current FPS of frame production. 130 */ 131 public static class FpsCounter { 132 //TODO: Hook this up to SystTrace? 133 private static final String TAG = "FpsCounter"; 134 private int mFrameCount = 0; 135 private long mLastTime = 0; 136 private long mLastPrintTime = 0; 137 private double mLastFps = 0; 138 private final String mStreamType; 139 private static final long NANO_PER_SECOND = 1000000000; //ns 140 FpsCounter(String streamType)141 public FpsCounter(String streamType) { 142 mStreamType = streamType; 143 } 144 countFrame()145 public synchronized void countFrame() { 146 mFrameCount++; 147 long nextTime = SystemClock.elapsedRealtimeNanos(); 148 if (mLastTime == 0) { 149 mLastTime = nextTime; 150 } 151 if (nextTime > mLastTime + NANO_PER_SECOND) { 152 long elapsed = nextTime - mLastTime; 153 mLastFps = mFrameCount * (NANO_PER_SECOND / (double) elapsed); 154 mFrameCount = 0; 155 mLastTime = nextTime; 156 } 157 } 158 checkFps()159 public synchronized double checkFps() { 160 return mLastFps; 161 } 162 staggeredLog()163 public synchronized void staggeredLog() { 164 if (mLastTime > mLastPrintTime + 5 * NANO_PER_SECOND) { 165 mLastPrintTime = mLastTime; 166 Log.d(TAG, "FPS for " + mStreamType + " stream: " + mLastFps ); 167 } 168 } 169 countAndLog()170 public synchronized void countAndLog() { 171 countFrame(); 172 staggeredLog(); 173 } 174 } 175 /** 176 * Fake preview for jpeg captures when there is no active preview 177 */ createDummySurface()178 private void createDummySurface() { 179 if (mDummyTexture == null || mDummySurface == null) { 180 mDummyTexture = new SurfaceTexture(/*ignored*/0); 181 // TODO: use smallest default sizes 182 mDummyTexture.setDefaultBufferSize(640, 480); 183 mDummySurface = new Surface(mDummyTexture); 184 } 185 } 186 187 private final Camera.ErrorCallback mErrorCallback = new Camera.ErrorCallback() { 188 @Override 189 public void onError(int i, Camera camera) { 190 switch(i) { 191 case Camera.CAMERA_ERROR_EVICTED: { 192 flush(); 193 mDeviceState.setError( 194 CameraDeviceImpl.CameraDeviceCallbacks.ERROR_CAMERA_DISCONNECTED); 195 } break; 196 case Camera.CAMERA_ERROR_DISABLED: { 197 flush(); 198 mDeviceState.setError( 199 CameraDeviceImpl.CameraDeviceCallbacks.ERROR_CAMERA_DISABLED); 200 } break; 201 default: { 202 Log.e(TAG, "Received error " + i + " from the Camera1 ErrorCallback"); 203 mDeviceState.setError( 204 CameraDeviceImpl.CameraDeviceCallbacks.ERROR_CAMERA_DEVICE); 205 } break; 206 } 207 } 208 }; 209 210 private final ConditionVariable mReceivedJpeg = new ConditionVariable(false); 211 212 private final Camera.PictureCallback mJpegCallback = new Camera.PictureCallback() { 213 @Override 214 public void onPictureTaken(byte[] data, Camera camera) { 215 Log.i(TAG, "Received jpeg."); 216 Pair<RequestHolder, Long> captureInfo = mCaptureCollector.jpegProduced(); 217 if (captureInfo == null || captureInfo.first == null) { 218 Log.e(TAG, "Dropping jpeg frame."); 219 return; 220 } 221 RequestHolder holder = captureInfo.first; 222 long timestamp = captureInfo.second; 223 for (Surface s : holder.getHolderTargets()) { 224 try { 225 if (LegacyCameraDevice.containsSurfaceId(s, mJpegSurfaceIds)) { 226 Log.i(TAG, "Producing jpeg buffer..."); 227 228 int totalSize = data.length + LegacyCameraDevice.nativeGetJpegFooterSize(); 229 totalSize = (totalSize + 3) & ~0x3; // round up to nearest octonibble 230 LegacyCameraDevice.setNextTimestamp(s, timestamp); 231 232 if (USE_BLOB_FORMAT_OVERRIDE) { 233 // Override to RGBA_8888 format. 234 LegacyCameraDevice.setSurfaceFormat(s, 235 LegacyMetadataMapper.HAL_PIXEL_FORMAT_RGBA_8888); 236 237 int dimen = (int) Math.ceil(Math.sqrt(totalSize)); 238 dimen = (dimen + 0xf) & ~0xf; // round up to nearest multiple of 16 239 LegacyCameraDevice.setSurfaceDimens(s, dimen, dimen); 240 LegacyCameraDevice.produceFrame(s, data, dimen, dimen, 241 CameraMetadataNative.NATIVE_JPEG_FORMAT); 242 } else { 243 LegacyCameraDevice.setSurfaceDimens(s, totalSize, /*height*/1); 244 LegacyCameraDevice.produceFrame(s, data, totalSize, /*height*/1, 245 CameraMetadataNative.NATIVE_JPEG_FORMAT); 246 } 247 } 248 } catch (LegacyExceptionUtils.BufferQueueAbandonedException e) { 249 Log.w(TAG, "Surface abandoned, dropping frame. ", e); 250 } 251 } 252 253 mReceivedJpeg.open(); 254 } 255 }; 256 257 private final Camera.ShutterCallback mJpegShutterCallback = new Camera.ShutterCallback() { 258 @Override 259 public void onShutter() { 260 mCaptureCollector.jpegCaptured(SystemClock.elapsedRealtimeNanos()); 261 } 262 }; 263 264 private final SurfaceTexture.OnFrameAvailableListener mPreviewCallback = 265 new SurfaceTexture.OnFrameAvailableListener() { 266 @Override 267 public void onFrameAvailable(SurfaceTexture surfaceTexture) { 268 if (DEBUG) { 269 mPrevCounter.countAndLog(); 270 } 271 mGLThreadManager.queueNewFrame(); 272 } 273 }; 274 stopPreview()275 private void stopPreview() { 276 if (VERBOSE) { 277 Log.v(TAG, "stopPreview - preview running? " + mPreviewRunning); 278 } 279 if (mPreviewRunning) { 280 mCamera.stopPreview(); 281 mPreviewRunning = false; 282 } 283 } 284 startPreview()285 private void startPreview() { 286 if (VERBOSE) { 287 Log.v(TAG, "startPreview - preview running? " + mPreviewRunning); 288 } 289 if (!mPreviewRunning) { 290 // XX: CameraClient:;startPreview is not getting called after a stop 291 mCamera.startPreview(); 292 mPreviewRunning = true; 293 } 294 } 295 doJpegCapturePrepare(RequestHolder request)296 private void doJpegCapturePrepare(RequestHolder request) throws IOException { 297 if (DEBUG) Log.d(TAG, "doJpegCapturePrepare - preview running? " + mPreviewRunning); 298 299 if (!mPreviewRunning) { 300 if (DEBUG) Log.d(TAG, "doJpegCapture - create fake surface"); 301 302 createDummySurface(); 303 mCamera.setPreviewTexture(mDummyTexture); 304 startPreview(); 305 } 306 } 307 doJpegCapture(RequestHolder request)308 private void doJpegCapture(RequestHolder request) { 309 if (DEBUG) Log.d(TAG, "doJpegCapturePrepare"); 310 311 mCamera.takePicture(mJpegShutterCallback, /*raw*/null, mJpegCallback); 312 mPreviewRunning = false; 313 } 314 doPreviewCapture(RequestHolder request)315 private void doPreviewCapture(RequestHolder request) throws IOException { 316 if (VERBOSE) { 317 Log.v(TAG, "doPreviewCapture - preview running? " + mPreviewRunning); 318 } 319 320 if (mPreviewRunning) { 321 return; // Already running 322 } 323 324 if (mPreviewTexture == null) { 325 throw new IllegalStateException( 326 "Preview capture called with no preview surfaces configured."); 327 } 328 329 mPreviewTexture.setDefaultBufferSize(mIntermediateBufferSize.getWidth(), 330 mIntermediateBufferSize.getHeight()); 331 mCamera.setPreviewTexture(mPreviewTexture); 332 333 startPreview(); 334 } 335 disconnectCallbackSurfaces()336 private void disconnectCallbackSurfaces() { 337 for (Surface s : mCallbackOutputs) { 338 try { 339 LegacyCameraDevice.disconnectSurface(s); 340 } catch (LegacyExceptionUtils.BufferQueueAbandonedException e) { 341 Log.d(TAG, "Surface abandoned, skipping...", e); 342 } 343 } 344 } 345 configureOutputs(Collection<Pair<Surface, Size>> outputs)346 private void configureOutputs(Collection<Pair<Surface, Size>> outputs) { 347 if (DEBUG) { 348 String outputsStr = outputs == null ? "null" : (outputs.size() + " surfaces"); 349 Log.d(TAG, "configureOutputs with " + outputsStr); 350 } 351 352 try { 353 stopPreview(); 354 } catch (RuntimeException e) { 355 Log.e(TAG, "Received device exception in configure call: ", e); 356 mDeviceState.setError( 357 CameraDeviceImpl.CameraDeviceCallbacks.ERROR_CAMERA_DEVICE); 358 return; 359 } 360 361 /* 362 * Try to release the previous preview's surface texture earlier if we end up 363 * using a different one; this also reduces the likelihood of getting into a deadlock 364 * when disconnecting from the old previous texture at a later time. 365 */ 366 try { 367 mCamera.setPreviewTexture(/*surfaceTexture*/null); 368 } catch (IOException e) { 369 Log.w(TAG, "Failed to clear prior SurfaceTexture, may cause GL deadlock: ", e); 370 } catch (RuntimeException e) { 371 Log.e(TAG, "Received device exception in configure call: ", e); 372 mDeviceState.setError( 373 CameraDeviceImpl.CameraDeviceCallbacks.ERROR_CAMERA_DEVICE); 374 return; 375 } 376 377 if (mGLThreadManager != null) { 378 mGLThreadManager.waitUntilStarted(); 379 mGLThreadManager.ignoreNewFrames(); 380 mGLThreadManager.waitUntilIdle(); 381 } 382 resetJpegSurfaceFormats(mCallbackOutputs); 383 disconnectCallbackSurfaces(); 384 385 mPreviewOutputs.clear(); 386 mCallbackOutputs.clear(); 387 mJpegSurfaceIds.clear(); 388 mPreviewTexture = null; 389 390 List<Size> previewOutputSizes = new ArrayList<>(); 391 List<Size> callbackOutputSizes = new ArrayList<>(); 392 393 int facing = mCharacteristics.get(CameraCharacteristics.LENS_FACING); 394 int orientation = mCharacteristics.get(CameraCharacteristics.SENSOR_ORIENTATION); 395 if (outputs != null) { 396 for (Pair<Surface, Size> outPair : outputs) { 397 Surface s = outPair.first; 398 Size outSize = outPair.second; 399 try { 400 int format = LegacyCameraDevice.detectSurfaceType(s); 401 LegacyCameraDevice.setSurfaceOrientation(s, facing, orientation); 402 switch (format) { 403 case CameraMetadataNative.NATIVE_JPEG_FORMAT: 404 if (USE_BLOB_FORMAT_OVERRIDE) { 405 // Override to RGBA_8888 format. 406 LegacyCameraDevice.setSurfaceFormat(s, 407 LegacyMetadataMapper.HAL_PIXEL_FORMAT_RGBA_8888); 408 } 409 mJpegSurfaceIds.add(LegacyCameraDevice.getSurfaceId(s)); 410 mCallbackOutputs.add(s); 411 callbackOutputSizes.add(outSize); 412 413 // LegacyCameraDevice is the producer of JPEG output surfaces 414 // so LegacyCameraDevice needs to connect to the surfaces. 415 LegacyCameraDevice.connectSurface(s); 416 break; 417 default: 418 LegacyCameraDevice.setScalingMode(s, LegacyCameraDevice. 419 NATIVE_WINDOW_SCALING_MODE_SCALE_TO_WINDOW); 420 mPreviewOutputs.add(s); 421 previewOutputSizes.add(outSize); 422 break; 423 } 424 } catch (LegacyExceptionUtils.BufferQueueAbandonedException e) { 425 Log.w(TAG, "Surface abandoned, skipping...", e); 426 } 427 } 428 } 429 try { 430 mParams = mCamera.getParameters(); 431 } catch (RuntimeException e) { 432 Log.e(TAG, "Received device exception: ", e); 433 mDeviceState.setError( 434 CameraDeviceImpl.CameraDeviceCallbacks.ERROR_CAMERA_DEVICE); 435 return; 436 } 437 438 List<int[]> supportedFpsRanges = mParams.getSupportedPreviewFpsRange(); 439 int[] bestRange = getPhotoPreviewFpsRange(supportedFpsRanges); 440 if (DEBUG) { 441 Log.d(TAG, "doPreviewCapture - Selected range [" + 442 bestRange[Camera.Parameters.PREVIEW_FPS_MIN_INDEX] + "," + 443 bestRange[Camera.Parameters.PREVIEW_FPS_MAX_INDEX] + "]"); 444 } 445 mParams.setPreviewFpsRange(bestRange[Camera.Parameters.PREVIEW_FPS_MIN_INDEX], 446 bestRange[Camera.Parameters.PREVIEW_FPS_MAX_INDEX]); 447 448 Size smallestSupportedJpegSize = calculatePictureSize(mCallbackOutputs, 449 callbackOutputSizes, mParams); 450 451 if (previewOutputSizes.size() > 0) { 452 453 Size largestOutput = SizeAreaComparator.findLargestByArea(previewOutputSizes); 454 455 // Find largest jpeg dimension - assume to have the same aspect ratio as sensor. 456 Size largestJpegDimen = ParameterUtils.getLargestSupportedJpegSizeByArea(mParams); 457 458 Size chosenJpegDimen = (smallestSupportedJpegSize != null) ? smallestSupportedJpegSize 459 : largestJpegDimen; 460 461 List<Size> supportedPreviewSizes = ParameterUtils.convertSizeList( 462 mParams.getSupportedPreviewSizes()); 463 464 // Use smallest preview dimension with same aspect ratio as sensor that is >= than all 465 // of the configured output dimensions. If none exists, fall back to using the largest 466 // supported preview size. 467 long largestOutputArea = largestOutput.getHeight() * (long) largestOutput.getWidth(); 468 Size bestPreviewDimen = SizeAreaComparator.findLargestByArea(supportedPreviewSizes); 469 for (Size s : supportedPreviewSizes) { 470 long currArea = s.getWidth() * s.getHeight(); 471 long bestArea = bestPreviewDimen.getWidth() * bestPreviewDimen.getHeight(); 472 if (checkAspectRatiosMatch(chosenJpegDimen, s) && (currArea < bestArea && 473 currArea >= largestOutputArea)) { 474 bestPreviewDimen = s; 475 } 476 } 477 478 mIntermediateBufferSize = bestPreviewDimen; 479 mParams.setPreviewSize(mIntermediateBufferSize.getWidth(), 480 mIntermediateBufferSize.getHeight()); 481 482 if (DEBUG) { 483 Log.d(TAG, "Intermediate buffer selected with dimens: " + 484 bestPreviewDimen.toString()); 485 } 486 } else { 487 mIntermediateBufferSize = null; 488 if (DEBUG) { 489 Log.d(TAG, "No Intermediate buffer selected, no preview outputs were configured"); 490 } 491 } 492 493 if (smallestSupportedJpegSize != null) { 494 /* 495 * Set takePicture size to the smallest supported JPEG size large enough 496 * to scale/crop out of for the bounding rectangle of the configured JPEG sizes. 497 */ 498 499 Log.i(TAG, "configureOutputs - set take picture size to " + smallestSupportedJpegSize); 500 mParams.setPictureSize( 501 smallestSupportedJpegSize.getWidth(), smallestSupportedJpegSize.getHeight()); 502 } 503 504 // TODO: Detect and optimize single-output paths here to skip stream teeing. 505 if (mGLThreadManager == null) { 506 mGLThreadManager = new GLThreadManager(mCameraId, facing, mDeviceState); 507 mGLThreadManager.start(); 508 } 509 mGLThreadManager.waitUntilStarted(); 510 List<Pair<Surface, Size>> previews = new ArrayList<>(); 511 Iterator<Size> previewSizeIter = previewOutputSizes.iterator(); 512 for (Surface p : mPreviewOutputs) { 513 previews.add(new Pair<>(p, previewSizeIter.next())); 514 } 515 mGLThreadManager.setConfigurationAndWait(previews, mCaptureCollector); 516 517 for (Surface p : mPreviewOutputs) { 518 try { 519 LegacyCameraDevice.setSurfaceOrientation(p, facing, orientation); 520 } catch (LegacyExceptionUtils.BufferQueueAbandonedException e) { 521 Log.e(TAG, "Surface abandoned, skipping setSurfaceOrientation()", e); 522 } 523 } 524 525 mGLThreadManager.allowNewFrames(); 526 mPreviewTexture = mGLThreadManager.getCurrentSurfaceTexture(); 527 if (mPreviewTexture != null) { 528 mPreviewTexture.setOnFrameAvailableListener(mPreviewCallback); 529 } 530 531 try { 532 mCamera.setParameters(mParams); 533 } catch (RuntimeException e) { 534 Log.e(TAG, "Received device exception while configuring: ", e); 535 mDeviceState.setError( 536 CameraDeviceImpl.CameraDeviceCallbacks.ERROR_CAMERA_DEVICE); 537 538 } 539 } 540 resetJpegSurfaceFormats(Collection<Surface> surfaces)541 private void resetJpegSurfaceFormats(Collection<Surface> surfaces) { 542 if (!USE_BLOB_FORMAT_OVERRIDE || surfaces == null) { 543 return; 544 } 545 for(Surface s : surfaces) { 546 if (s == null || !s.isValid()) { 547 Log.w(TAG, "Jpeg surface is invalid, skipping..."); 548 continue; 549 } 550 try { 551 LegacyCameraDevice.setSurfaceFormat(s, LegacyMetadataMapper.HAL_PIXEL_FORMAT_BLOB); 552 } catch (LegacyExceptionUtils.BufferQueueAbandonedException e) { 553 Log.w(TAG, "Surface abandoned, skipping...", e); 554 } 555 } 556 } 557 558 /** 559 * Find a JPEG size (that is supported by the legacy camera device) which is equal to or larger 560 * than all of the configured {@code JPEG} outputs (by both width and height). 561 * 562 * <p>If multiple supported JPEG sizes are larger, select the smallest of them which 563 * still satisfies the above constraint.</p> 564 * 565 * <p>As a result, the returned size is guaranteed to be usable without needing 566 * to upscale any of the outputs. If only one {@code JPEG} surface is used, 567 * then no scaling/cropping is necessary between the taken picture and 568 * the {@code JPEG} output surface.</p> 569 * 570 * @param callbackOutputs a non-{@code null} list of {@code Surface}s with any image formats 571 * @param params api1 parameters (used for reading only) 572 * 573 * @return a size large enough to fit all of the configured {@code JPEG} outputs, or 574 * {@code null} if the {@code callbackOutputs} did not have any {@code JPEG} 575 * surfaces. 576 */ calculatePictureSize( List<Surface> callbackOutputs, List<Size> callbackSizes, Camera.Parameters params)577 private Size calculatePictureSize( List<Surface> callbackOutputs, 578 List<Size> callbackSizes, Camera.Parameters params) { 579 /* 580 * Find the largest JPEG size (if any), from the configured outputs: 581 * - the api1 picture size should be set to the smallest legal size that's at least as large 582 * as the largest configured JPEG size 583 */ 584 if (callbackOutputs.size() != callbackSizes.size()) { 585 throw new IllegalStateException("Input collections must be same length"); 586 } 587 List<Size> configuredJpegSizes = new ArrayList<>(); 588 Iterator<Size> sizeIterator = callbackSizes.iterator(); 589 for (Surface callbackSurface : callbackOutputs) { 590 Size jpegSize = sizeIterator.next(); 591 if (!LegacyCameraDevice.containsSurfaceId(callbackSurface, mJpegSurfaceIds)) { 592 continue; // Ignore non-JPEG callback formats 593 } 594 595 configuredJpegSizes.add(jpegSize); 596 } 597 if (!configuredJpegSizes.isEmpty()) { 598 /* 599 * Find the largest configured JPEG width, and height, independently 600 * of the rest. 601 * 602 * The rest of the JPEG streams can be cropped out of this smallest bounding 603 * rectangle. 604 */ 605 int maxConfiguredJpegWidth = -1; 606 int maxConfiguredJpegHeight = -1; 607 for (Size jpegSize : configuredJpegSizes) { 608 maxConfiguredJpegWidth = jpegSize.getWidth() > maxConfiguredJpegWidth ? 609 jpegSize.getWidth() : maxConfiguredJpegWidth; 610 maxConfiguredJpegHeight = jpegSize.getHeight() > maxConfiguredJpegHeight ? 611 jpegSize.getHeight() : maxConfiguredJpegHeight; 612 } 613 Size smallestBoundJpegSize = new Size(maxConfiguredJpegWidth, maxConfiguredJpegHeight); 614 615 List<Size> supportedJpegSizes = ParameterUtils.convertSizeList( 616 params.getSupportedPictureSizes()); 617 618 /* 619 * Find the smallest supported JPEG size that can fit the smallest bounding 620 * rectangle for the configured JPEG sizes. 621 */ 622 List<Size> candidateSupportedJpegSizes = new ArrayList<>(); 623 for (Size supportedJpegSize : supportedJpegSizes) { 624 if (supportedJpegSize.getWidth() >= maxConfiguredJpegWidth && 625 supportedJpegSize.getHeight() >= maxConfiguredJpegHeight) { 626 candidateSupportedJpegSizes.add(supportedJpegSize); 627 } 628 } 629 630 if (candidateSupportedJpegSizes.isEmpty()) { 631 throw new AssertionError( 632 "Could not find any supported JPEG sizes large enough to fit " + 633 smallestBoundJpegSize); 634 } 635 636 Size smallestSupportedJpegSize = Collections.min(candidateSupportedJpegSizes, 637 new SizeAreaComparator()); 638 639 if (!smallestSupportedJpegSize.equals(smallestBoundJpegSize)) { 640 Log.w(TAG, 641 String.format( 642 "configureOutputs - Will need to crop picture %s into " 643 + "smallest bound size %s", 644 smallestSupportedJpegSize, smallestBoundJpegSize)); 645 } 646 647 return smallestSupportedJpegSize; 648 } 649 650 return null; 651 } 652 checkAspectRatiosMatch(Size a, Size b)653 private static boolean checkAspectRatiosMatch(Size a, Size b) { 654 float aAspect = a.getWidth() / (float) a.getHeight(); 655 float bAspect = b.getWidth() / (float) b.getHeight(); 656 657 return Math.abs(aAspect - bAspect) < ASPECT_RATIO_TOLERANCE; 658 } 659 660 // Calculate the highest FPS range supported getPhotoPreviewFpsRange(List<int[]> frameRates)661 private int[] getPhotoPreviewFpsRange(List<int[]> frameRates) { 662 if (frameRates.size() == 0) { 663 Log.e(TAG, "No supported frame rates returned!"); 664 return null; 665 } 666 667 int bestMin = 0; 668 int bestMax = 0; 669 int bestIndex = 0; 670 int index = 0; 671 for (int[] rate : frameRates) { 672 int minFps = rate[Camera.Parameters.PREVIEW_FPS_MIN_INDEX]; 673 int maxFps = rate[Camera.Parameters.PREVIEW_FPS_MAX_INDEX]; 674 if (maxFps > bestMax || (maxFps == bestMax && minFps > bestMin)) { 675 bestMin = minFps; 676 bestMax = maxFps; 677 bestIndex = index; 678 } 679 index++; 680 } 681 682 return frameRates.get(bestIndex); 683 } 684 685 private final Handler.Callback mRequestHandlerCb = new Handler.Callback() { 686 private boolean mCleanup = false; 687 private final LegacyResultMapper mMapper = new LegacyResultMapper(); 688 689 @Override 690 public boolean handleMessage(Message msg) { 691 if (mCleanup) { 692 return true; 693 } 694 695 if (DEBUG) { 696 Log.d(TAG, "Request thread handling message:" + msg.what); 697 } 698 long startTime = 0; 699 if (DEBUG) { 700 startTime = SystemClock.elapsedRealtimeNanos(); 701 } 702 switch (msg.what) { 703 case MSG_CONFIGURE_OUTPUTS: 704 ConfigureHolder config = (ConfigureHolder) msg.obj; 705 int sizes = config.surfaces != null ? config.surfaces.size() : 0; 706 Log.i(TAG, "Configure outputs: " + sizes + " surfaces configured."); 707 708 try { 709 boolean success = mCaptureCollector.waitForEmpty(JPEG_FRAME_TIMEOUT, 710 TimeUnit.MILLISECONDS); 711 if (!success) { 712 Log.e(TAG, "Timed out while queueing configure request."); 713 mCaptureCollector.failAll(); 714 } 715 } catch (InterruptedException e) { 716 Log.e(TAG, "Interrupted while waiting for requests to complete."); 717 mDeviceState.setError( 718 CameraDeviceImpl.CameraDeviceCallbacks.ERROR_CAMERA_DEVICE); 719 break; 720 } 721 722 configureOutputs(config.surfaces); 723 config.condition.open(); 724 if (DEBUG) { 725 long totalTime = SystemClock.elapsedRealtimeNanos() - startTime; 726 Log.d(TAG, "Configure took " + totalTime + " ns"); 727 } 728 break; 729 case MSG_SUBMIT_CAPTURE_REQUEST: 730 Handler handler = RequestThreadManager.this.mRequestThread.getHandler(); 731 boolean anyRequestOutputAbandoned = false; 732 733 // Get the next burst from the request queue. 734 RequestQueue.RequestQueueEntry nextBurst = mRequestQueue.getNext(); 735 736 if (nextBurst == null) { 737 // If there are no further requests queued, wait for any currently executing 738 // requests to complete, then switch to idle state. 739 try { 740 boolean success = mCaptureCollector.waitForEmpty(JPEG_FRAME_TIMEOUT, 741 TimeUnit.MILLISECONDS); 742 if (!success) { 743 Log.e(TAG, 744 "Timed out while waiting for prior requests to complete."); 745 mCaptureCollector.failAll(); 746 } 747 } catch (InterruptedException e) { 748 Log.e(TAG, "Interrupted while waiting for requests to complete: ", e); 749 mDeviceState.setError( 750 CameraDeviceImpl.CameraDeviceCallbacks.ERROR_CAMERA_DEVICE); 751 break; 752 } 753 754 synchronized (mIdleLock) { 755 // Retry the the request queue. 756 nextBurst = mRequestQueue.getNext(); 757 758 // If we still have no queued requests, go idle. 759 if (nextBurst == null) { 760 mDeviceState.setIdle(); 761 break; 762 } 763 } 764 } 765 766 if (nextBurst != null) { 767 // Queue another capture if we did not get the last burst. 768 handler.sendEmptyMessage(MSG_SUBMIT_CAPTURE_REQUEST); 769 770 // Check whether capture queue becomes empty 771 if (nextBurst.isQueueEmpty()) { 772 mDeviceState.setRequestQueueEmpty(); 773 } 774 } 775 776 // Complete each request in the burst 777 BurstHolder burstHolder = nextBurst.getBurstHolder(); 778 List<RequestHolder> requests = 779 burstHolder.produceRequestHolders(nextBurst.getFrameNumber()); 780 for (RequestHolder holder : requests) { 781 CaptureRequest request = holder.getRequest(); 782 783 boolean paramsChanged = false; 784 785 // Only update parameters if the request has changed 786 if (mLastRequest == null || mLastRequest.captureRequest != request) { 787 788 // The intermediate buffer is sometimes null, but we always need 789 // the Camera1 API configured preview size 790 Size previewSize = ParameterUtils.convertSize(mParams.getPreviewSize()); 791 792 LegacyRequest legacyRequest = new LegacyRequest(mCharacteristics, 793 request, previewSize, mParams); // params are copied 794 795 796 // Parameters are mutated as a side-effect 797 LegacyMetadataMapper.convertRequestMetadata(/*inout*/legacyRequest); 798 799 // If the parameters have changed, set them in the Camera1 API. 800 if (!mParams.same(legacyRequest.parameters)) { 801 try { 802 mCamera.setParameters(legacyRequest.parameters); 803 } catch (RuntimeException e) { 804 // If setting the parameters failed, report a request error to 805 // the camera client, and skip any further work for this request 806 Log.e(TAG, "Exception while setting camera parameters: ", e); 807 holder.failRequest(); 808 mDeviceState.setCaptureStart(holder, /*timestamp*/0, 809 CameraDeviceImpl.CameraDeviceCallbacks. 810 ERROR_CAMERA_REQUEST); 811 continue; 812 } 813 paramsChanged = true; 814 mParams = legacyRequest.parameters; 815 } 816 817 mLastRequest = legacyRequest; 818 } 819 820 try { 821 boolean success = mCaptureCollector.queueRequest(holder, 822 mLastRequest, JPEG_FRAME_TIMEOUT, TimeUnit.MILLISECONDS); 823 824 if (!success) { 825 // Report a request error if we timed out while queuing this. 826 Log.e(TAG, "Timed out while queueing capture request."); 827 holder.failRequest(); 828 mDeviceState.setCaptureStart(holder, /*timestamp*/0, 829 CameraDeviceImpl.CameraDeviceCallbacks. 830 ERROR_CAMERA_REQUEST); 831 continue; 832 } 833 834 // Starting the preview needs to happen before enabling 835 // face detection or auto focus 836 if (holder.hasPreviewTargets()) { 837 doPreviewCapture(holder); 838 } 839 if (holder.hasJpegTargets()) { 840 while(!mCaptureCollector.waitForPreviewsEmpty(PREVIEW_FRAME_TIMEOUT, 841 TimeUnit.MILLISECONDS)) { 842 // Fail preview requests until the queue is empty. 843 Log.e(TAG, "Timed out while waiting for preview requests to " + 844 "complete."); 845 mCaptureCollector.failNextPreview(); 846 } 847 mReceivedJpeg.close(); 848 doJpegCapturePrepare(holder); 849 } 850 851 /* 852 * Do all the actions that require a preview to have been started 853 */ 854 855 // Toggle face detection on/off 856 // - do this before AF to give AF a chance to use faces 857 mFaceDetectMapper.processFaceDetectMode(request, /*in*/mParams); 858 859 // Unconditionally process AF triggers, since they're non-idempotent 860 // - must be done after setting the most-up-to-date AF mode 861 mFocusStateMapper.processRequestTriggers(request, mParams); 862 863 if (holder.hasJpegTargets()) { 864 doJpegCapture(holder); 865 if (!mReceivedJpeg.block(JPEG_FRAME_TIMEOUT)) { 866 Log.e(TAG, "Hit timeout for jpeg callback!"); 867 mCaptureCollector.failNextJpeg(); 868 } 869 } 870 871 } catch (IOException e) { 872 Log.e(TAG, "Received device exception during capture call: ", e); 873 mDeviceState.setError( 874 CameraDeviceImpl.CameraDeviceCallbacks.ERROR_CAMERA_DEVICE); 875 break; 876 } catch (InterruptedException e) { 877 Log.e(TAG, "Interrupted during capture: ", e); 878 mDeviceState.setError( 879 CameraDeviceImpl.CameraDeviceCallbacks.ERROR_CAMERA_DEVICE); 880 break; 881 } catch (RuntimeException e) { 882 Log.e(TAG, "Received device exception during capture call: ", e); 883 mDeviceState.setError( 884 CameraDeviceImpl.CameraDeviceCallbacks.ERROR_CAMERA_DEVICE); 885 break; 886 } 887 888 if (paramsChanged) { 889 if (DEBUG) { 890 Log.d(TAG, "Params changed -- getting new Parameters from HAL."); 891 } 892 try { 893 mParams = mCamera.getParameters(); 894 } catch (RuntimeException e) { 895 Log.e(TAG, "Received device exception: ", e); 896 mDeviceState.setError( 897 CameraDeviceImpl.CameraDeviceCallbacks.ERROR_CAMERA_DEVICE); 898 break; 899 } 900 901 // Update parameters to the latest that we think the camera is using 902 mLastRequest.setParameters(mParams); 903 } 904 905 MutableLong timestampMutable = new MutableLong(/*value*/0L); 906 try { 907 boolean success = mCaptureCollector.waitForRequestCompleted(holder, 908 REQUEST_COMPLETE_TIMEOUT, TimeUnit.MILLISECONDS, 909 /*out*/timestampMutable); 910 911 if (!success) { 912 Log.e(TAG, "Timed out while waiting for request to complete."); 913 mCaptureCollector.failAll(); 914 } 915 } catch (InterruptedException e) { 916 Log.e(TAG, "Interrupted waiting for request completion: ", e); 917 mDeviceState.setError( 918 CameraDeviceImpl.CameraDeviceCallbacks.ERROR_CAMERA_DEVICE); 919 break; 920 } 921 922 CameraMetadataNative result = mMapper.cachedConvertResultMetadata( 923 mLastRequest, timestampMutable.value); 924 /* 925 * Order matters: The default result mapper is state-less; the 926 * other mappers carry state and may override keys set by the default 927 * mapper with their own values. 928 */ 929 930 // Update AF state 931 mFocusStateMapper.mapResultTriggers(result); 932 // Update face-related results 933 mFaceDetectMapper.mapResultFaces(result, mLastRequest); 934 935 if (!holder.requestFailed()) { 936 mDeviceState.setCaptureResult(holder, result); 937 } 938 939 if (holder.isOutputAbandoned()) { 940 anyRequestOutputAbandoned = true; 941 } 942 } 943 944 // Stop the repeating request if any of its output surfaces is abandoned. 945 if (anyRequestOutputAbandoned && burstHolder.isRepeating()) { 946 long lastFrameNumber = cancelRepeating(burstHolder.getRequestId()); 947 if (DEBUG) { 948 Log.d(TAG, "Stopped repeating request. Last frame number is " + 949 lastFrameNumber); 950 } 951 if (lastFrameNumber != RequestQueue.INVALID_FRAME) { 952 mDeviceState.setRepeatingRequestError(lastFrameNumber, 953 burstHolder.getRequestId()); 954 } else { 955 Log.e(TAG, "Repeating request id: " + burstHolder.getRequestId() + 956 " already canceled!"); 957 } 958 } 959 960 if (DEBUG) { 961 long totalTime = SystemClock.elapsedRealtimeNanos() - startTime; 962 Log.d(TAG, "Capture request took " + totalTime + " ns"); 963 mRequestCounter.countAndLog(); 964 } 965 break; 966 case MSG_CLEANUP: 967 mCleanup = true; 968 try { 969 boolean success = mCaptureCollector.waitForEmpty(JPEG_FRAME_TIMEOUT, 970 TimeUnit.MILLISECONDS); 971 if (!success) { 972 Log.e(TAG, "Timed out while queueing cleanup request."); 973 mCaptureCollector.failAll(); 974 } 975 } catch (InterruptedException e) { 976 Log.e(TAG, "Interrupted while waiting for requests to complete: ", e); 977 mDeviceState.setError( 978 CameraDeviceImpl.CameraDeviceCallbacks.ERROR_CAMERA_DEVICE); 979 } 980 if (mGLThreadManager != null) { 981 mGLThreadManager.quit(); 982 mGLThreadManager = null; 983 } 984 disconnectCallbackSurfaces(); 985 if (mCamera != null) { 986 mCamera.release(); 987 mCamera = null; 988 } 989 break; 990 case RequestHandlerThread.MSG_POKE_IDLE_HANDLER: 991 // OK: Ignore message. 992 break; 993 default: 994 throw new AssertionError("Unhandled message " + msg.what + 995 " on RequestThread."); 996 } 997 return true; 998 } 999 }; 1000 1001 /** 1002 * Create a new RequestThreadManager. 1003 * 1004 * @param cameraId the id of the camera to use. 1005 * @param camera an open camera object. The RequestThreadManager takes ownership of this camera 1006 * object, and is responsible for closing it. 1007 * @param characteristics the static camera characteristics corresponding to this camera device 1008 * @param deviceState a {@link CameraDeviceState} state machine. 1009 */ RequestThreadManager(int cameraId, Camera camera, CameraCharacteristics characteristics, CameraDeviceState deviceState)1010 public RequestThreadManager(int cameraId, Camera camera, CameraCharacteristics characteristics, 1011 CameraDeviceState deviceState) { 1012 mCamera = checkNotNull(camera, "camera must not be null"); 1013 mCameraId = cameraId; 1014 mCharacteristics = checkNotNull(characteristics, "characteristics must not be null"); 1015 String name = String.format("RequestThread-%d", cameraId); 1016 TAG = name; 1017 mDeviceState = checkNotNull(deviceState, "deviceState must not be null"); 1018 mFocusStateMapper = new LegacyFocusStateMapper(mCamera); 1019 mFaceDetectMapper = new LegacyFaceDetectMapper(mCamera, mCharacteristics); 1020 mCaptureCollector = new CaptureCollector(MAX_IN_FLIGHT_REQUESTS, mDeviceState); 1021 mRequestThread = new RequestHandlerThread(name, mRequestHandlerCb); 1022 mCamera.setDetailedErrorCallback(mErrorCallback); 1023 } 1024 1025 /** 1026 * Start the request thread. 1027 */ start()1028 public void start() { 1029 mRequestThread.start(); 1030 } 1031 1032 /** 1033 * Flush any pending requests. 1034 * 1035 * @return the last frame number. 1036 */ flush()1037 public long flush() { 1038 Log.i(TAG, "Flushing all pending requests."); 1039 long lastFrame = mRequestQueue.stopRepeating(); 1040 mCaptureCollector.failAll(); 1041 return lastFrame; 1042 } 1043 1044 /** 1045 * Quit the request thread, and clean up everything. 1046 */ quit()1047 public void quit() { 1048 if (!mQuit.getAndSet(true)) { // Avoid sending messages on dead thread's handler. 1049 Handler handler = mRequestThread.waitAndGetHandler(); 1050 handler.sendMessageAtFrontOfQueue(handler.obtainMessage(MSG_CLEANUP)); 1051 mRequestThread.quitSafely(); 1052 try { 1053 mRequestThread.join(); 1054 } catch (InterruptedException e) { 1055 Log.e(TAG, String.format("Thread %s (%d) interrupted while quitting.", 1056 mRequestThread.getName(), mRequestThread.getId())); 1057 } 1058 } 1059 } 1060 1061 /** 1062 * Submit the given burst of requests to be captured. 1063 * 1064 * <p>If the burst is repeating, replace the current repeating burst.</p> 1065 * 1066 * @param requests the burst of requests to add to the queue. 1067 * @param repeating true if the burst is repeating. 1068 * @return the submission info, including the new request id, and the last frame number, which 1069 * contains either the frame number of the last frame that will be returned for this request, 1070 * or the frame number of the last frame that will be returned for the current repeating 1071 * request if this burst is set to be repeating. 1072 */ submitCaptureRequests(CaptureRequest[] requests, boolean repeating)1073 public SubmitInfo submitCaptureRequests(CaptureRequest[] requests, boolean repeating) { 1074 Handler handler = mRequestThread.waitAndGetHandler(); 1075 SubmitInfo info; 1076 synchronized (mIdleLock) { 1077 info = mRequestQueue.submit(requests, repeating); 1078 handler.sendEmptyMessage(MSG_SUBMIT_CAPTURE_REQUEST); 1079 } 1080 return info; 1081 } 1082 1083 /** 1084 * Cancel a repeating request. 1085 * 1086 * @param requestId the id of the repeating request to cancel. 1087 * @return the last frame to be returned from the HAL for the given repeating request, or 1088 * {@code INVALID_FRAME} if none exists. 1089 */ cancelRepeating(int requestId)1090 public long cancelRepeating(int requestId) { 1091 return mRequestQueue.stopRepeating(requestId); 1092 } 1093 1094 /** 1095 * Configure with the current list of output Surfaces. 1096 * 1097 * <p> 1098 * This operation blocks until the configuration is complete. 1099 * </p> 1100 * 1101 * <p>Using a {@code null} or empty {@code outputs} list is the equivalent of unconfiguring.</p> 1102 * 1103 * @param outputs a {@link java.util.Collection} of outputs to configure. 1104 */ configure(Collection<Pair<Surface, Size>> outputs)1105 public void configure(Collection<Pair<Surface, Size>> outputs) { 1106 Handler handler = mRequestThread.waitAndGetHandler(); 1107 final ConditionVariable condition = new ConditionVariable(/*closed*/false); 1108 ConfigureHolder holder = new ConfigureHolder(condition, outputs); 1109 handler.sendMessage(handler.obtainMessage(MSG_CONFIGURE_OUTPUTS, 0, 0, holder)); 1110 condition.block(); 1111 } 1112 } 1113