1 /* 2 * Copyright (C) 2013 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package android.media; 18 19 import android.annotation.IntRange; 20 import android.annotation.NonNull; 21 import android.graphics.ImageFormat; 22 import android.graphics.ImageFormat.Format; 23 import android.hardware.HardwareBuffer; 24 import android.hardware.HardwareBuffer.Usage; 25 import android.os.Handler; 26 import android.os.Looper; 27 import android.os.Message; 28 import android.view.Surface; 29 30 import dalvik.system.VMRuntime; 31 32 import java.lang.ref.WeakReference; 33 import java.nio.ByteBuffer; 34 import java.nio.ByteOrder; 35 import java.nio.NioUtils; 36 import java.util.List; 37 import java.util.concurrent.CopyOnWriteArrayList; 38 import java.util.concurrent.atomic.AtomicBoolean; 39 40 /** 41 * <p>The ImageReader class allows direct application access to image data 42 * rendered into a {@link android.view.Surface}</p> 43 * 44 * <p>Several Android media API classes accept Surface objects as targets to 45 * render to, including {@link MediaPlayer}, {@link MediaCodec}, 46 * {@link android.hardware.camera2.CameraDevice}, {@link ImageWriter} and 47 * {@link android.renderscript.Allocation RenderScript Allocations}. The image 48 * sizes and formats that can be used with each source vary, and should be 49 * checked in the documentation for the specific API.</p> 50 * 51 * <p>The image data is encapsulated in {@link Image} objects, and multiple such 52 * objects can be accessed at the same time, up to the number specified by the 53 * {@code maxImages} constructor parameter. New images sent to an ImageReader 54 * through its {@link Surface} are queued until accessed through the {@link #acquireLatestImage} 55 * or {@link #acquireNextImage} call. Due to memory limits, an image source will 56 * eventually stall or drop Images in trying to render to the Surface if the 57 * ImageReader does not obtain and release Images at a rate equal to the 58 * production rate.</p> 59 */ 60 public class ImageReader implements AutoCloseable { 61 62 /** 63 * Returned by nativeImageSetup when acquiring the image was successful. 64 */ 65 private static final int ACQUIRE_SUCCESS = 0; 66 /** 67 * Returned by nativeImageSetup when we couldn't acquire the buffer, 68 * because there were no buffers available to acquire. 69 */ 70 private static final int ACQUIRE_NO_BUFS = 1; 71 /** 72 * Returned by nativeImageSetup when we couldn't acquire the buffer 73 * because the consumer has already acquired {@maxImages} and cannot 74 * acquire more than that. 75 */ 76 private static final int ACQUIRE_MAX_IMAGES = 2; 77 78 /** 79 * <p> 80 * Create a new reader for images of the desired size and format. 81 * </p> 82 * <p> 83 * The {@code maxImages} parameter determines the maximum number of 84 * {@link Image} objects that can be be acquired from the 85 * {@code ImageReader} simultaneously. Requesting more buffers will use up 86 * more memory, so it is important to use only the minimum number necessary 87 * for the use case. 88 * </p> 89 * <p> 90 * The valid sizes and formats depend on the source of the image data. 91 * </p> 92 * <p> 93 * If the {@code format} is {@link ImageFormat#PRIVATE PRIVATE}, the created 94 * {@link ImageReader} will produce images that are not directly accessible 95 * by the application. The application can still acquire images from this 96 * {@link ImageReader}, and send them to the 97 * {@link android.hardware.camera2.CameraDevice camera} for reprocessing via 98 * {@link ImageWriter} interface. However, the {@link Image#getPlanes() 99 * getPlanes()} will return an empty array for {@link ImageFormat#PRIVATE 100 * PRIVATE} format images. The application can check if an existing reader's 101 * format by calling {@link #getImageFormat()}. 102 * </p> 103 * <p> 104 * {@link ImageFormat#PRIVATE PRIVATE} format {@link ImageReader 105 * ImageReaders} are more efficient to use when application access to image 106 * data is not necessary, compared to ImageReaders using other format such 107 * as {@link ImageFormat#YUV_420_888 YUV_420_888}. 108 * </p> 109 * 110 * @param width The default width in pixels of the Images that this reader 111 * will produce. 112 * @param height The default height in pixels of the Images that this reader 113 * will produce. 114 * @param format The format of the Image that this reader will produce. This 115 * must be one of the {@link android.graphics.ImageFormat} or 116 * {@link android.graphics.PixelFormat} constants. Note that not 117 * all formats are supported, like ImageFormat.NV21. 118 * @param maxImages The maximum number of images the user will want to 119 * access simultaneously. This should be as small as possible to 120 * limit memory use. Once maxImages Images are obtained by the 121 * user, one of them has to be released before a new Image will 122 * become available for access through 123 * {@link #acquireLatestImage()} or {@link #acquireNextImage()}. 124 * Must be greater than 0. 125 * @see Image 126 */ newInstance( @ntRangefrom = 1) int width, @IntRange(from = 1) int height, @Format int format, @IntRange(from = 1) int maxImages)127 public static @NonNull ImageReader newInstance( 128 @IntRange(from = 1) int width, 129 @IntRange(from = 1) int height, 130 @Format int format, 131 @IntRange(from = 1) int maxImages) { 132 // If the format is private don't default to USAGE_CPU_READ_OFTEN since it may not 133 // work, and is inscrutable anyway 134 return new ImageReader(width, height, format, maxImages, 135 format == ImageFormat.PRIVATE ? 0 : HardwareBuffer.USAGE_CPU_READ_OFTEN); 136 } 137 138 /** 139 * <p> 140 * Create a new reader for images of the desired size, format and consumer usage flag. 141 * </p> 142 * <p> 143 * The {@code maxImages} parameter determines the maximum number of {@link Image} objects that 144 * can be be acquired from the {@code ImageReader} simultaneously. Requesting more buffers will 145 * use up more memory, so it is important to use only the minimum number necessary for the use 146 * case. 147 * </p> 148 * <p> 149 * The valid sizes and formats depend on the source of the image data. 150 * </p> 151 * <p> 152 * The format and usage flag combination describes how the buffer will be used by 153 * consumer end-points. For example, if the application intends to send the images to 154 * {@link android.media.MediaCodec} or {@link android.media.MediaRecorder} for hardware video 155 * encoding, the format and usage flag combination needs to be 156 * {@link ImageFormat#PRIVATE PRIVATE} and {@link HardwareBuffer#USAGE_VIDEO_ENCODE}. When an 157 * {@link ImageReader} object is created with a valid size and such format/usage flag 158 * combination, the application can send the {@link Image images} to an {@link ImageWriter} that 159 * is created with the input {@link android.view.Surface} provided by the 160 * {@link android.media.MediaCodec} or {@link android.media.MediaRecorder}. 161 * </p> 162 * <p> 163 * If the {@code format} is {@link ImageFormat#PRIVATE PRIVATE}, the created {@link ImageReader} 164 * will produce images that are not directly accessible by the application. The application can 165 * still acquire images from this {@link ImageReader}, and send them to the 166 * {@link android.hardware.camera2.CameraDevice camera} for reprocessing, or to the 167 * {@link android.media.MediaCodec} / {@link android.media.MediaRecorder} for hardware video 168 * encoding via {@link ImageWriter} interface. However, the {@link Image#getPlanes() 169 * getPlanes()} will return an empty array for {@link ImageFormat#PRIVATE PRIVATE} format 170 * images. The application can check if an existing reader's format by calling 171 * {@link #getImageFormat()}. 172 * </p> 173 * <p> 174 * {@link ImageFormat#PRIVATE PRIVATE} format {@link ImageReader ImageReaders} are more 175 * efficient to use when application access to image data is not necessary, compared to 176 * ImageReaders using other format such as {@link ImageFormat#YUV_420_888 YUV_420_888}. 177 * </p> 178 * <p> 179 * Note that not all format and usage flag combinations are supported by the 180 * {@link ImageReader}. Below are the supported combinations by the {@link ImageReader} 181 * (assuming the consumer end-points support the such image consumption, e.g., hardware video 182 * encoding). 183 * <table> 184 * <tr> 185 * <th>Format</th> 186 * <th>Compatible usage flags</th> 187 * </tr> 188 * <tr> 189 * <td>non-{@link android.graphics.ImageFormat#PRIVATE PRIVATE} formats defined by 190 * {@link android.graphics.ImageFormat ImageFormat} or 191 * {@link android.graphics.PixelFormat PixelFormat}</td> 192 * <td>{@link HardwareBuffer#USAGE_CPU_READ_RARELY} or 193 * {@link HardwareBuffer#USAGE_CPU_READ_OFTEN}</td> 194 * </tr> 195 * <tr> 196 * <td>{@link android.graphics.ImageFormat#PRIVATE}</td> 197 * <td>{@link HardwareBuffer#USAGE_VIDEO_ENCODE} or 198 * {@link HardwareBuffer#USAGE_GPU_SAMPLED_IMAGE}, or combined</td> 199 * </tr> 200 * </table> 201 * Using other combinations may result in {@link IllegalArgumentException}. 202 * </p> 203 * @param width The default width in pixels of the Images that this reader will produce. 204 * @param height The default height in pixels of the Images that this reader will produce. 205 * @param format The format of the Image that this reader will produce. This must be one of the 206 * {@link android.graphics.ImageFormat} or {@link android.graphics.PixelFormat} 207 * constants. Note that not all formats are supported, like ImageFormat.NV21. 208 * @param maxImages The maximum number of images the user will want to access simultaneously. 209 * This should be as small as possible to limit memory use. Once maxImages Images are 210 * obtained by the user, one of them has to be released before a new Image will 211 * become available for access through {@link #acquireLatestImage()} or 212 * {@link #acquireNextImage()}. Must be greater than 0. 213 * @param usage The intended usage of the images produced by this ImageReader. See the usages 214 * on {@link HardwareBuffer} for a list of valid usage bits. See also 215 * {@link HardwareBuffer#isSupported(int, int, int, int, long)} for checking 216 * if a combination is supported. If it's not supported this will throw 217 * an {@link IllegalArgumentException}. 218 * @see Image 219 * @see HardwareBuffer 220 */ newInstance( @ntRangefrom = 1) int width, @IntRange(from = 1) int height, @Format int format, @IntRange(from = 1) int maxImages, @Usage long usage)221 public static @NonNull ImageReader newInstance( 222 @IntRange(from = 1) int width, 223 @IntRange(from = 1) int height, 224 @Format int format, 225 @IntRange(from = 1) int maxImages, 226 @Usage long usage) { 227 // TODO: Check this - can't do it just yet because format support is different 228 // Unify formats! The only reliable way to validate usage is to just try it and see. 229 230 // if (!HardwareBuffer.isSupported(width, height, format, 1, usage)) { 231 // throw new IllegalArgumentException("The given format=" + Integer.toHexString(format) 232 // + " & usage=" + Long.toHexString(usage) + " is not supported"); 233 // } 234 return new ImageReader(width, height, format, maxImages, usage); 235 } 236 237 /** 238 * @hide 239 */ ImageReader(int width, int height, int format, int maxImages, long usage)240 protected ImageReader(int width, int height, int format, int maxImages, long usage) { 241 mWidth = width; 242 mHeight = height; 243 mFormat = format; 244 mMaxImages = maxImages; 245 246 if (width < 1 || height < 1) { 247 throw new IllegalArgumentException( 248 "The image dimensions must be positive"); 249 } 250 if (mMaxImages < 1) { 251 throw new IllegalArgumentException( 252 "Maximum outstanding image count must be at least 1"); 253 } 254 255 if (format == ImageFormat.NV21) { 256 throw new IllegalArgumentException( 257 "NV21 format is not supported"); 258 } 259 260 mNumPlanes = ImageUtils.getNumPlanesForFormat(mFormat); 261 262 nativeInit(new WeakReference<>(this), width, height, format, maxImages, usage); 263 264 mSurface = nativeGetSurface(); 265 266 mIsReaderValid = true; 267 // Estimate the native buffer allocation size and register it so it gets accounted for 268 // during GC. Note that this doesn't include the buffers required by the buffer queue 269 // itself and the buffers requested by the producer. 270 // Only include memory for 1 buffer, since actually accounting for the memory used is 271 // complex, and 1 buffer is enough for the VM to treat the ImageReader as being of some 272 // size. 273 mEstimatedNativeAllocBytes = ImageUtils.getEstimatedNativeAllocBytes( 274 width, height, format, /*buffer count*/ 1); 275 VMRuntime.getRuntime().registerNativeAllocation(mEstimatedNativeAllocBytes); 276 } 277 278 /** 279 * The default width of {@link Image Images}, in pixels. 280 * 281 * <p>The width may be overridden by the producer sending buffers to this 282 * ImageReader's Surface. If so, the actual width of the images can be 283 * found using {@link Image#getWidth}.</p> 284 * 285 * @return the expected width of an Image 286 */ getWidth()287 public int getWidth() { 288 return mWidth; 289 } 290 291 /** 292 * The default height of {@link Image Images}, in pixels. 293 * 294 * <p>The height may be overridden by the producer sending buffers to this 295 * ImageReader's Surface. If so, the actual height of the images can be 296 * found using {@link Image#getHeight}.</p> 297 * 298 * @return the expected height of an Image 299 */ getHeight()300 public int getHeight() { 301 return mHeight; 302 } 303 304 /** 305 * The default {@link ImageFormat image format} of {@link Image Images}. 306 * 307 * <p>Some color formats may be overridden by the producer sending buffers to 308 * this ImageReader's Surface if the default color format allows. ImageReader 309 * guarantees that all {@link Image Images} acquired from ImageReader 310 * (for example, with {@link #acquireNextImage}) will have a "compatible" 311 * format to what was specified in {@link #newInstance}. 312 * As of now, each format is only compatible to itself. 313 * The actual format of the images can be found using {@link Image#getFormat}.</p> 314 * 315 * @return the expected format of an Image 316 * 317 * @see ImageFormat 318 */ getImageFormat()319 public int getImageFormat() { 320 return mFormat; 321 } 322 323 /** 324 * Maximum number of images that can be acquired from the ImageReader by any time (for example, 325 * with {@link #acquireNextImage}). 326 * 327 * <p>An image is considered acquired after it's returned by a function from ImageReader, and 328 * until the Image is {@link Image#close closed} to release the image back to the ImageReader. 329 * </p> 330 * 331 * <p>Attempting to acquire more than {@code maxImages} concurrently will result in the 332 * acquire function throwing a {@link IllegalStateException}. Furthermore, 333 * while the max number of images have been acquired by the ImageReader user, the producer 334 * enqueueing additional images may stall until at least one image has been released. </p> 335 * 336 * @return Maximum number of images for this ImageReader. 337 * 338 * @see Image#close 339 */ getMaxImages()340 public int getMaxImages() { 341 return mMaxImages; 342 } 343 344 /** 345 * <p>Get a {@link Surface} that can be used to produce {@link Image Images} for this 346 * {@code ImageReader}.</p> 347 * 348 * <p>Until valid image data is rendered into this {@link Surface}, the 349 * {@link #acquireNextImage} method will return {@code null}. Only one source 350 * can be producing data into this Surface at the same time, although the 351 * same {@link Surface} can be reused with a different API once the first source is 352 * disconnected from the {@link Surface}.</p> 353 * 354 * <p>Please note that holding on to the Surface object returned by this method is not enough 355 * to keep its parent ImageReader from being reclaimed. In that sense, a Surface acts like a 356 * {@link java.lang.ref.WeakReference weak reference} to the ImageReader that provides it.</p> 357 * 358 * @return A {@link Surface} to use for a drawing target for various APIs. 359 */ getSurface()360 public Surface getSurface() { 361 return mSurface; 362 } 363 364 /** 365 * <p> 366 * Acquire the latest {@link Image} from the ImageReader's queue, dropping older 367 * {@link Image images}. Returns {@code null} if no new image is available. 368 * </p> 369 * <p> 370 * This operation will acquire all the images possible from the ImageReader, 371 * but {@link #close} all images that aren't the latest. This function is 372 * recommended to use over {@link #acquireNextImage} for most use-cases, as it's 373 * more suited for real-time processing. 374 * </p> 375 * <p> 376 * Note that {@link #getMaxImages maxImages} should be at least 2 for 377 * {@link #acquireLatestImage} to be any different than {@link #acquireNextImage} - 378 * discarding all-but-the-newest {@link Image} requires temporarily acquiring two 379 * {@link Image Images} at once. Or more generally, calling {@link #acquireLatestImage} 380 * with less than two images of margin, that is 381 * {@code (maxImages - currentAcquiredImages < 2)} will not discard as expected. 382 * </p> 383 * <p> 384 * This operation will fail by throwing an {@link IllegalStateException} if 385 * {@code maxImages} have been acquired with {@link #acquireLatestImage} or 386 * {@link #acquireNextImage}. In particular a sequence of {@link #acquireLatestImage} 387 * calls greater than {@link #getMaxImages} without calling {@link Image#close} in-between 388 * will exhaust the underlying queue. At such a time, {@link IllegalStateException} 389 * will be thrown until more images are 390 * released with {@link Image#close}. 391 * </p> 392 * 393 * @return latest frame of image data, or {@code null} if no image data is available. 394 * @throws IllegalStateException if too many images are currently acquired 395 */ acquireLatestImage()396 public Image acquireLatestImage() { 397 Image image = acquireNextImage(); 398 if (image == null) { 399 return null; 400 } 401 try { 402 for (;;) { 403 Image next = acquireNextImageNoThrowISE(); 404 if (next == null) { 405 Image result = image; 406 image = null; 407 return result; 408 } 409 image.close(); 410 image = next; 411 } 412 } finally { 413 if (image != null) { 414 image.close(); 415 } 416 } 417 } 418 419 /** 420 * Don't throw IllegalStateException if there are too many images acquired. 421 * 422 * @return Image if acquiring succeeded, or null otherwise. 423 * 424 * @hide 425 */ acquireNextImageNoThrowISE()426 public Image acquireNextImageNoThrowISE() { 427 SurfaceImage si = new SurfaceImage(mFormat); 428 return acquireNextSurfaceImage(si) == ACQUIRE_SUCCESS ? si : null; 429 } 430 431 /** 432 * Attempts to acquire the next image from the underlying native implementation. 433 * 434 * <p> 435 * Note that unexpected failures will throw at the JNI level. 436 * </p> 437 * 438 * @param si A blank SurfaceImage. 439 * @return One of the {@code ACQUIRE_*} codes that determine success or failure. 440 * 441 * @see #ACQUIRE_MAX_IMAGES 442 * @see #ACQUIRE_NO_BUFS 443 * @see #ACQUIRE_SUCCESS 444 */ acquireNextSurfaceImage(SurfaceImage si)445 private int acquireNextSurfaceImage(SurfaceImage si) { 446 synchronized (mCloseLock) { 447 // A null image will eventually be returned if ImageReader is already closed. 448 int status = ACQUIRE_NO_BUFS; 449 if (mIsReaderValid) { 450 status = nativeImageSetup(si); 451 } 452 453 switch (status) { 454 case ACQUIRE_SUCCESS: 455 si.mIsImageValid = true; 456 case ACQUIRE_NO_BUFS: 457 case ACQUIRE_MAX_IMAGES: 458 break; 459 default: 460 throw new AssertionError("Unknown nativeImageSetup return code " + status); 461 } 462 463 // Only keep track the successfully acquired image, as the native buffer is only mapped 464 // for such case. 465 if (status == ACQUIRE_SUCCESS) { 466 mAcquiredImages.add(si); 467 } 468 return status; 469 } 470 } 471 472 /** 473 * <p> 474 * Acquire the next Image from the ImageReader's queue. Returns {@code null} if 475 * no new image is available. 476 * </p> 477 * 478 * <p><i>Warning:</i> Consider using {@link #acquireLatestImage()} instead, as it will 479 * automatically release older images, and allow slower-running processing routines to catch 480 * up to the newest frame. Usage of {@link #acquireNextImage} is recommended for 481 * batch/background processing. Incorrectly using this function can cause images to appear 482 * with an ever-increasing delay, followed by a complete stall where no new images seem to 483 * appear. 484 * </p> 485 * 486 * <p> 487 * This operation will fail by throwing an {@link IllegalStateException} if 488 * {@code maxImages} have been acquired with {@link #acquireNextImage} or 489 * {@link #acquireLatestImage}. In particular a sequence of {@link #acquireNextImage} or 490 * {@link #acquireLatestImage} calls greater than {@link #getMaxImages maxImages} without 491 * calling {@link Image#close} in-between will exhaust the underlying queue. At such a time, 492 * {@link IllegalStateException} will be thrown until more images are released with 493 * {@link Image#close}. 494 * </p> 495 * 496 * @return a new frame of image data, or {@code null} if no image data is available. 497 * @throws IllegalStateException if {@code maxImages} images are currently acquired 498 * @see #acquireLatestImage 499 */ acquireNextImage()500 public Image acquireNextImage() { 501 // Initialize with reader format, but can be overwritten by native if the image 502 // format is different from the reader format. 503 SurfaceImage si = new SurfaceImage(mFormat); 504 int status = acquireNextSurfaceImage(si); 505 506 switch (status) { 507 case ACQUIRE_SUCCESS: 508 return si; 509 case ACQUIRE_NO_BUFS: 510 return null; 511 case ACQUIRE_MAX_IMAGES: 512 throw new IllegalStateException( 513 String.format( 514 "maxImages (%d) has already been acquired, " + 515 "call #close before acquiring more.", mMaxImages)); 516 default: 517 throw new AssertionError("Unknown nativeImageSetup return code " + status); 518 } 519 } 520 521 /** 522 * <p>Return the frame to the ImageReader for reuse.</p> 523 */ releaseImage(Image i)524 private void releaseImage(Image i) { 525 if (! (i instanceof SurfaceImage) ) { 526 throw new IllegalArgumentException( 527 "This image was not produced by an ImageReader"); 528 } 529 SurfaceImage si = (SurfaceImage) i; 530 if (si.mIsImageValid == false) { 531 return; 532 } 533 534 if (si.getReader() != this || !mAcquiredImages.contains(i)) { 535 throw new IllegalArgumentException( 536 "This image was not produced by this ImageReader"); 537 } 538 539 si.clearSurfacePlanes(); 540 nativeReleaseImage(i); 541 si.mIsImageValid = false; 542 mAcquiredImages.remove(i); 543 } 544 545 /** 546 * Register a listener to be invoked when a new image becomes available 547 * from the ImageReader. 548 * 549 * @param listener 550 * The listener that will be run. 551 * @param handler 552 * The handler on which the listener should be invoked, or null 553 * if the listener should be invoked on the calling thread's looper. 554 * @throws IllegalArgumentException 555 * If no handler specified and the calling thread has no looper. 556 */ setOnImageAvailableListener(OnImageAvailableListener listener, Handler handler)557 public void setOnImageAvailableListener(OnImageAvailableListener listener, Handler handler) { 558 synchronized (mListenerLock) { 559 if (listener != null) { 560 Looper looper = handler != null ? handler.getLooper() : Looper.myLooper(); 561 if (looper == null) { 562 throw new IllegalArgumentException( 563 "handler is null but the current thread is not a looper"); 564 } 565 if (mListenerHandler == null || mListenerHandler.getLooper() != looper) { 566 mListenerHandler = new ListenerHandler(looper); 567 } 568 mListener = listener; 569 } else { 570 mListener = null; 571 mListenerHandler = null; 572 } 573 } 574 } 575 576 /** 577 * Callback interface for being notified that a new image is available. 578 * 579 * <p> 580 * The onImageAvailable is called per image basis, that is, callback fires for every new frame 581 * available from ImageReader. 582 * </p> 583 */ 584 public interface OnImageAvailableListener { 585 /** 586 * Callback that is called when a new image is available from ImageReader. 587 * 588 * @param reader the ImageReader the callback is associated with. 589 * @see ImageReader 590 * @see Image 591 */ onImageAvailable(ImageReader reader)592 void onImageAvailable(ImageReader reader); 593 } 594 595 /** 596 * Free up all the resources associated with this ImageReader. 597 * 598 * <p> 599 * After calling this method, this ImageReader can not be used. Calling 600 * any methods on this ImageReader and Images previously provided by 601 * {@link #acquireNextImage} or {@link #acquireLatestImage} 602 * will result in an {@link IllegalStateException}, and attempting to read from 603 * {@link ByteBuffer ByteBuffers} returned by an earlier 604 * {@link Image.Plane#getBuffer Plane#getBuffer} call will 605 * have undefined behavior. 606 * </p> 607 */ 608 @Override close()609 public void close() { 610 setOnImageAvailableListener(null, null); 611 if (mSurface != null) mSurface.release(); 612 613 /** 614 * Close all outstanding acquired images before closing the ImageReader. It is a good 615 * practice to close all the images as soon as it is not used to reduce system instantaneous 616 * memory pressure. CopyOnWrite list will use a copy of current list content. For the images 617 * being closed by other thread (e.g., GC thread), doubling the close call is harmless. For 618 * the image being acquired by other threads, mCloseLock is used to synchronize close and 619 * acquire operations. 620 */ 621 synchronized (mCloseLock) { 622 mIsReaderValid = false; 623 for (Image image : mAcquiredImages) { 624 image.close(); 625 } 626 mAcquiredImages.clear(); 627 628 nativeClose(); 629 630 if (mEstimatedNativeAllocBytes > 0) { 631 VMRuntime.getRuntime().registerNativeFree(mEstimatedNativeAllocBytes); 632 mEstimatedNativeAllocBytes = 0; 633 } 634 } 635 } 636 637 /** 638 * Discard any free buffers owned by this ImageReader. 639 * 640 * <p> 641 * Generally, the ImageReader caches buffers for reuse once they have been 642 * allocated, for best performance. However, sometimes it may be important to 643 * release all the cached, unused buffers to save on memory. 644 * </p> 645 * <p> 646 * Calling this method will discard all free cached buffers. This does not include any buffers 647 * associated with Images acquired from the ImageReader, any filled buffers waiting to be 648 * acquired, and any buffers currently in use by the source rendering buffers into the 649 * ImageReader's Surface. 650 * <p> 651 * The ImageReader continues to be usable after this call, but may need to reallocate buffers 652 * when more buffers are needed for rendering. 653 * </p> 654 */ discardFreeBuffers()655 public void discardFreeBuffers() { 656 synchronized (mCloseLock) { 657 nativeDiscardFreeBuffers(); 658 } 659 } 660 661 @Override finalize()662 protected void finalize() throws Throwable { 663 try { 664 close(); 665 } finally { 666 super.finalize(); 667 } 668 } 669 670 /** 671 * <p> 672 * Remove the ownership of this image from the ImageReader. 673 * </p> 674 * <p> 675 * After this call, the ImageReader no longer owns this image, and the image 676 * ownership can be transfered to another entity like {@link ImageWriter} 677 * via {@link ImageWriter#queueInputImage}. It's up to the new owner to 678 * release the resources held by this image. For example, if the ownership 679 * of this image is transfered to an {@link ImageWriter}, the image will be 680 * freed by the ImageWriter after the image data consumption is done. 681 * </p> 682 * <p> 683 * This method can be used to achieve zero buffer copy for use cases like 684 * {@link android.hardware.camera2.CameraDevice Camera2 API} PRIVATE and YUV 685 * reprocessing, where the application can select an output image from 686 * {@link ImageReader} and transfer this image directly to 687 * {@link ImageWriter}, where this image can be consumed by camera directly. 688 * For PRIVATE reprocessing, this is the only way to send input buffers to 689 * the {@link android.hardware.camera2.CameraDevice camera} for 690 * reprocessing. 691 * </p> 692 * <p> 693 * This is a package private method that is only used internally. 694 * </p> 695 * 696 * @param image The image to be detached from this ImageReader. 697 * @throws IllegalStateException If the ImageReader or image have been 698 * closed, or the has been detached, or has not yet been 699 * acquired. 700 */ detachImage(Image image)701 void detachImage(Image image) { 702 if (image == null) { 703 throw new IllegalArgumentException("input image must not be null"); 704 } 705 if (!isImageOwnedbyMe(image)) { 706 throw new IllegalArgumentException("Trying to detach an image that is not owned by" 707 + " this ImageReader"); 708 } 709 710 SurfaceImage si = (SurfaceImage) image; 711 si.throwISEIfImageIsInvalid(); 712 713 if (si.isAttachable()) { 714 throw new IllegalStateException("Image was already detached from this ImageReader"); 715 } 716 717 nativeDetachImage(image); 718 si.clearSurfacePlanes(); 719 si.mPlanes = null; 720 si.setDetached(true); 721 } 722 isImageOwnedbyMe(Image image)723 private boolean isImageOwnedbyMe(Image image) { 724 if (!(image instanceof SurfaceImage)) { 725 return false; 726 } 727 SurfaceImage si = (SurfaceImage) image; 728 return si.getReader() == this; 729 } 730 731 /** 732 * Called from Native code when an Event happens. 733 * 734 * This may be called from an arbitrary Binder thread, so access to the ImageReader must be 735 * synchronized appropriately. 736 */ postEventFromNative(Object selfRef)737 private static void postEventFromNative(Object selfRef) { 738 @SuppressWarnings("unchecked") 739 WeakReference<ImageReader> weakSelf = (WeakReference<ImageReader>)selfRef; 740 final ImageReader ir = weakSelf.get(); 741 if (ir == null) { 742 return; 743 } 744 745 final Handler handler; 746 synchronized (ir.mListenerLock) { 747 handler = ir.mListenerHandler; 748 } 749 if (handler != null) { 750 handler.sendEmptyMessage(0); 751 } 752 } 753 754 private final int mWidth; 755 private final int mHeight; 756 private final int mFormat; 757 private final int mMaxImages; 758 private final int mNumPlanes; 759 private final Surface mSurface; 760 private int mEstimatedNativeAllocBytes; 761 762 private final Object mListenerLock = new Object(); 763 private final Object mCloseLock = new Object(); 764 private boolean mIsReaderValid = false; 765 private OnImageAvailableListener mListener; 766 private ListenerHandler mListenerHandler; 767 // Keep track of the successfully acquired Images. This need to be thread safe as the images 768 // could be closed by different threads (e.g., application thread and GC thread). 769 private List<Image> mAcquiredImages = new CopyOnWriteArrayList<>(); 770 771 /** 772 * This field is used by native code, do not access or modify. 773 */ 774 private long mNativeContext; 775 776 /** 777 * This custom handler runs asynchronously so callbacks don't get queued behind UI messages. 778 */ 779 private final class ListenerHandler extends Handler { ListenerHandler(Looper looper)780 public ListenerHandler(Looper looper) { 781 super(looper, null, true /*async*/); 782 } 783 784 @Override handleMessage(Message msg)785 public void handleMessage(Message msg) { 786 OnImageAvailableListener listener; 787 synchronized (mListenerLock) { 788 listener = mListener; 789 } 790 791 // It's dangerous to fire onImageAvailable() callback when the ImageReader is being 792 // closed, as application could acquire next image in the onImageAvailable() callback. 793 boolean isReaderValid = false; 794 synchronized (mCloseLock) { 795 isReaderValid = mIsReaderValid; 796 } 797 if (listener != null && isReaderValid) { 798 listener.onImageAvailable(ImageReader.this); 799 } 800 } 801 } 802 803 private class SurfaceImage extends android.media.Image { SurfaceImage(int format)804 public SurfaceImage(int format) { 805 mFormat = format; 806 } 807 808 @Override close()809 public void close() { 810 ImageReader.this.releaseImage(this); 811 } 812 getReader()813 public ImageReader getReader() { 814 return ImageReader.this; 815 } 816 817 @Override getFormat()818 public int getFormat() { 819 throwISEIfImageIsInvalid(); 820 int readerFormat = ImageReader.this.getImageFormat(); 821 // Assume opaque reader always produce opaque images. 822 mFormat = (readerFormat == ImageFormat.PRIVATE) ? readerFormat : 823 nativeGetFormat(readerFormat); 824 return mFormat; 825 } 826 827 @Override getWidth()828 public int getWidth() { 829 throwISEIfImageIsInvalid(); 830 int width; 831 switch(getFormat()) { 832 case ImageFormat.JPEG: 833 case ImageFormat.DEPTH_POINT_CLOUD: 834 case ImageFormat.RAW_PRIVATE: 835 case ImageFormat.DEPTH_JPEG: 836 case ImageFormat.HEIC: 837 width = ImageReader.this.getWidth(); 838 break; 839 default: 840 width = nativeGetWidth(); 841 } 842 return width; 843 } 844 845 @Override getHeight()846 public int getHeight() { 847 throwISEIfImageIsInvalid(); 848 int height; 849 switch(getFormat()) { 850 case ImageFormat.JPEG: 851 case ImageFormat.DEPTH_POINT_CLOUD: 852 case ImageFormat.RAW_PRIVATE: 853 case ImageFormat.DEPTH_JPEG: 854 case ImageFormat.HEIC: 855 height = ImageReader.this.getHeight(); 856 break; 857 default: 858 height = nativeGetHeight(); 859 } 860 return height; 861 } 862 863 @Override getTimestamp()864 public long getTimestamp() { 865 throwISEIfImageIsInvalid(); 866 return mTimestamp; 867 } 868 869 @Override getTransform()870 public int getTransform() { 871 throwISEIfImageIsInvalid(); 872 return mTransform; 873 } 874 875 @Override getScalingMode()876 public int getScalingMode() { 877 throwISEIfImageIsInvalid(); 878 return mScalingMode; 879 } 880 881 @Override getHardwareBuffer()882 public HardwareBuffer getHardwareBuffer() { 883 throwISEIfImageIsInvalid(); 884 return nativeGetHardwareBuffer(); 885 } 886 887 @Override setTimestamp(long timestampNs)888 public void setTimestamp(long timestampNs) { 889 throwISEIfImageIsInvalid(); 890 mTimestamp = timestampNs; 891 } 892 893 @Override getPlanes()894 public Plane[] getPlanes() { 895 throwISEIfImageIsInvalid(); 896 897 if (mPlanes == null) { 898 mPlanes = nativeCreatePlanes(ImageReader.this.mNumPlanes, ImageReader.this.mFormat); 899 } 900 // Shallow copy is fine. 901 return mPlanes.clone(); 902 } 903 904 @Override finalize()905 protected final void finalize() throws Throwable { 906 try { 907 close(); 908 } finally { 909 super.finalize(); 910 } 911 } 912 913 @Override isAttachable()914 boolean isAttachable() { 915 throwISEIfImageIsInvalid(); 916 return mIsDetached.get(); 917 } 918 919 @Override getOwner()920 ImageReader getOwner() { 921 throwISEIfImageIsInvalid(); 922 return ImageReader.this; 923 } 924 925 @Override getNativeContext()926 long getNativeContext() { 927 throwISEIfImageIsInvalid(); 928 return mNativeBuffer; 929 } 930 setDetached(boolean detached)931 private void setDetached(boolean detached) { 932 throwISEIfImageIsInvalid(); 933 mIsDetached.getAndSet(detached); 934 } 935 clearSurfacePlanes()936 private void clearSurfacePlanes() { 937 // Image#getPlanes may not be called before the image is closed. 938 if (mIsImageValid && mPlanes != null) { 939 for (int i = 0; i < mPlanes.length; i++) { 940 if (mPlanes[i] != null) { 941 mPlanes[i].clearBuffer(); 942 mPlanes[i] = null; 943 } 944 } 945 } 946 } 947 948 private class SurfacePlane extends android.media.Image.Plane { 949 // SurfacePlane instance is created by native code when SurfaceImage#getPlanes() is 950 // called SurfacePlane(int rowStride, int pixelStride, ByteBuffer buffer)951 private SurfacePlane(int rowStride, int pixelStride, ByteBuffer buffer) { 952 mRowStride = rowStride; 953 mPixelStride = pixelStride; 954 mBuffer = buffer; 955 /** 956 * Set the byteBuffer order according to host endianness (native 957 * order), otherwise, the byteBuffer order defaults to 958 * ByteOrder.BIG_ENDIAN. 959 */ 960 mBuffer.order(ByteOrder.nativeOrder()); 961 } 962 963 @Override getBuffer()964 public ByteBuffer getBuffer() { 965 throwISEIfImageIsInvalid(); 966 return mBuffer; 967 } 968 969 @Override getPixelStride()970 public int getPixelStride() { 971 SurfaceImage.this.throwISEIfImageIsInvalid(); 972 if (ImageReader.this.mFormat == ImageFormat.RAW_PRIVATE) { 973 throw new UnsupportedOperationException( 974 "getPixelStride is not supported for RAW_PRIVATE plane"); 975 } 976 return mPixelStride; 977 } 978 979 @Override getRowStride()980 public int getRowStride() { 981 SurfaceImage.this.throwISEIfImageIsInvalid(); 982 if (ImageReader.this.mFormat == ImageFormat.RAW_PRIVATE) { 983 throw new UnsupportedOperationException( 984 "getRowStride is not supported for RAW_PRIVATE plane"); 985 } 986 return mRowStride; 987 } 988 clearBuffer()989 private void clearBuffer() { 990 // Need null check first, as the getBuffer() may not be called before an image 991 // is closed. 992 if (mBuffer == null) { 993 return; 994 } 995 996 if (mBuffer.isDirect()) { 997 NioUtils.freeDirectBuffer(mBuffer); 998 } 999 mBuffer = null; 1000 } 1001 1002 final private int mPixelStride; 1003 final private int mRowStride; 1004 1005 private ByteBuffer mBuffer; 1006 } 1007 1008 /** 1009 * This field is used to keep track of native object and used by native code only. 1010 * Don't modify. 1011 */ 1012 private long mNativeBuffer; 1013 1014 /** 1015 * These fields are set by native code during nativeImageSetup(). 1016 */ 1017 private long mTimestamp; 1018 private int mTransform; 1019 private int mScalingMode; 1020 1021 private SurfacePlane[] mPlanes; 1022 private int mFormat = ImageFormat.UNKNOWN; 1023 // If this image is detached from the ImageReader. 1024 private AtomicBoolean mIsDetached = new AtomicBoolean(false); 1025 nativeCreatePlanes(int numPlanes, int readerFormat)1026 private synchronized native SurfacePlane[] nativeCreatePlanes(int numPlanes, 1027 int readerFormat); nativeGetWidth()1028 private synchronized native int nativeGetWidth(); nativeGetHeight()1029 private synchronized native int nativeGetHeight(); nativeGetFormat(int readerFormat)1030 private synchronized native int nativeGetFormat(int readerFormat); nativeGetHardwareBuffer()1031 private synchronized native HardwareBuffer nativeGetHardwareBuffer(); 1032 } 1033 nativeInit(Object weakSelf, int w, int h, int fmt, int maxImgs, long consumerUsage)1034 private synchronized native void nativeInit(Object weakSelf, int w, int h, 1035 int fmt, int maxImgs, long consumerUsage); nativeClose()1036 private synchronized native void nativeClose(); nativeReleaseImage(Image i)1037 private synchronized native void nativeReleaseImage(Image i); nativeGetSurface()1038 private synchronized native Surface nativeGetSurface(); nativeDetachImage(Image i)1039 private synchronized native int nativeDetachImage(Image i); nativeDiscardFreeBuffers()1040 private synchronized native void nativeDiscardFreeBuffers(); 1041 1042 /** 1043 * @return A return code {@code ACQUIRE_*} 1044 * 1045 * @see #ACQUIRE_SUCCESS 1046 * @see #ACQUIRE_NO_BUFS 1047 * @see #ACQUIRE_MAX_IMAGES 1048 */ nativeImageSetup(Image i)1049 private synchronized native int nativeImageSetup(Image i); 1050 1051 /** 1052 * We use a class initializer to allow the native code to cache some 1053 * field offsets. 1054 */ nativeClassInit()1055 private static native void nativeClassInit(); 1056 static { 1057 System.loadLibrary("media_jni"); nativeClassInit()1058 nativeClassInit(); 1059 } 1060 } 1061