1 /*
2  * Copyright 2013 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 //#define LOG_NDEBUG 0
18 #define LOG_TAG "ImageReader_JNI"
19 #include "android_media_Utils.h"
20 #include <cutils/atomic.h>
21 #include <utils/Log.h>
22 #include <utils/misc.h>
23 #include <utils/List.h>
24 #include <utils/String8.h>
25 
26 #include <cstdio>
27 
28 #include <gui/BufferItemConsumer.h>
29 #include <gui/Surface.h>
30 
31 #include <android_runtime/AndroidRuntime.h>
32 #include <android_runtime/android_view_Surface.h>
33 #include <android_runtime/android_hardware_HardwareBuffer.h>
34 #include <grallocusage/GrallocUsageConversion.h>
35 
36 #include <private/android/AHardwareBufferHelpers.h>
37 
38 #include <jni.h>
39 #include <nativehelper/JNIHelp.h>
40 
41 #include <stdint.h>
42 #include <inttypes.h>
43 #include <android/hardware_buffer_jni.h>
44 
45 #define ANDROID_MEDIA_IMAGEREADER_CTX_JNI_ID       "mNativeContext"
46 #define ANDROID_MEDIA_SURFACEIMAGE_BUFFER_JNI_ID   "mNativeBuffer"
47 #define ANDROID_MEDIA_SURFACEIMAGE_TS_JNI_ID       "mTimestamp"
48 #define ANDROID_MEDIA_SURFACEIMAGE_TF_JNI_ID       "mTransform"
49 #define ANDROID_MEDIA_SURFACEIMAGE_SM_JNI_ID       "mScalingMode"
50 
51 #define CONSUMER_BUFFER_USAGE_UNKNOWN              0;
52 // ----------------------------------------------------------------------------
53 
54 using namespace android;
55 
56 
57 enum {
58     ACQUIRE_SUCCESS = 0,
59     ACQUIRE_NO_BUFFERS = 1,
60     ACQUIRE_MAX_IMAGES = 2,
61 };
62 
63 static struct {
64     jfieldID mNativeContext;
65     jmethodID postEventFromNative;
66 } gImageReaderClassInfo;
67 
68 static struct {
69     jfieldID mNativeBuffer;
70     jfieldID mTimestamp;
71     jfieldID mTransform;
72     jfieldID mScalingMode;
73     jfieldID mPlanes;
74 } gSurfaceImageClassInfo;
75 
76 static struct {
77     jclass clazz;
78     jmethodID ctor;
79 } gSurfacePlaneClassInfo;
80 
81 // Get an ID that's unique within this process.
createProcessUniqueId()82 static int32_t createProcessUniqueId() {
83     static volatile int32_t globalCounter = 0;
84     return android_atomic_inc(&globalCounter);
85 }
86 
87 // ----------------------------------------------------------------------------
88 
89 class JNIImageReaderContext : public ConsumerBase::FrameAvailableListener
90 {
91 public:
92     JNIImageReaderContext(JNIEnv* env, jobject weakThiz, jclass clazz, int maxImages);
93 
94     virtual ~JNIImageReaderContext();
95 
96     virtual void onFrameAvailable(const BufferItem& item);
97 
98     BufferItem* getBufferItem();
99     void returnBufferItem(BufferItem* buffer);
100 
101 
setBufferConsumer(const sp<BufferItemConsumer> & consumer)102     void setBufferConsumer(const sp<BufferItemConsumer>& consumer) { mConsumer = consumer; }
getBufferConsumer()103     BufferItemConsumer* getBufferConsumer() { return mConsumer.get(); }
104 
setProducer(const sp<IGraphicBufferProducer> & producer)105     void setProducer(const sp<IGraphicBufferProducer>& producer) { mProducer = producer; }
getProducer()106     IGraphicBufferProducer* getProducer() { return mProducer.get(); }
107 
setBufferFormat(int format)108     void setBufferFormat(int format) { mFormat = format; }
getBufferFormat()109     int getBufferFormat() { return mFormat; }
110 
setBufferDataspace(android_dataspace dataSpace)111     void setBufferDataspace(android_dataspace dataSpace) { mDataSpace = dataSpace; }
getBufferDataspace()112     android_dataspace getBufferDataspace() { return mDataSpace; }
113 
setBufferWidth(int width)114     void setBufferWidth(int width) { mWidth = width; }
getBufferWidth()115     int getBufferWidth() { return mWidth; }
116 
setBufferHeight(int height)117     void setBufferHeight(int height) { mHeight = height; }
getBufferHeight()118     int getBufferHeight() { return mHeight; }
119 
120 private:
121     static JNIEnv* getJNIEnv(bool* needsDetach);
122     static void detachJNI();
123 
124     List<BufferItem*> mBuffers;
125     sp<BufferItemConsumer> mConsumer;
126     sp<IGraphicBufferProducer> mProducer;
127     jobject mWeakThiz;
128     jclass mClazz;
129     int mFormat;
130     android_dataspace mDataSpace;
131     int mWidth;
132     int mHeight;
133 };
134 
JNIImageReaderContext(JNIEnv * env,jobject weakThiz,jclass clazz,int maxImages)135 JNIImageReaderContext::JNIImageReaderContext(JNIEnv* env,
136         jobject weakThiz, jclass clazz, int maxImages) :
137     mWeakThiz(env->NewGlobalRef(weakThiz)),
138     mClazz((jclass)env->NewGlobalRef(clazz)),
139     mFormat(0),
140     mDataSpace(HAL_DATASPACE_UNKNOWN),
141     mWidth(-1),
142     mHeight(-1) {
143     for (int i = 0; i < maxImages; i++) {
144         BufferItem* buffer = new BufferItem;
145         mBuffers.push_back(buffer);
146     }
147 }
148 
getJNIEnv(bool * needsDetach)149 JNIEnv* JNIImageReaderContext::getJNIEnv(bool* needsDetach) {
150     LOG_ALWAYS_FATAL_IF(needsDetach == NULL, "needsDetach is null!!!");
151     *needsDetach = false;
152     JNIEnv* env = AndroidRuntime::getJNIEnv();
153     if (env == NULL) {
154         JavaVMAttachArgs args = {JNI_VERSION_1_4, NULL, NULL};
155         JavaVM* vm = AndroidRuntime::getJavaVM();
156         int result = vm->AttachCurrentThread(&env, (void*) &args);
157         if (result != JNI_OK) {
158             ALOGE("thread attach failed: %#x", result);
159             return NULL;
160         }
161         *needsDetach = true;
162     }
163     return env;
164 }
165 
detachJNI()166 void JNIImageReaderContext::detachJNI() {
167     JavaVM* vm = AndroidRuntime::getJavaVM();
168     int result = vm->DetachCurrentThread();
169     if (result != JNI_OK) {
170         ALOGE("thread detach failed: %#x", result);
171     }
172 }
173 
getBufferItem()174 BufferItem* JNIImageReaderContext::getBufferItem() {
175     if (mBuffers.empty()) {
176         return NULL;
177     }
178     // Return a BufferItem pointer and remove it from the list
179     List<BufferItem*>::iterator it = mBuffers.begin();
180     BufferItem* buffer = *it;
181     mBuffers.erase(it);
182     return buffer;
183 }
184 
returnBufferItem(BufferItem * buffer)185 void JNIImageReaderContext::returnBufferItem(BufferItem* buffer) {
186     buffer->mGraphicBuffer = nullptr;
187     mBuffers.push_back(buffer);
188 }
189 
~JNIImageReaderContext()190 JNIImageReaderContext::~JNIImageReaderContext() {
191     bool needsDetach = false;
192     JNIEnv* env = getJNIEnv(&needsDetach);
193     if (env != NULL) {
194         env->DeleteGlobalRef(mWeakThiz);
195         env->DeleteGlobalRef(mClazz);
196     } else {
197         ALOGW("leaking JNI object references");
198     }
199     if (needsDetach) {
200         detachJNI();
201     }
202 
203     // Delete buffer items.
204     for (List<BufferItem *>::iterator it = mBuffers.begin();
205             it != mBuffers.end(); it++) {
206         delete *it;
207     }
208 
209     if (mConsumer != 0) {
210         mConsumer.clear();
211     }
212 }
213 
onFrameAvailable(const BufferItem &)214 void JNIImageReaderContext::onFrameAvailable(const BufferItem& /*item*/)
215 {
216     ALOGV("%s: frame available", __FUNCTION__);
217     bool needsDetach = false;
218     JNIEnv* env = getJNIEnv(&needsDetach);
219     if (env != NULL) {
220         env->CallStaticVoidMethod(mClazz, gImageReaderClassInfo.postEventFromNative, mWeakThiz);
221     } else {
222         ALOGW("onFrameAvailable event will not posted");
223     }
224     if (needsDetach) {
225         detachJNI();
226     }
227 }
228 
229 // ----------------------------------------------------------------------------
230 
231 extern "C" {
232 
ImageReader_getContext(JNIEnv * env,jobject thiz)233 static JNIImageReaderContext* ImageReader_getContext(JNIEnv* env, jobject thiz)
234 {
235     JNIImageReaderContext *ctx;
236     ctx = reinterpret_cast<JNIImageReaderContext *>
237               (env->GetLongField(thiz, gImageReaderClassInfo.mNativeContext));
238     return ctx;
239 }
240 
ImageReader_getProducer(JNIEnv * env,jobject thiz)241 static IGraphicBufferProducer* ImageReader_getProducer(JNIEnv* env, jobject thiz)
242 {
243     ALOGV("%s:", __FUNCTION__);
244     JNIImageReaderContext* const ctx = ImageReader_getContext(env, thiz);
245     if (ctx == NULL) {
246         jniThrowRuntimeException(env, "ImageReaderContext is not initialized");
247         return NULL;
248     }
249 
250     return ctx->getProducer();
251 }
252 
ImageReader_setNativeContext(JNIEnv * env,jobject thiz,sp<JNIImageReaderContext> ctx)253 static void ImageReader_setNativeContext(JNIEnv* env,
254         jobject thiz, sp<JNIImageReaderContext> ctx)
255 {
256     ALOGV("%s:", __FUNCTION__);
257     JNIImageReaderContext* const p = ImageReader_getContext(env, thiz);
258     if (ctx != 0) {
259         ctx->incStrong((void*)ImageReader_setNativeContext);
260     }
261     if (p) {
262         p->decStrong((void*)ImageReader_setNativeContext);
263     }
264     env->SetLongField(thiz, gImageReaderClassInfo.mNativeContext,
265             reinterpret_cast<jlong>(ctx.get()));
266 }
267 
ImageReader_getBufferConsumer(JNIEnv * env,jobject thiz)268 static BufferItemConsumer* ImageReader_getBufferConsumer(JNIEnv* env, jobject thiz)
269 {
270     ALOGV("%s:", __FUNCTION__);
271     JNIImageReaderContext* const ctx = ImageReader_getContext(env, thiz);
272     if (ctx == NULL) {
273         jniThrowRuntimeException(env, "ImageReaderContext is not initialized");
274         return NULL;
275     }
276 
277     return ctx->getBufferConsumer();
278 }
279 
Image_setBufferItem(JNIEnv * env,jobject thiz,const BufferItem * buffer)280 static void Image_setBufferItem(JNIEnv* env, jobject thiz,
281         const BufferItem* buffer)
282 {
283     env->SetLongField(thiz, gSurfaceImageClassInfo.mNativeBuffer, reinterpret_cast<jlong>(buffer));
284 }
285 
Image_getBufferItem(JNIEnv * env,jobject image)286 static BufferItem* Image_getBufferItem(JNIEnv* env, jobject image)
287 {
288     return reinterpret_cast<BufferItem*>(
289             env->GetLongField(image, gSurfaceImageClassInfo.mNativeBuffer));
290 }
291 
292 
293 // ----------------------------------------------------------------------------
294 
ImageReader_classInit(JNIEnv * env,jclass clazz)295 static void ImageReader_classInit(JNIEnv* env, jclass clazz)
296 {
297     ALOGV("%s:", __FUNCTION__);
298 
299     jclass imageClazz = env->FindClass("android/media/ImageReader$SurfaceImage");
300     LOG_ALWAYS_FATAL_IF(imageClazz == NULL,
301                         "can't find android/graphics/ImageReader$SurfaceImage");
302     gSurfaceImageClassInfo.mNativeBuffer = env->GetFieldID(
303             imageClazz, ANDROID_MEDIA_SURFACEIMAGE_BUFFER_JNI_ID, "J");
304     LOG_ALWAYS_FATAL_IF(gSurfaceImageClassInfo.mNativeBuffer == NULL,
305                         "can't find android/graphics/ImageReader.%s",
306                         ANDROID_MEDIA_SURFACEIMAGE_BUFFER_JNI_ID);
307 
308     gSurfaceImageClassInfo.mTimestamp = env->GetFieldID(
309             imageClazz, ANDROID_MEDIA_SURFACEIMAGE_TS_JNI_ID, "J");
310     LOG_ALWAYS_FATAL_IF(gSurfaceImageClassInfo.mTimestamp == NULL,
311                         "can't find android/graphics/ImageReader.%s",
312                         ANDROID_MEDIA_SURFACEIMAGE_TS_JNI_ID);
313 
314     gSurfaceImageClassInfo.mTransform = env->GetFieldID(
315             imageClazz, ANDROID_MEDIA_SURFACEIMAGE_TF_JNI_ID, "I");
316     LOG_ALWAYS_FATAL_IF(gSurfaceImageClassInfo.mTransform == NULL,
317                         "can't find android/graphics/ImageReader.%s",
318                         ANDROID_MEDIA_SURFACEIMAGE_TF_JNI_ID);
319 
320     gSurfaceImageClassInfo.mScalingMode = env->GetFieldID(
321             imageClazz, ANDROID_MEDIA_SURFACEIMAGE_SM_JNI_ID, "I");
322     LOG_ALWAYS_FATAL_IF(gSurfaceImageClassInfo.mScalingMode == NULL,
323                         "can't find android/graphics/ImageReader.%s",
324                         ANDROID_MEDIA_SURFACEIMAGE_SM_JNI_ID);
325 
326     gSurfaceImageClassInfo.mPlanes = env->GetFieldID(
327             imageClazz, "mPlanes", "[Landroid/media/ImageReader$SurfaceImage$SurfacePlane;");
328     LOG_ALWAYS_FATAL_IF(gSurfaceImageClassInfo.mPlanes == NULL,
329             "can't find android/media/ImageReader$ReaderSurfaceImage.mPlanes");
330 
331     gImageReaderClassInfo.mNativeContext = env->GetFieldID(
332             clazz, ANDROID_MEDIA_IMAGEREADER_CTX_JNI_ID, "J");
333     LOG_ALWAYS_FATAL_IF(gImageReaderClassInfo.mNativeContext == NULL,
334                         "can't find android/graphics/ImageReader.%s",
335                           ANDROID_MEDIA_IMAGEREADER_CTX_JNI_ID);
336 
337     gImageReaderClassInfo.postEventFromNative = env->GetStaticMethodID(
338             clazz, "postEventFromNative", "(Ljava/lang/Object;)V");
339     LOG_ALWAYS_FATAL_IF(gImageReaderClassInfo.postEventFromNative == NULL,
340                         "can't find android/graphics/ImageReader.postEventFromNative");
341 
342     jclass planeClazz = env->FindClass("android/media/ImageReader$SurfaceImage$SurfacePlane");
343     LOG_ALWAYS_FATAL_IF(planeClazz == NULL, "Can not find SurfacePlane class");
344     // FindClass only gives a local reference of jclass object.
345     gSurfacePlaneClassInfo.clazz = (jclass) env->NewGlobalRef(planeClazz);
346     gSurfacePlaneClassInfo.ctor = env->GetMethodID(gSurfacePlaneClassInfo.clazz, "<init>",
347             "(Landroid/media/ImageReader$SurfaceImage;IILjava/nio/ByteBuffer;)V");
348     LOG_ALWAYS_FATAL_IF(gSurfacePlaneClassInfo.ctor == NULL,
349             "Can not find SurfacePlane constructor");
350 }
351 
ImageReader_init(JNIEnv * env,jobject thiz,jobject weakThiz,jint width,jint height,jint format,jint maxImages,jlong ndkUsage)352 static void ImageReader_init(JNIEnv* env, jobject thiz, jobject weakThiz, jint width, jint height,
353                              jint format, jint maxImages, jlong ndkUsage)
354 {
355     status_t res;
356     int nativeFormat;
357     android_dataspace nativeDataspace;
358 
359     ALOGV("%s: width:%d, height: %d, format: 0x%x, maxImages:%d",
360           __FUNCTION__, width, height, format, maxImages);
361 
362     PublicFormat publicFormat = static_cast<PublicFormat>(format);
363     nativeFormat = android_view_Surface_mapPublicFormatToHalFormat(
364         publicFormat);
365     nativeDataspace = android_view_Surface_mapPublicFormatToHalDataspace(
366         publicFormat);
367 
368     jclass clazz = env->GetObjectClass(thiz);
369     if (clazz == NULL) {
370         jniThrowRuntimeException(env, "Can't find android/graphics/ImageReader");
371         return;
372     }
373     sp<JNIImageReaderContext> ctx(new JNIImageReaderContext(env, weakThiz, clazz, maxImages));
374 
375     sp<IGraphicBufferProducer> gbProducer;
376     sp<IGraphicBufferConsumer> gbConsumer;
377     BufferQueue::createBufferQueue(&gbProducer, &gbConsumer);
378     sp<BufferItemConsumer> bufferConsumer;
379     String8 consumerName = String8::format("ImageReader-%dx%df%xm%d-%d-%d",
380             width, height, format, maxImages, getpid(),
381             createProcessUniqueId());
382     uint64_t consumerUsage =
383             android_hardware_HardwareBuffer_convertToGrallocUsageBits(ndkUsage);
384 
385     bufferConsumer = new BufferItemConsumer(gbConsumer, consumerUsage, maxImages,
386             /*controlledByApp*/true);
387     if (bufferConsumer == nullptr) {
388         jniThrowExceptionFmt(env, "java/lang/RuntimeException",
389                 "Failed to allocate native buffer consumer for format 0x%x and usage 0x%x",
390                 nativeFormat, consumerUsage);
391         return;
392     }
393 
394     if (consumerUsage & GRALLOC_USAGE_PROTECTED) {
395         gbConsumer->setConsumerIsProtected(true);
396     }
397 
398     ctx->setBufferConsumer(bufferConsumer);
399     bufferConsumer->setName(consumerName);
400 
401     ctx->setProducer(gbProducer);
402     bufferConsumer->setFrameAvailableListener(ctx);
403     ImageReader_setNativeContext(env, thiz, ctx);
404     ctx->setBufferFormat(nativeFormat);
405     ctx->setBufferDataspace(nativeDataspace);
406     ctx->setBufferWidth(width);
407     ctx->setBufferHeight(height);
408 
409     // Set the width/height/format/dataspace to the bufferConsumer.
410     res = bufferConsumer->setDefaultBufferSize(width, height);
411     if (res != OK) {
412         jniThrowExceptionFmt(env, "java/lang/IllegalStateException",
413                           "Failed to set buffer consumer default size (%dx%d) for format 0x%x",
414                           width, height, nativeFormat);
415         return;
416     }
417     res = bufferConsumer->setDefaultBufferFormat(nativeFormat);
418     if (res != OK) {
419         jniThrowExceptionFmt(env, "java/lang/IllegalStateException",
420                           "Failed to set buffer consumer default format 0x%x", nativeFormat);
421     }
422     res = bufferConsumer->setDefaultBufferDataSpace(nativeDataspace);
423     if (res != OK) {
424         jniThrowExceptionFmt(env, "java/lang/IllegalStateException",
425                           "Failed to set buffer consumer default dataSpace 0x%x", nativeDataspace);
426     }
427 }
428 
ImageReader_close(JNIEnv * env,jobject thiz)429 static void ImageReader_close(JNIEnv* env, jobject thiz)
430 {
431     ALOGV("%s:", __FUNCTION__);
432 
433     JNIImageReaderContext* const ctx = ImageReader_getContext(env, thiz);
434     if (ctx == NULL) {
435         // ImageReader is already closed.
436         return;
437     }
438 
439     BufferItemConsumer* consumer = NULL;
440     consumer = ImageReader_getBufferConsumer(env, thiz);
441 
442     if (consumer != NULL) {
443         consumer->abandon();
444         consumer->setFrameAvailableListener(NULL);
445     }
446     ImageReader_setNativeContext(env, thiz, NULL);
447 }
448 
Image_unlockIfLocked(JNIEnv * env,jobject image)449 static sp<Fence> Image_unlockIfLocked(JNIEnv* env, jobject image) {
450     ALOGV("%s", __FUNCTION__);
451     BufferItem* buffer = Image_getBufferItem(env, image);
452     if (buffer == NULL) {
453         jniThrowException(env, "java/lang/IllegalStateException",
454                 "Image is not initialized");
455         return Fence::NO_FENCE;
456     }
457 
458     // Is locked?
459     bool wasBufferLocked = false;
460     jobject planes = NULL;
461     if (!isFormatOpaque(buffer->mGraphicBuffer->getPixelFormat())) {
462         planes = env->GetObjectField(image, gSurfaceImageClassInfo.mPlanes);
463     }
464     wasBufferLocked = (planes != NULL);
465     if (wasBufferLocked) {
466         status_t res = OK;
467         int fenceFd = -1;
468         if (wasBufferLocked) {
469             res = buffer->mGraphicBuffer->unlockAsync(&fenceFd);
470             if (res != OK) {
471                 jniThrowRuntimeException(env, "unlock buffer failed");
472                 return Fence::NO_FENCE;
473             }
474         }
475         sp<Fence> releaseFence = new Fence(fenceFd);
476         return releaseFence;
477         ALOGV("Successfully unlocked the image");
478     }
479     return Fence::NO_FENCE;
480 }
481 
ImageReader_imageRelease(JNIEnv * env,jobject thiz,jobject image)482 static void ImageReader_imageRelease(JNIEnv* env, jobject thiz, jobject image)
483 {
484     ALOGV("%s:", __FUNCTION__);
485     JNIImageReaderContext* ctx = ImageReader_getContext(env, thiz);
486     if (ctx == NULL) {
487         ALOGW("ImageReader#close called before Image#close, consider calling Image#close first");
488         return;
489     }
490 
491     BufferItemConsumer* bufferConsumer = ctx->getBufferConsumer();
492     BufferItem* buffer = Image_getBufferItem(env, image);
493     if (buffer == nullptr) {
494         // Release an already closed image is harmless.
495         return;
496     }
497 
498     sp<Fence> releaseFence = Image_unlockIfLocked(env, image);
499     bufferConsumer->releaseBuffer(*buffer, releaseFence);
500     Image_setBufferItem(env, image, NULL);
501     ctx->returnBufferItem(buffer);
502     ALOGV("%s: Image (format: 0x%x) has been released", __FUNCTION__, ctx->getBufferFormat());
503 }
504 
ImageReader_imageSetup(JNIEnv * env,jobject thiz,jobject image)505 static jint ImageReader_imageSetup(JNIEnv* env, jobject thiz, jobject image) {
506     ALOGV("%s:", __FUNCTION__);
507     JNIImageReaderContext* ctx = ImageReader_getContext(env, thiz);
508     if (ctx == NULL) {
509         jniThrowException(env, "java/lang/IllegalStateException",
510                 "ImageReader is not initialized or was already closed");
511         return -1;
512     }
513 
514     BufferItemConsumer* bufferConsumer = ctx->getBufferConsumer();
515     BufferItem* buffer = ctx->getBufferItem();
516     if (buffer == NULL) {
517         ALOGW("Unable to acquire a buffer item, very likely client tried to acquire more than"
518             " maxImages buffers");
519         return ACQUIRE_MAX_IMAGES;
520     }
521 
522     status_t res = bufferConsumer->acquireBuffer(buffer, 0);
523     if (res != OK) {
524         ctx->returnBufferItem(buffer);
525         if (res != BufferQueue::NO_BUFFER_AVAILABLE) {
526             if (res == INVALID_OPERATION) {
527                 // Max number of images were already acquired.
528                 ALOGE("%s: Max number of buffers allowed are already acquired : %s (%d)",
529                         __FUNCTION__, strerror(-res), res);
530                 return ACQUIRE_MAX_IMAGES;
531             } else {
532                 ALOGE("%s: Acquire image failed with some unknown error: %s (%d)",
533                         __FUNCTION__, strerror(-res), res);
534                 jniThrowExceptionFmt(env, "java/lang/IllegalStateException",
535                         "Unknown error (%d) when we tried to acquire an image.",
536                                           res);
537                 return ACQUIRE_NO_BUFFERS;
538             }
539         }
540         // This isn't really an error case, as the application may acquire buffer at any time.
541         return ACQUIRE_NO_BUFFERS;
542     }
543 
544     // Add some extra checks for non-opaque formats.
545     if (!isFormatOpaque(ctx->getBufferFormat())) {
546         // Check if the left-top corner of the crop rect is origin, we currently assume this point is
547         // zero, will revisit this once this assumption turns out problematic.
548         Point lt = buffer->mCrop.leftTop();
549         if (lt.x != 0 || lt.y != 0) {
550             jniThrowExceptionFmt(env, "java/lang/UnsupportedOperationException",
551                     "crop left top corner [%d, %d] need to be at origin", lt.x, lt.y);
552             return -1;
553         }
554 
555         // Check if the producer buffer configurations match what ImageReader configured.
556         int outputWidth = getBufferWidth(buffer);
557         int outputHeight = getBufferHeight(buffer);
558 
559         int imgReaderFmt = ctx->getBufferFormat();
560         int imageReaderWidth = ctx->getBufferWidth();
561         int imageReaderHeight = ctx->getBufferHeight();
562         int bufferFormat = buffer->mGraphicBuffer->getPixelFormat();
563         if ((bufferFormat != HAL_PIXEL_FORMAT_BLOB) && (imgReaderFmt != HAL_PIXEL_FORMAT_BLOB) &&
564                 (imageReaderWidth != outputWidth || imageReaderHeight != outputHeight)) {
565             ALOGV("%s: Producer buffer size: %dx%d, doesn't match ImageReader configured size: %dx%d",
566                     __FUNCTION__, outputWidth, outputHeight, imageReaderWidth, imageReaderHeight);
567         }
568         if (imgReaderFmt != bufferFormat) {
569             if (imgReaderFmt == HAL_PIXEL_FORMAT_YCbCr_420_888 &&
570                     isPossiblyYUV(bufferFormat)) {
571                 // Treat formats that are compatible with flexible YUV
572                 // (HAL_PIXEL_FORMAT_YCbCr_420_888) as HAL_PIXEL_FORMAT_YCbCr_420_888.
573                 ALOGV("%s: Treat buffer format to 0x%x as HAL_PIXEL_FORMAT_YCbCr_420_888",
574                         __FUNCTION__, bufferFormat);
575             } else if (imgReaderFmt == HAL_PIXEL_FORMAT_BLOB &&
576                     bufferFormat == HAL_PIXEL_FORMAT_RGBA_8888) {
577                 // Using HAL_PIXEL_FORMAT_RGBA_8888 Gralloc buffers containing JPEGs to get around
578                 // SW write limitations for (b/17379185).
579                 ALOGV("%s: Receiving JPEG in HAL_PIXEL_FORMAT_RGBA_8888 buffer.", __FUNCTION__);
580             } else {
581                 // Return the buffer to the queue. No need to provide fence, as this buffer wasn't
582                 // used anywhere yet.
583                 bufferConsumer->releaseBuffer(*buffer);
584                 ctx->returnBufferItem(buffer);
585 
586                 // Throw exception
587                 ALOGE("Producer output buffer format: 0x%x, ImageReader configured format: 0x%x",
588                         bufferFormat, ctx->getBufferFormat());
589                 String8 msg;
590                 msg.appendFormat("The producer output buffer format 0x%x doesn't "
591                         "match the ImageReader's configured buffer format 0x%x.",
592                         bufferFormat, ctx->getBufferFormat());
593                 jniThrowException(env, "java/lang/UnsupportedOperationException",
594                         msg.string());
595                 return -1;
596             }
597         }
598 
599     }
600 
601     // Set SurfaceImage instance member variables
602     Image_setBufferItem(env, image, buffer);
603     env->SetLongField(image, gSurfaceImageClassInfo.mTimestamp,
604             static_cast<jlong>(buffer->mTimestamp));
605     auto transform = buffer->mTransform;
606     if (buffer->mTransformToDisplayInverse) {
607         transform |= NATIVE_WINDOW_TRANSFORM_INVERSE_DISPLAY;
608     }
609     env->SetIntField(image, gSurfaceImageClassInfo.mTransform,
610             static_cast<jint>(transform));
611     env->SetIntField(image, gSurfaceImageClassInfo.mScalingMode,
612             static_cast<jint>(buffer->mScalingMode));
613 
614     return ACQUIRE_SUCCESS;
615 }
616 
ImageReader_detachImage(JNIEnv * env,jobject thiz,jobject image)617 static jint ImageReader_detachImage(JNIEnv* env, jobject thiz, jobject image) {
618     ALOGV("%s:", __FUNCTION__);
619     JNIImageReaderContext* ctx = ImageReader_getContext(env, thiz);
620     if (ctx == NULL) {
621         jniThrowException(env, "java/lang/IllegalStateException", "ImageReader was already closed");
622         return -1;
623     }
624 
625     BufferItemConsumer* bufferConsumer = ctx->getBufferConsumer();
626     BufferItem* buffer = Image_getBufferItem(env, image);
627     if (!buffer) {
628         ALOGE(
629                 "Image already released and can not be detached from ImageReader!!!");
630         jniThrowException(env, "java/lang/IllegalStateException",
631                 "Image detach from ImageReader failed: buffer was already released");
632         return -1;
633     }
634 
635     status_t res = OK;
636     Image_unlockIfLocked(env, image);
637     res = bufferConsumer->detachBuffer(buffer->mSlot);
638     if (res != OK) {
639         ALOGE("Image detach failed: %s (%d)!!!", strerror(-res), res);
640         jniThrowRuntimeException(env,
641                 "nativeDetachImage failed for image!!!");
642         return res;
643     }
644     return OK;
645 }
646 
ImageReader_discardFreeBuffers(JNIEnv * env,jobject thiz)647 static void ImageReader_discardFreeBuffers(JNIEnv* env, jobject thiz) {
648     ALOGV("%s:", __FUNCTION__);
649     JNIImageReaderContext* ctx = ImageReader_getContext(env, thiz);
650     if (ctx == NULL) {
651         jniThrowException(env, "java/lang/IllegalStateException", "ImageReader was already closed");
652         return;
653     }
654 
655     BufferItemConsumer* bufferConsumer = ctx->getBufferConsumer();
656     status_t res = bufferConsumer->discardFreeBuffers();
657     if (res != OK) {
658         ALOGE("Buffer discard failed: %s (%d)", strerror(-res), res);
659         jniThrowRuntimeException(env,
660                 "nativeDicardFreebuffers failed");
661     }
662 }
663 
ImageReader_getSurface(JNIEnv * env,jobject thiz)664 static jobject ImageReader_getSurface(JNIEnv* env, jobject thiz)
665 {
666     ALOGV("%s: ", __FUNCTION__);
667 
668     IGraphicBufferProducer* gbp = ImageReader_getProducer(env, thiz);
669     if (gbp == NULL) {
670         jniThrowRuntimeException(env, "Buffer consumer is uninitialized");
671         return NULL;
672     }
673 
674     // Wrap the IGBP in a Java-language Surface.
675     return android_view_Surface_createFromIGraphicBufferProducer(env, gbp);
676 }
677 
Image_getLockedImage(JNIEnv * env,jobject thiz,LockedImage * image)678 static void Image_getLockedImage(JNIEnv* env, jobject thiz, LockedImage *image) {
679     ALOGV("%s", __FUNCTION__);
680     BufferItem* buffer = Image_getBufferItem(env, thiz);
681     if (buffer == NULL) {
682         jniThrowException(env, "java/lang/IllegalStateException",
683                 "Image is not initialized");
684         return;
685     }
686 
687     status_t res = lockImageFromBuffer(buffer,
688             GRALLOC_USAGE_SW_READ_OFTEN, buffer->mFence->dup(), image);
689     if (res != OK) {
690         jniThrowExceptionFmt(env, "java/lang/RuntimeException",
691                 "lock buffer failed for format 0x%x",
692                 buffer->mGraphicBuffer->getPixelFormat());
693         return;
694     }
695 
696     // Carry over some fields from BufferItem.
697     image->crop        = buffer->mCrop;
698     image->transform   = buffer->mTransform;
699     image->scalingMode = buffer->mScalingMode;
700     image->timestamp   = buffer->mTimestamp;
701     image->dataSpace   = buffer->mDataSpace;
702     image->frameNumber = buffer->mFrameNumber;
703 
704     ALOGV("%s: Successfully locked the image", __FUNCTION__);
705     // crop, transform, scalingMode, timestamp, and frameNumber should be set by producer,
706     // and we don't set them here.
707 }
708 
Image_getLockedImageInfo(JNIEnv * env,LockedImage * buffer,int idx,int32_t writerFormat,uint8_t ** base,uint32_t * size,int * pixelStride,int * rowStride)709 static void Image_getLockedImageInfo(JNIEnv* env, LockedImage* buffer, int idx,
710         int32_t writerFormat, uint8_t **base, uint32_t *size, int *pixelStride, int *rowStride) {
711     ALOGV("%s", __FUNCTION__);
712 
713     status_t res = getLockedImageInfo(buffer, idx, writerFormat, base, size,
714             pixelStride, rowStride);
715     if (res != OK) {
716         jniThrowExceptionFmt(env, "java/lang/UnsupportedOperationException",
717                              "Pixel format: 0x%x is unsupported", buffer->flexFormat);
718     }
719 }
720 
Image_createSurfacePlanes(JNIEnv * env,jobject thiz,int numPlanes,int readerFormat)721 static jobjectArray Image_createSurfacePlanes(JNIEnv* env, jobject thiz,
722         int numPlanes, int readerFormat)
723 {
724     ALOGV("%s: create SurfacePlane array with size %d", __FUNCTION__, numPlanes);
725     int rowStride = 0;
726     int pixelStride = 0;
727     uint8_t *pData = NULL;
728     uint32_t dataSize = 0;
729     jobject byteBuffer = NULL;
730 
731     PublicFormat publicReaderFormat = static_cast<PublicFormat>(readerFormat);
732     int halReaderFormat = android_view_Surface_mapPublicFormatToHalFormat(
733         publicReaderFormat);
734 
735     if (isFormatOpaque(halReaderFormat) && numPlanes > 0) {
736         String8 msg;
737         msg.appendFormat("Format 0x%x is opaque, thus not writable, the number of planes (%d)"
738                 " must be 0", halReaderFormat, numPlanes);
739         jniThrowException(env, "java/lang/IllegalArgumentException", msg.string());
740         return NULL;
741     }
742 
743     jobjectArray surfacePlanes = env->NewObjectArray(numPlanes, gSurfacePlaneClassInfo.clazz,
744             /*initial_element*/NULL);
745     if (surfacePlanes == NULL) {
746         jniThrowRuntimeException(env, "Failed to create SurfacePlane arrays,"
747                 " probably out of memory");
748         return NULL;
749     }
750     if (isFormatOpaque(halReaderFormat)) {
751         // Return 0 element surface array.
752         return surfacePlanes;
753     }
754 
755     LockedImage lockedImg = LockedImage();
756     Image_getLockedImage(env, thiz, &lockedImg);
757     if (env->ExceptionCheck()) {
758         return NULL;
759     }
760     // Create all SurfacePlanes
761     for (int i = 0; i < numPlanes; i++) {
762         Image_getLockedImageInfo(env, &lockedImg, i, halReaderFormat,
763                 &pData, &dataSize, &pixelStride, &rowStride);
764         byteBuffer = env->NewDirectByteBuffer(pData, dataSize);
765         if ((byteBuffer == NULL) && (env->ExceptionCheck() == false)) {
766             jniThrowException(env, "java/lang/IllegalStateException",
767                     "Failed to allocate ByteBuffer");
768             return NULL;
769         }
770 
771         // Finally, create this SurfacePlane.
772         jobject surfacePlane = env->NewObject(gSurfacePlaneClassInfo.clazz,
773                     gSurfacePlaneClassInfo.ctor, thiz, rowStride, pixelStride, byteBuffer);
774         env->SetObjectArrayElement(surfacePlanes, i, surfacePlane);
775     }
776 
777     return surfacePlanes;
778 }
779 
Image_getWidth(JNIEnv * env,jobject thiz)780 static jint Image_getWidth(JNIEnv* env, jobject thiz)
781 {
782     BufferItem* buffer = Image_getBufferItem(env, thiz);
783     return getBufferWidth(buffer);
784 }
785 
Image_getHeight(JNIEnv * env,jobject thiz)786 static jint Image_getHeight(JNIEnv* env, jobject thiz)
787 {
788     BufferItem* buffer = Image_getBufferItem(env, thiz);
789     return getBufferHeight(buffer);
790 }
791 
Image_getFormat(JNIEnv * env,jobject thiz,jint readerFormat)792 static jint Image_getFormat(JNIEnv* env, jobject thiz, jint readerFormat)
793 {
794     if (isFormatOpaque(readerFormat)) {
795         // Assuming opaque reader produce opaque images.
796         return static_cast<jint>(PublicFormat::PRIVATE);
797     } else {
798         BufferItem* buffer = Image_getBufferItem(env, thiz);
799         int readerHalFormat = android_view_Surface_mapPublicFormatToHalFormat(
800                 static_cast<PublicFormat>(readerFormat));
801         int32_t fmt = applyFormatOverrides(
802                 buffer->mGraphicBuffer->getPixelFormat(), readerHalFormat);
803         // Override the image format to HAL_PIXEL_FORMAT_YCbCr_420_888 if the actual format is
804         // NV21 or YV12. This could only happen when the Gralloc HAL version is v0.1 thus doesn't
805         // support lockycbcr(), the CpuConsumer need to use the lock() method in the
806         // lockNextBuffer() call. For Gralloc HAL v0.2 or newer, this format should already be
807         // overridden to HAL_PIXEL_FORMAT_YCbCr_420_888 for the flexible YUV compatible formats.
808         if (isPossiblyYUV(fmt)) {
809             fmt = HAL_PIXEL_FORMAT_YCbCr_420_888;
810         }
811         PublicFormat publicFmt = android_view_Surface_mapHalFormatDataspaceToPublicFormat(
812                 fmt, buffer->mDataSpace);
813         return static_cast<jint>(publicFmt);
814     }
815 }
816 
Image_getHardwareBuffer(JNIEnv * env,jobject thiz)817 static jobject Image_getHardwareBuffer(JNIEnv* env, jobject thiz) {
818     BufferItem* buffer = Image_getBufferItem(env, thiz);
819     AHardwareBuffer* b = AHardwareBuffer_from_GraphicBuffer(buffer->mGraphicBuffer.get());
820     // don't user the public AHardwareBuffer_toHardwareBuffer() because this would force us
821     // to link against libandroid.so
822     return android_hardware_HardwareBuffer_createFromAHardwareBuffer(env, b);
823 }
824 
825 } // extern "C"
826 
827 // ----------------------------------------------------------------------------
828 
829 static const JNINativeMethod gImageReaderMethods[] = {
830     {"nativeClassInit",        "()V",                        (void*)ImageReader_classInit },
831     {"nativeInit",             "(Ljava/lang/Object;IIIIJ)V",  (void*)ImageReader_init },
832     {"nativeClose",            "()V",                        (void*)ImageReader_close },
833     {"nativeReleaseImage",     "(Landroid/media/Image;)V",   (void*)ImageReader_imageRelease },
834     {"nativeImageSetup",       "(Landroid/media/Image;)I",   (void*)ImageReader_imageSetup },
835     {"nativeGetSurface",       "()Landroid/view/Surface;",   (void*)ImageReader_getSurface },
836     {"nativeDetachImage",      "(Landroid/media/Image;)I",   (void*)ImageReader_detachImage },
837     {"nativeDiscardFreeBuffers", "()V",                      (void*)ImageReader_discardFreeBuffers }
838 };
839 
840 static const JNINativeMethod gImageMethods[] = {
841     {"nativeCreatePlanes",      "(II)[Landroid/media/ImageReader$SurfaceImage$SurfacePlane;",
842                                                              (void*)Image_createSurfacePlanes },
843     {"nativeGetWidth",          "()I",                       (void*)Image_getWidth },
844     {"nativeGetHeight",         "()I",                       (void*)Image_getHeight },
845     {"nativeGetFormat",         "(I)I",                      (void*)Image_getFormat },
846     {"nativeGetHardwareBuffer", "()Landroid/hardware/HardwareBuffer;",
847                                                              (void*)Image_getHardwareBuffer },
848 };
849 
register_android_media_ImageReader(JNIEnv * env)850 int register_android_media_ImageReader(JNIEnv *env) {
851 
852     int ret1 = AndroidRuntime::registerNativeMethods(env,
853                    "android/media/ImageReader", gImageReaderMethods, NELEM(gImageReaderMethods));
854 
855     int ret2 = AndroidRuntime::registerNativeMethods(env,
856                    "android/media/ImageReader$SurfaceImage", gImageMethods, NELEM(gImageMethods));
857 
858     return (ret1 || ret2);
859 }
860