1 /*
2  * Copyright (C) 2010 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 package android.graphics;
18 
19 import android.annotation.Nullable;
20 import android.compat.annotation.UnsupportedAppUsage;
21 import android.os.Handler;
22 import android.os.Looper;
23 import android.os.Message;
24 import android.view.Surface;
25 
26 import java.lang.ref.WeakReference;
27 
28 /**
29  * Captures frames from an image stream as an OpenGL ES texture.
30  *
31  * <p>The image stream may come from either camera preview or video decode. A
32  * {@link android.view.Surface} created from a SurfaceTexture can be used as an output
33  * destination for the {@link android.hardware.camera2}, {@link android.media.MediaCodec},
34  * {@link android.media.MediaPlayer}, and {@link android.renderscript.Allocation} APIs.
35  * When {@link #updateTexImage} is called, the contents of the texture object specified
36  * when the SurfaceTexture was created are updated to contain the most recent image from the image
37  * stream.  This may cause some frames of the stream to be skipped.
38  *
39  * <p>A SurfaceTexture may also be used in place of a SurfaceHolder when specifying the output
40  * destination of the older {@link android.hardware.Camera} API. Doing so will cause all the
41  * frames from the image stream to be sent to the SurfaceTexture object rather than to the device's
42  * display.
43  *
44  * <p>When sampling from the texture one should first transform the texture coordinates using the
45  * matrix queried via {@link #getTransformMatrix(float[])}.  The transform matrix may change each
46  * time {@link #updateTexImage} is called, so it should be re-queried each time the texture image
47  * is updated.
48  * This matrix transforms traditional 2D OpenGL ES texture coordinate column vectors of the form (s,
49  * t, 0, 1) where s and t are on the inclusive interval [0, 1] to the proper sampling location in
50  * the streamed texture.  This transform compensates for any properties of the image stream source
51  * that cause it to appear different from a traditional OpenGL ES texture.  For example, sampling
52  * from the bottom left corner of the image can be accomplished by transforming the column vector
53  * (0, 0, 0, 1) using the queried matrix, while sampling from the top right corner of the image can
54  * be done by transforming (1, 1, 0, 1).
55  *
56  * <p>The texture object uses the GL_TEXTURE_EXTERNAL_OES texture target, which is defined by the
57  * <a href="http://www.khronos.org/registry/gles/extensions/OES/OES_EGL_image_external.txt">
58  * GL_OES_EGL_image_external</a> OpenGL ES extension.  This limits how the texture may be used.
59  * Each time the texture is bound it must be bound to the GL_TEXTURE_EXTERNAL_OES target rather than
60  * the GL_TEXTURE_2D target.  Additionally, any OpenGL ES 2.0 shader that samples from the texture
61  * must declare its use of this extension using, for example, an "#extension
62  * GL_OES_EGL_image_external : require" directive.  Such shaders must also access the texture using
63  * the samplerExternalOES GLSL sampler type.
64  *
65  * <p>SurfaceTexture objects may be created on any thread.  {@link #updateTexImage} may only be
66  * called on the thread with the OpenGL ES context that contains the texture object.  The
67  * frame-available callback is called on an arbitrary thread, so unless special care is taken {@link
68  * #updateTexImage} should not be called directly from the callback.
69  */
70 public class SurfaceTexture {
71     private final Looper mCreatorLooper;
72     @UnsupportedAppUsage
73     private Handler mOnFrameAvailableHandler;
74 
75     /**
76      * These fields are used by native code, do not access or modify.
77      */
78     @UnsupportedAppUsage
79     private long mSurfaceTexture;
80     @UnsupportedAppUsage
81     private long mProducer;
82     @UnsupportedAppUsage
83     private long mFrameAvailableListener;
84 
85     private boolean mIsSingleBuffered;
86 
87     /**
88      * Callback interface for being notified that a new stream frame is available.
89      */
90     public interface OnFrameAvailableListener {
onFrameAvailable(SurfaceTexture surfaceTexture)91         void onFrameAvailable(SurfaceTexture surfaceTexture);
92     }
93 
94     /**
95      * Exception thrown when a SurfaceTexture couldn't be created or resized.
96      *
97      * @deprecated No longer thrown. {@link android.view.Surface.OutOfResourcesException}
98      * is used instead.
99      */
100     @SuppressWarnings("serial")
101     @Deprecated
102     public static class OutOfResourcesException extends Exception {
OutOfResourcesException()103         public OutOfResourcesException() {
104         }
OutOfResourcesException(String name)105         public OutOfResourcesException(String name) {
106             super(name);
107         }
108     }
109 
110     /**
111      * Construct a new SurfaceTexture to stream images to a given OpenGL texture.
112      *
113      * @param texName the OpenGL texture object name (e.g. generated via glGenTextures)
114      *
115      * @throws android.view.Surface.OutOfResourcesException If the SurfaceTexture cannot be created.
116      */
SurfaceTexture(int texName)117     public SurfaceTexture(int texName) {
118         this(texName, false);
119     }
120 
121     /**
122      * Construct a new SurfaceTexture to stream images to a given OpenGL texture.
123      *
124      * In single buffered mode the application is responsible for serializing access to the image
125      * content buffer. Each time the image content is to be updated, the
126      * {@link #releaseTexImage()} method must be called before the image content producer takes
127      * ownership of the buffer. For example, when producing image content with the NDK
128      * ANativeWindow_lock and ANativeWindow_unlockAndPost functions, {@link #releaseTexImage()}
129      * must be called before each ANativeWindow_lock, or that call will fail. When producing
130      * image content with OpenGL ES, {@link #releaseTexImage()} must be called before the first
131      * OpenGL ES function call each frame.
132      *
133      * @param texName the OpenGL texture object name (e.g. generated via glGenTextures)
134      * @param singleBufferMode whether the SurfaceTexture will be in single buffered mode.
135      *
136      * @throws android.view.Surface.OutOfResourcesException If the SurfaceTexture cannot be created.
137      */
SurfaceTexture(int texName, boolean singleBufferMode)138     public SurfaceTexture(int texName, boolean singleBufferMode) {
139         mCreatorLooper = Looper.myLooper();
140         mIsSingleBuffered = singleBufferMode;
141         nativeInit(false, texName, singleBufferMode, new WeakReference<SurfaceTexture>(this));
142     }
143 
144     /**
145      * Construct a new SurfaceTexture to stream images to a given OpenGL texture.
146      *
147      * In single buffered mode the application is responsible for serializing access to the image
148      * content buffer. Each time the image content is to be updated, the
149      * {@link #releaseTexImage()} method must be called before the image content producer takes
150      * ownership of the buffer. For example, when producing image content with the NDK
151      * ANativeWindow_lock and ANativeWindow_unlockAndPost functions, {@link #releaseTexImage()}
152      * must be called before each ANativeWindow_lock, or that call will fail. When producing
153      * image content with OpenGL ES, {@link #releaseTexImage()} must be called before the first
154      * OpenGL ES function call each frame.
155      *
156      * Unlike {@link #SurfaceTexture(int, boolean)}, which takes an OpenGL texture object name,
157      * this constructor creates the SurfaceTexture in detached mode. A texture name must be passed
158      * in using {@link #attachToGLContext} before calling {@link #releaseTexImage()} and producing
159      * image content using OpenGL ES.
160      *
161      * @param singleBufferMode whether the SurfaceTexture will be in single buffered mode.
162      *
163      * @throws android.view.Surface.OutOfResourcesException If the SurfaceTexture cannot be created.
164      */
SurfaceTexture(boolean singleBufferMode)165     public SurfaceTexture(boolean singleBufferMode) {
166         mCreatorLooper = Looper.myLooper();
167         mIsSingleBuffered = singleBufferMode;
168         nativeInit(true, 0, singleBufferMode, new WeakReference<SurfaceTexture>(this));
169     }
170 
171     /**
172      * Register a callback to be invoked when a new image frame becomes available to the
173      * SurfaceTexture.
174      * <p>
175      * The callback may be called on an arbitrary thread, so it is not
176      * safe to call {@link #updateTexImage} without first binding the OpenGL ES context to the
177      * thread invoking the callback.
178      * </p>
179      *
180      * @param listener The listener to use, or null to remove the listener.
181      */
setOnFrameAvailableListener(@ullable OnFrameAvailableListener listener)182     public void setOnFrameAvailableListener(@Nullable OnFrameAvailableListener listener) {
183         setOnFrameAvailableListener(listener, null);
184     }
185 
186     /**
187      * Register a callback to be invoked when a new image frame becomes available to the
188      * SurfaceTexture.
189      * <p>
190      * If a handler is specified, the callback will be invoked on that handler's thread.
191      * If no handler is specified, then the callback may be called on an arbitrary thread,
192      * so it is not safe to call {@link #updateTexImage} without first binding the OpenGL ES
193      * context to the thread invoking the callback.
194      * </p>
195      *
196      * @param listener The listener to use, or null to remove the listener.
197      * @param handler The handler on which the listener should be invoked, or null
198      * to use an arbitrary thread.
199      */
setOnFrameAvailableListener(@ullable final OnFrameAvailableListener listener, @Nullable Handler handler)200     public void setOnFrameAvailableListener(@Nullable final OnFrameAvailableListener listener,
201             @Nullable Handler handler) {
202         if (listener != null) {
203             // Although we claim the thread is arbitrary, earlier implementation would
204             // prefer to send the callback on the creating looper or the main looper
205             // so we preserve this behavior here.
206             Looper looper = handler != null ? handler.getLooper() :
207                     mCreatorLooper != null ? mCreatorLooper : Looper.getMainLooper();
208             mOnFrameAvailableHandler = new Handler(looper, null, true /*async*/) {
209                 @Override
210                 public void handleMessage(Message msg) {
211                     listener.onFrameAvailable(SurfaceTexture.this);
212                 }
213             };
214         } else {
215             mOnFrameAvailableHandler = null;
216         }
217     }
218 
219     /**
220      * Set the default size of the image buffers.  The image producer may override the buffer size,
221      * in which case the producer-set buffer size will be used, not the default size set by this
222      * method.  Both video and camera based image producers do override the size.  This method may
223      * be used to set the image size when producing images with {@link android.graphics.Canvas} (via
224      * {@link android.view.Surface#lockCanvas}), or OpenGL ES (via an EGLSurface).
225      *
226      * The new default buffer size will take effect the next time the image producer requests a
227      * buffer to fill.  For {@link android.graphics.Canvas} this will be the next time {@link
228      * android.view.Surface#lockCanvas} is called.  For OpenGL ES, the EGLSurface should be
229      * destroyed (via eglDestroySurface), made not-current (via eglMakeCurrent), and then recreated
230      * (via eglCreateWindowSurface) to ensure that the new default size has taken effect.
231      *
232      * The width and height parameters must be no greater than the minimum of
233      * GL_MAX_VIEWPORT_DIMS and GL_MAX_TEXTURE_SIZE (see
234      * {@link javax.microedition.khronos.opengles.GL10#glGetIntegerv glGetIntegerv}).
235      * An error due to invalid dimensions might not be reported until
236      * updateTexImage() is called.
237      */
setDefaultBufferSize(int width, int height)238     public void setDefaultBufferSize(int width, int height) {
239         nativeSetDefaultBufferSize(width, height);
240     }
241 
242     /**
243      * Update the texture image to the most recent frame from the image stream.  This may only be
244      * called while the OpenGL ES context that owns the texture is current on the calling thread.
245      * It will implicitly bind its texture to the GL_TEXTURE_EXTERNAL_OES texture target.
246      */
updateTexImage()247     public void updateTexImage() {
248         nativeUpdateTexImage();
249     }
250 
251     /**
252      * Releases the the texture content. This is needed in single buffered mode to allow the image
253      * content producer to take ownership of the image buffer.
254      * For more information see {@link #SurfaceTexture(int, boolean)}.
255      */
releaseTexImage()256     public void releaseTexImage() {
257         nativeReleaseTexImage();
258     }
259 
260     /**
261      * Detach the SurfaceTexture from the OpenGL ES context that owns the OpenGL ES texture object.
262      * This call must be made with the OpenGL ES context current on the calling thread.  The OpenGL
263      * ES texture object will be deleted as a result of this call.  After calling this method all
264      * calls to {@link #updateTexImage} will throw an {@link java.lang.IllegalStateException} until
265      * a successful call to {@link #attachToGLContext} is made.
266      *
267      * This can be used to access the SurfaceTexture image contents from multiple OpenGL ES
268      * contexts.  Note, however, that the image contents are only accessible from one OpenGL ES
269      * context at a time.
270      */
detachFromGLContext()271     public void detachFromGLContext() {
272         int err = nativeDetachFromGLContext();
273         if (err != 0) {
274             throw new RuntimeException("Error during detachFromGLContext (see logcat for details)");
275         }
276     }
277 
278     /**
279      * Attach the SurfaceTexture to the OpenGL ES context that is current on the calling thread.  A
280      * new OpenGL ES texture object is created and populated with the SurfaceTexture image frame
281      * that was current at the time of the last call to {@link #detachFromGLContext}.  This new
282      * texture is bound to the GL_TEXTURE_EXTERNAL_OES texture target.
283      *
284      * This can be used to access the SurfaceTexture image contents from multiple OpenGL ES
285      * contexts.  Note, however, that the image contents are only accessible from one OpenGL ES
286      * context at a time.
287      *
288      * @param texName The name of the OpenGL ES texture that will be created.  This texture name
289      * must be unusued in the OpenGL ES context that is current on the calling thread.
290      */
attachToGLContext(int texName)291     public void attachToGLContext(int texName) {
292         int err = nativeAttachToGLContext(texName);
293         if (err != 0) {
294             throw new RuntimeException("Error during attachToGLContext (see logcat for details)");
295         }
296     }
297 
298     /**
299      * Retrieve the 4x4 texture coordinate transform matrix associated with the texture image set by
300      * the most recent call to updateTexImage.
301      *
302      * This transform matrix maps 2D homogeneous texture coordinates of the form (s, t, 0, 1) with s
303      * and t in the inclusive range [0, 1] to the texture coordinate that should be used to sample
304      * that location from the texture.  Sampling the texture outside of the range of this transform
305      * is undefined.
306      *
307      * The matrix is stored in column-major order so that it may be passed directly to OpenGL ES via
308      * the glLoadMatrixf or glUniformMatrix4fv functions.
309      *
310      * @param mtx the array into which the 4x4 matrix will be stored.  The array must have exactly
311      *     16 elements.
312      */
getTransformMatrix(float[] mtx)313     public void getTransformMatrix(float[] mtx) {
314         // Note we intentionally don't check mtx for null, so this will result in a
315         // NullPointerException. But it's safe because it happens before the call to native.
316         if (mtx.length != 16) {
317             throw new IllegalArgumentException();
318         }
319         nativeGetTransformMatrix(mtx);
320     }
321 
322     /**
323      * Retrieve the timestamp associated with the texture image set by the most recent call to
324      * updateTexImage.
325      *
326      * <p>This timestamp is in nanoseconds, and is normally monotonically increasing. The timestamp
327      * should be unaffected by time-of-day adjustments. The specific meaning and zero point of the
328      * timestamp depends on the source providing images to the SurfaceTexture. Unless otherwise
329      * specified by the image source, timestamps cannot generally be compared across SurfaceTexture
330      * instances, or across multiple program invocations. It is mostly useful for determining time
331      * offsets between subsequent frames.</p>
332      *
333      * <p>For camera sources, timestamps should be strictly monotonic. Timestamps from MediaPlayer
334      * sources may be reset when the playback position is set. For EGL and Vulkan producers, the
335      * timestamp is the desired present time set with the EGL_ANDROID_presentation_time or
336      * VK_GOOGLE_display_timing extensions.</p>
337      */
338 
getTimestamp()339     public long getTimestamp() {
340         return nativeGetTimestamp();
341     }
342 
343     /**
344      * release() frees all the buffers and puts the SurfaceTexture into the
345      * 'abandoned' state. Once put in this state the SurfaceTexture can never
346      * leave it. When in the 'abandoned' state, all methods of the
347      * IGraphicBufferProducer interface will fail with the NO_INIT error.
348      *
349      * Note that while calling this method causes all the buffers to be freed
350      * from the perspective of the the SurfaceTexture, if there are additional
351      * references on the buffers (e.g. if a buffer is referenced by a client or
352      * by OpenGL ES as a texture) then those buffer will remain allocated.
353      *
354      * Always call this method when you are done with SurfaceTexture. Failing
355      * to do so may delay resource deallocation for a significant amount of
356      * time.
357      *
358      * @see #isReleased()
359      */
release()360     public void release() {
361         nativeRelease();
362     }
363 
364     /**
365      * Returns true if the SurfaceTexture was released.
366      *
367      * @see #release()
368      */
isReleased()369     public boolean isReleased() {
370         return nativeIsReleased();
371     }
372 
373     @Override
finalize()374     protected void finalize() throws Throwable {
375         try {
376             nativeFinalize();
377         } finally {
378             super.finalize();
379         }
380     }
381 
382     /**
383      * This method is invoked from native code only.
384      */
385     @SuppressWarnings({"UnusedDeclaration"})
386     @UnsupportedAppUsage
postEventFromNative(WeakReference<SurfaceTexture> weakSelf)387     private static void postEventFromNative(WeakReference<SurfaceTexture> weakSelf) {
388         SurfaceTexture st = weakSelf.get();
389         if (st != null) {
390             Handler handler = st.mOnFrameAvailableHandler;
391             if (handler != null) {
392                 handler.sendEmptyMessage(0);
393             }
394         }
395     }
396 
397     /**
398      * Returns true if the SurfaceTexture is single-buffered
399      * @hide
400      */
isSingleBuffered()401     public boolean isSingleBuffered() {
402         return mIsSingleBuffered;
403     }
404 
nativeInit(boolean isDetached, int texName, boolean singleBufferMode, WeakReference<SurfaceTexture> weakSelf)405     private native void nativeInit(boolean isDetached, int texName,
406             boolean singleBufferMode, WeakReference<SurfaceTexture> weakSelf)
407             throws Surface.OutOfResourcesException;
nativeFinalize()408     private native void nativeFinalize();
nativeGetTransformMatrix(float[] mtx)409     private native void nativeGetTransformMatrix(float[] mtx);
nativeGetTimestamp()410     private native long nativeGetTimestamp();
nativeSetDefaultBufferSize(int width, int height)411     private native void nativeSetDefaultBufferSize(int width, int height);
nativeUpdateTexImage()412     private native void nativeUpdateTexImage();
nativeReleaseTexImage()413     private native void nativeReleaseTexImage();
414     @UnsupportedAppUsage
nativeDetachFromGLContext()415     private native int nativeDetachFromGLContext();
nativeAttachToGLContext(int texName)416     private native int nativeAttachToGLContext(int texName);
nativeRelease()417     private native void nativeRelease();
nativeIsReleased()418     private native boolean nativeIsReleased();
419 }
420