1 /* 2 * Copyright (C) 2014 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package android.hardware.camera2.legacy; 17 18 import android.graphics.ImageFormat; 19 import android.graphics.RectF; 20 import android.graphics.SurfaceTexture; 21 import android.hardware.camera2.CameraCharacteristics; 22 import android.os.Environment; 23 import android.opengl.EGL14; 24 import android.opengl.EGLConfig; 25 import android.opengl.EGLContext; 26 import android.opengl.EGLDisplay; 27 import android.opengl.EGLSurface; 28 import android.opengl.GLES11Ext; 29 import android.opengl.GLES20; 30 import android.opengl.Matrix; 31 import android.text.format.Time; 32 import android.util.Log; 33 import android.util.Pair; 34 import android.util.Size; 35 import android.view.Surface; 36 import android.os.SystemProperties; 37 38 import java.io.File; 39 import java.nio.ByteBuffer; 40 import java.nio.ByteOrder; 41 import java.nio.FloatBuffer; 42 import java.util.ArrayList; 43 import java.util.Collection; 44 import java.util.List; 45 46 /** 47 * A renderer class that manages the GL state, and can draw a frame into a set of output 48 * {@link Surface}s. 49 */ 50 public class SurfaceTextureRenderer { 51 private static final String TAG = SurfaceTextureRenderer.class.getSimpleName(); 52 private static final boolean DEBUG = false; 53 private static final int EGL_RECORDABLE_ANDROID = 0x3142; // from EGL/eglext.h 54 private static final int GL_MATRIX_SIZE = 16; 55 private static final int VERTEX_POS_SIZE = 3; 56 private static final int VERTEX_UV_SIZE = 2; 57 private static final int EGL_COLOR_BITLENGTH = 8; 58 private static final int GLES_VERSION = 2; 59 private static final int PBUFFER_PIXEL_BYTES = 4; 60 61 private static final int FLIP_TYPE_NONE = 0; 62 private static final int FLIP_TYPE_HORIZONTAL = 1; 63 private static final int FLIP_TYPE_VERTICAL = 2; 64 private static final int FLIP_TYPE_BOTH = FLIP_TYPE_HORIZONTAL | FLIP_TYPE_VERTICAL; 65 66 private EGLDisplay mEGLDisplay = EGL14.EGL_NO_DISPLAY; 67 private EGLContext mEGLContext = EGL14.EGL_NO_CONTEXT; 68 private EGLConfig mConfigs; 69 70 private class EGLSurfaceHolder { 71 Surface surface; 72 EGLSurface eglSurface; 73 int width; 74 int height; 75 } 76 77 private List<EGLSurfaceHolder> mSurfaces = new ArrayList<EGLSurfaceHolder>(); 78 private List<EGLSurfaceHolder> mConversionSurfaces = new ArrayList<EGLSurfaceHolder>(); 79 80 private ByteBuffer mPBufferPixels; 81 82 // Hold this to avoid GC 83 private volatile SurfaceTexture mSurfaceTexture; 84 85 private static final int FLOAT_SIZE_BYTES = 4; 86 private static final int TRIANGLE_VERTICES_DATA_STRIDE_BYTES = 5 * FLOAT_SIZE_BYTES; 87 private static final int TRIANGLE_VERTICES_DATA_POS_OFFSET = 0; 88 private static final int TRIANGLE_VERTICES_DATA_UV_OFFSET = 3; 89 90 // Sampling is mirrored across the horizontal axis 91 private static final float[] sHorizontalFlipTriangleVertices = { 92 // X, Y, Z, U, V 93 -1.0f, -1.0f, 0, 1.f, 0.f, 94 1.0f, -1.0f, 0, 0.f, 0.f, 95 -1.0f, 1.0f, 0, 1.f, 1.f, 96 1.0f, 1.0f, 0, 0.f, 1.f, 97 }; 98 99 // Sampling is mirrored across the vertical axis 100 private static final float[] sVerticalFlipTriangleVertices = { 101 // X, Y, Z, U, V 102 -1.0f, -1.0f, 0, 0.f, 1.f, 103 1.0f, -1.0f, 0, 1.f, 1.f, 104 -1.0f, 1.0f, 0, 0.f, 0.f, 105 1.0f, 1.0f, 0, 1.f, 0.f, 106 }; 107 108 // Sampling is mirrored across the both axes 109 private static final float[] sBothFlipTriangleVertices = { 110 // X, Y, Z, U, V 111 -1.0f, -1.0f, 0, 1.f, 1.f, 112 1.0f, -1.0f, 0, 0.f, 1.f, 113 -1.0f, 1.0f, 0, 1.f, 0.f, 114 1.0f, 1.0f, 0, 0.f, 0.f, 115 }; 116 117 // Sampling is 1:1 for a straight copy for the back camera 118 private static final float[] sRegularTriangleVertices = { 119 // X, Y, Z, U, V 120 -1.0f, -1.0f, 0, 0.f, 0.f, 121 1.0f, -1.0f, 0, 1.f, 0.f, 122 -1.0f, 1.0f, 0, 0.f, 1.f, 123 1.0f, 1.0f, 0, 1.f, 1.f, 124 }; 125 126 private FloatBuffer mRegularTriangleVertices; 127 private FloatBuffer mHorizontalFlipTriangleVertices; 128 private FloatBuffer mVerticalFlipTriangleVertices; 129 private FloatBuffer mBothFlipTriangleVertices; 130 private final int mFacing; 131 132 /** 133 * As used in this file, this vertex shader maps a unit square to the view, and 134 * tells the fragment shader to interpolate over it. Each surface pixel position 135 * is mapped to a 2D homogeneous texture coordinate of the form (s, t, 0, 1) with 136 * s and t in the inclusive range [0, 1], and the matrix from 137 * {@link SurfaceTexture#getTransformMatrix(float[])} is used to map this 138 * coordinate to a texture location. 139 */ 140 private static final String VERTEX_SHADER = 141 "uniform mat4 uMVPMatrix;\n" + 142 "uniform mat4 uSTMatrix;\n" + 143 "attribute vec4 aPosition;\n" + 144 "attribute vec4 aTextureCoord;\n" + 145 "varying vec2 vTextureCoord;\n" + 146 "void main() {\n" + 147 " gl_Position = uMVPMatrix * aPosition;\n" + 148 " vTextureCoord = (uSTMatrix * aTextureCoord).xy;\n" + 149 "}\n"; 150 151 /** 152 * This fragment shader simply draws the color in the 2D texture at 153 * the location from the {@code VERTEX_SHADER}. 154 */ 155 private static final String FRAGMENT_SHADER = 156 "#extension GL_OES_EGL_image_external : require\n" + 157 "precision mediump float;\n" + 158 "varying vec2 vTextureCoord;\n" + 159 "uniform samplerExternalOES sTexture;\n" + 160 "void main() {\n" + 161 " gl_FragColor = texture2D(sTexture, vTextureCoord);\n" + 162 "}\n"; 163 164 private float[] mMVPMatrix = new float[GL_MATRIX_SIZE]; 165 private float[] mSTMatrix = new float[GL_MATRIX_SIZE]; 166 167 private int mProgram; 168 private int mTextureID = 0; 169 private int muMVPMatrixHandle; 170 private int muSTMatrixHandle; 171 private int maPositionHandle; 172 private int maTextureHandle; 173 174 private PerfMeasurement mPerfMeasurer = null; 175 private static final String LEGACY_PERF_PROPERTY = "persist.camera.legacy_perf"; 176 SurfaceTextureRenderer(int facing)177 public SurfaceTextureRenderer(int facing) { 178 mFacing = facing; 179 180 mRegularTriangleVertices = ByteBuffer.allocateDirect(sRegularTriangleVertices.length * 181 FLOAT_SIZE_BYTES).order(ByteOrder.nativeOrder()).asFloatBuffer(); 182 mRegularTriangleVertices.put(sRegularTriangleVertices).position(0); 183 184 mHorizontalFlipTriangleVertices = ByteBuffer.allocateDirect( 185 sHorizontalFlipTriangleVertices.length * FLOAT_SIZE_BYTES). 186 order(ByteOrder.nativeOrder()).asFloatBuffer(); 187 mHorizontalFlipTriangleVertices.put(sHorizontalFlipTriangleVertices).position(0); 188 189 mVerticalFlipTriangleVertices = ByteBuffer.allocateDirect( 190 sVerticalFlipTriangleVertices.length * FLOAT_SIZE_BYTES). 191 order(ByteOrder.nativeOrder()).asFloatBuffer(); 192 mVerticalFlipTriangleVertices.put(sVerticalFlipTriangleVertices).position(0); 193 194 mBothFlipTriangleVertices = ByteBuffer.allocateDirect( 195 sBothFlipTriangleVertices.length * FLOAT_SIZE_BYTES). 196 order(ByteOrder.nativeOrder()).asFloatBuffer(); 197 mBothFlipTriangleVertices.put(sBothFlipTriangleVertices).position(0); 198 199 Matrix.setIdentityM(mSTMatrix, 0); 200 } 201 loadShader(int shaderType, String source)202 private int loadShader(int shaderType, String source) { 203 int shader = GLES20.glCreateShader(shaderType); 204 checkGlError("glCreateShader type=" + shaderType); 205 GLES20.glShaderSource(shader, source); 206 GLES20.glCompileShader(shader); 207 int[] compiled = new int[1]; 208 GLES20.glGetShaderiv(shader, GLES20.GL_COMPILE_STATUS, compiled, 0); 209 if (compiled[0] == 0) { 210 Log.e(TAG, "Could not compile shader " + shaderType + ":"); 211 Log.e(TAG, " " + GLES20.glGetShaderInfoLog(shader)); 212 GLES20.glDeleteShader(shader); 213 // TODO: handle this more gracefully 214 throw new IllegalStateException("Could not compile shader " + shaderType); 215 } 216 return shader; 217 } 218 createProgram(String vertexSource, String fragmentSource)219 private int createProgram(String vertexSource, String fragmentSource) { 220 int vertexShader = loadShader(GLES20.GL_VERTEX_SHADER, vertexSource); 221 if (vertexShader == 0) { 222 return 0; 223 } 224 int pixelShader = loadShader(GLES20.GL_FRAGMENT_SHADER, fragmentSource); 225 if (pixelShader == 0) { 226 return 0; 227 } 228 229 int program = GLES20.glCreateProgram(); 230 checkGlError("glCreateProgram"); 231 if (program == 0) { 232 Log.e(TAG, "Could not create program"); 233 } 234 GLES20.glAttachShader(program, vertexShader); 235 checkGlError("glAttachShader"); 236 GLES20.glAttachShader(program, pixelShader); 237 checkGlError("glAttachShader"); 238 GLES20.glLinkProgram(program); 239 int[] linkStatus = new int[1]; 240 GLES20.glGetProgramiv(program, GLES20.GL_LINK_STATUS, linkStatus, 0); 241 if (linkStatus[0] != GLES20.GL_TRUE) { 242 Log.e(TAG, "Could not link program: "); 243 Log.e(TAG, GLES20.glGetProgramInfoLog(program)); 244 GLES20.glDeleteProgram(program); 245 // TODO: handle this more gracefully 246 throw new IllegalStateException("Could not link program"); 247 } 248 return program; 249 } 250 drawFrame(SurfaceTexture st, int width, int height, int flipType)251 private void drawFrame(SurfaceTexture st, int width, int height, int flipType) 252 throws LegacyExceptionUtils.BufferQueueAbandonedException { 253 checkGlError("onDrawFrame start"); 254 st.getTransformMatrix(mSTMatrix); 255 256 Matrix.setIdentityM(mMVPMatrix, /*smOffset*/0); 257 258 // Find intermediate buffer dimensions 259 Size dimens; 260 try { 261 dimens = LegacyCameraDevice.getTextureSize(st); 262 } catch (LegacyExceptionUtils.BufferQueueAbandonedException e) { 263 // Should never hit this. 264 throw new IllegalStateException("Surface abandoned, skipping drawFrame...", e); 265 } 266 float texWidth = dimens.getWidth(); 267 float texHeight = dimens.getHeight(); 268 269 if (texWidth <= 0 || texHeight <= 0) { 270 throw new IllegalStateException("Illegal intermediate texture with dimension of 0"); 271 } 272 273 // Letterbox or pillar-box output dimensions into intermediate dimensions. 274 RectF intermediate = new RectF(/*left*/0, /*top*/0, /*right*/texWidth, /*bottom*/texHeight); 275 RectF output = new RectF(/*left*/0, /*top*/0, /*right*/width, /*bottom*/height); 276 android.graphics.Matrix boxingXform = new android.graphics.Matrix(); 277 boxingXform.setRectToRect(output, intermediate, android.graphics.Matrix.ScaleToFit.CENTER); 278 boxingXform.mapRect(output); 279 280 // Find scaling factor from pillar-boxed/letter-boxed output dimensions to intermediate 281 // buffer dimensions. 282 float scaleX = intermediate.width() / output.width(); 283 float scaleY = intermediate.height() / output.height(); 284 285 // Intermediate texture is implicitly scaled to 'fill' the output dimensions in clip space 286 // coordinates in the shader. To avoid stretching, we need to scale the larger dimension 287 // of the intermediate buffer so that the output buffer is actually letter-boxed 288 // or pillar-boxed into the intermediate buffer after clipping. 289 Matrix.scaleM(mMVPMatrix, /*offset*/0, /*x*/scaleX, /*y*/scaleY, /*z*/1); 290 291 if (DEBUG) { 292 Log.d(TAG, "Scaling factors (S_x = " + scaleX + ",S_y = " + scaleY + ") used for " + 293 width + "x" + height + " surface, intermediate buffer size is " + texWidth + 294 "x" + texHeight); 295 } 296 297 // Set viewport to be output buffer dimensions 298 GLES20.glViewport(0, 0, width, height); 299 300 if (DEBUG) { 301 GLES20.glClearColor(1.0f, 0.0f, 0.0f, 1.0f); 302 GLES20.glClear(GLES20.GL_DEPTH_BUFFER_BIT | GLES20.GL_COLOR_BUFFER_BIT); 303 } 304 305 GLES20.glUseProgram(mProgram); 306 checkGlError("glUseProgram"); 307 308 GLES20.glActiveTexture(GLES20.GL_TEXTURE0); 309 GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, mTextureID); 310 311 FloatBuffer triangleVertices; 312 switch(flipType) { 313 case FLIP_TYPE_HORIZONTAL: 314 triangleVertices = mHorizontalFlipTriangleVertices; 315 break; 316 case FLIP_TYPE_VERTICAL: 317 triangleVertices = mVerticalFlipTriangleVertices; 318 break; 319 case FLIP_TYPE_BOTH: 320 triangleVertices = mBothFlipTriangleVertices; 321 break; 322 default: 323 triangleVertices = mRegularTriangleVertices; 324 break; 325 } 326 327 triangleVertices.position(TRIANGLE_VERTICES_DATA_POS_OFFSET); 328 GLES20.glVertexAttribPointer(maPositionHandle, VERTEX_POS_SIZE, GLES20.GL_FLOAT, 329 /*normalized*/ false, TRIANGLE_VERTICES_DATA_STRIDE_BYTES, triangleVertices); 330 checkGlError("glVertexAttribPointer maPosition"); 331 GLES20.glEnableVertexAttribArray(maPositionHandle); 332 checkGlError("glEnableVertexAttribArray maPositionHandle"); 333 334 triangleVertices.position(TRIANGLE_VERTICES_DATA_UV_OFFSET); 335 GLES20.glVertexAttribPointer(maTextureHandle, VERTEX_UV_SIZE, GLES20.GL_FLOAT, 336 /*normalized*/ false, TRIANGLE_VERTICES_DATA_STRIDE_BYTES, triangleVertices); 337 checkGlError("glVertexAttribPointer maTextureHandle"); 338 GLES20.glEnableVertexAttribArray(maTextureHandle); 339 checkGlError("glEnableVertexAttribArray maTextureHandle"); 340 341 GLES20.glUniformMatrix4fv(muMVPMatrixHandle, /*count*/ 1, /*transpose*/ false, mMVPMatrix, 342 /*offset*/ 0); 343 GLES20.glUniformMatrix4fv(muSTMatrixHandle, /*count*/ 1, /*transpose*/ false, mSTMatrix, 344 /*offset*/ 0); 345 346 GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, /*offset*/ 0, /*count*/ 4); 347 checkGlDrawError("glDrawArrays"); 348 } 349 350 /** 351 * Initializes GL state. Call this after the EGL surface has been created and made current. 352 */ initializeGLState()353 private void initializeGLState() { 354 mProgram = createProgram(VERTEX_SHADER, FRAGMENT_SHADER); 355 if (mProgram == 0) { 356 throw new IllegalStateException("failed creating program"); 357 } 358 maPositionHandle = GLES20.glGetAttribLocation(mProgram, "aPosition"); 359 checkGlError("glGetAttribLocation aPosition"); 360 if (maPositionHandle == -1) { 361 throw new IllegalStateException("Could not get attrib location for aPosition"); 362 } 363 maTextureHandle = GLES20.glGetAttribLocation(mProgram, "aTextureCoord"); 364 checkGlError("glGetAttribLocation aTextureCoord"); 365 if (maTextureHandle == -1) { 366 throw new IllegalStateException("Could not get attrib location for aTextureCoord"); 367 } 368 369 muMVPMatrixHandle = GLES20.glGetUniformLocation(mProgram, "uMVPMatrix"); 370 checkGlError("glGetUniformLocation uMVPMatrix"); 371 if (muMVPMatrixHandle == -1) { 372 throw new IllegalStateException("Could not get attrib location for uMVPMatrix"); 373 } 374 375 muSTMatrixHandle = GLES20.glGetUniformLocation(mProgram, "uSTMatrix"); 376 checkGlError("glGetUniformLocation uSTMatrix"); 377 if (muSTMatrixHandle == -1) { 378 throw new IllegalStateException("Could not get attrib location for uSTMatrix"); 379 } 380 381 int[] textures = new int[1]; 382 GLES20.glGenTextures(/*n*/ 1, textures, /*offset*/ 0); 383 384 mTextureID = textures[0]; 385 GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, mTextureID); 386 checkGlError("glBindTexture mTextureID"); 387 388 GLES20.glTexParameterf(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_MIN_FILTER, 389 GLES20.GL_NEAREST); 390 GLES20.glTexParameterf(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_MAG_FILTER, 391 GLES20.GL_LINEAR); 392 GLES20.glTexParameteri(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_WRAP_S, 393 GLES20.GL_CLAMP_TO_EDGE); 394 GLES20.glTexParameteri(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_WRAP_T, 395 GLES20.GL_CLAMP_TO_EDGE); 396 checkGlError("glTexParameter"); 397 } 398 getTextureId()399 private int getTextureId() { 400 return mTextureID; 401 } 402 clearState()403 private void clearState() { 404 mSurfaces.clear(); 405 for (EGLSurfaceHolder holder : mConversionSurfaces) { 406 try { 407 LegacyCameraDevice.disconnectSurface(holder.surface); 408 } catch (LegacyExceptionUtils.BufferQueueAbandonedException e) { 409 Log.w(TAG, "Surface abandoned, skipping...", e); 410 } 411 } 412 mConversionSurfaces.clear(); 413 mPBufferPixels = null; 414 if (mSurfaceTexture != null) { 415 mSurfaceTexture.release(); 416 } 417 mSurfaceTexture = null; 418 } 419 configureEGLContext()420 private void configureEGLContext() { 421 mEGLDisplay = EGL14.eglGetDisplay(EGL14.EGL_DEFAULT_DISPLAY); 422 if (mEGLDisplay == EGL14.EGL_NO_DISPLAY) { 423 throw new IllegalStateException("No EGL14 display"); 424 } 425 int[] version = new int[2]; 426 if (!EGL14.eglInitialize(mEGLDisplay, version, /*offset*/ 0, version, /*offset*/ 1)) { 427 throw new IllegalStateException("Cannot initialize EGL14"); 428 } 429 430 int[] attribList = { 431 EGL14.EGL_RED_SIZE, EGL_COLOR_BITLENGTH, 432 EGL14.EGL_GREEN_SIZE, EGL_COLOR_BITLENGTH, 433 EGL14.EGL_BLUE_SIZE, EGL_COLOR_BITLENGTH, 434 EGL14.EGL_RENDERABLE_TYPE, EGL14.EGL_OPENGL_ES2_BIT, 435 EGL_RECORDABLE_ANDROID, 1, 436 EGL14.EGL_SURFACE_TYPE, EGL14.EGL_PBUFFER_BIT | EGL14.EGL_WINDOW_BIT, 437 EGL14.EGL_NONE 438 }; 439 EGLConfig[] configs = new EGLConfig[1]; 440 int[] numConfigs = new int[1]; 441 EGL14.eglChooseConfig(mEGLDisplay, attribList, /*offset*/ 0, configs, /*offset*/ 0, 442 configs.length, numConfigs, /*offset*/ 0); 443 checkEglError("eglCreateContext RGB888+recordable ES2"); 444 mConfigs = configs[0]; 445 int[] attrib_list = { 446 EGL14.EGL_CONTEXT_CLIENT_VERSION, GLES_VERSION, 447 EGL14.EGL_NONE 448 }; 449 mEGLContext = EGL14.eglCreateContext(mEGLDisplay, configs[0], EGL14.EGL_NO_CONTEXT, 450 attrib_list, /*offset*/ 0); 451 checkEglError("eglCreateContext"); 452 if(mEGLContext == EGL14.EGL_NO_CONTEXT) { 453 throw new IllegalStateException("No EGLContext could be made"); 454 } 455 } 456 configureEGLOutputSurfaces(Collection<EGLSurfaceHolder> surfaces)457 private void configureEGLOutputSurfaces(Collection<EGLSurfaceHolder> surfaces) { 458 if (surfaces == null || surfaces.size() == 0) { 459 throw new IllegalStateException("No Surfaces were provided to draw to"); 460 } 461 int[] surfaceAttribs = { 462 EGL14.EGL_NONE 463 }; 464 for (EGLSurfaceHolder holder : surfaces) { 465 holder.eglSurface = EGL14.eglCreateWindowSurface(mEGLDisplay, mConfigs, 466 holder.surface, surfaceAttribs, /*offset*/ 0); 467 checkEglError("eglCreateWindowSurface"); 468 } 469 } 470 configureEGLPbufferSurfaces(Collection<EGLSurfaceHolder> surfaces)471 private void configureEGLPbufferSurfaces(Collection<EGLSurfaceHolder> surfaces) { 472 if (surfaces == null || surfaces.size() == 0) { 473 throw new IllegalStateException("No Surfaces were provided to draw to"); 474 } 475 476 int maxLength = 0; 477 for (EGLSurfaceHolder holder : surfaces) { 478 int length = holder.width * holder.height; 479 // Find max surface size, ensure PBuffer can hold this many pixels 480 maxLength = (length > maxLength) ? length : maxLength; 481 int[] surfaceAttribs = { 482 EGL14.EGL_WIDTH, holder.width, 483 EGL14.EGL_HEIGHT, holder.height, 484 EGL14.EGL_NONE 485 }; 486 holder.eglSurface = 487 EGL14.eglCreatePbufferSurface(mEGLDisplay, mConfigs, surfaceAttribs, 0); 488 checkEglError("eglCreatePbufferSurface"); 489 } 490 mPBufferPixels = ByteBuffer.allocateDirect(maxLength * PBUFFER_PIXEL_BYTES) 491 .order(ByteOrder.nativeOrder()); 492 } 493 releaseEGLContext()494 private void releaseEGLContext() { 495 if (mEGLDisplay != EGL14.EGL_NO_DISPLAY) { 496 EGL14.eglMakeCurrent(mEGLDisplay, EGL14.EGL_NO_SURFACE, EGL14.EGL_NO_SURFACE, 497 EGL14.EGL_NO_CONTEXT); 498 dumpGlTiming(); 499 if (mSurfaces != null) { 500 for (EGLSurfaceHolder holder : mSurfaces) { 501 if (holder.eglSurface != null) { 502 EGL14.eglDestroySurface(mEGLDisplay, holder.eglSurface); 503 } 504 } 505 } 506 if (mConversionSurfaces != null) { 507 for (EGLSurfaceHolder holder : mConversionSurfaces) { 508 if (holder.eglSurface != null) { 509 EGL14.eglDestroySurface(mEGLDisplay, holder.eglSurface); 510 } 511 } 512 } 513 EGL14.eglDestroyContext(mEGLDisplay, mEGLContext); 514 EGL14.eglReleaseThread(); 515 EGL14.eglTerminate(mEGLDisplay); 516 } 517 518 mConfigs = null; 519 mEGLDisplay = EGL14.EGL_NO_DISPLAY; 520 mEGLContext = EGL14.EGL_NO_CONTEXT; 521 clearState(); 522 } 523 makeCurrent(EGLSurface surface)524 private void makeCurrent(EGLSurface surface) 525 throws LegacyExceptionUtils.BufferQueueAbandonedException { 526 EGL14.eglMakeCurrent(mEGLDisplay, surface, surface, mEGLContext); 527 checkEglDrawError("makeCurrent"); 528 } 529 swapBuffers(EGLSurface surface)530 private boolean swapBuffers(EGLSurface surface) 531 throws LegacyExceptionUtils.BufferQueueAbandonedException { 532 boolean result = EGL14.eglSwapBuffers(mEGLDisplay, surface); 533 534 int error = EGL14.eglGetError(); 535 switch (error) { 536 case EGL14.EGL_SUCCESS: 537 return result; 538 539 // Check for an abandoned buffer queue, or other error conditions out 540 // of the user's control. 541 // 542 // From the EGL 1.4 spec (2013-12-04), Section 3.9.4 Posting Errors: 543 // 544 // If eglSwapBuffers is called and the native window associated with 545 // surface is no longer valid, an EGL_BAD_NATIVE_WINDOW error is 546 // generated. 547 // 548 // We also interpret EGL_BAD_SURFACE as indicating an abandoned 549 // surface, even though the EGL spec does not document it as such, for 550 // backwards compatibility with older versions of this file. 551 case EGL14.EGL_BAD_NATIVE_WINDOW: 552 case EGL14.EGL_BAD_SURFACE: 553 throw new LegacyExceptionUtils.BufferQueueAbandonedException(); 554 555 default: 556 throw new IllegalStateException( 557 "swapBuffers: EGL error: 0x" + Integer.toHexString(error)); 558 } 559 } 560 checkEglDrawError(String msg)561 private void checkEglDrawError(String msg) 562 throws LegacyExceptionUtils.BufferQueueAbandonedException { 563 int error; 564 if ((error = EGL14.eglGetError()) == EGL14.EGL_BAD_NATIVE_WINDOW) { 565 throw new LegacyExceptionUtils.BufferQueueAbandonedException(); 566 } 567 if ((error = EGL14.eglGetError()) != EGL14.EGL_SUCCESS) { 568 throw new IllegalStateException(msg + ": EGL error: 0x" + Integer.toHexString(error)); 569 } 570 } 571 checkEglError(String msg)572 private void checkEglError(String msg) { 573 int error; 574 if ((error = EGL14.eglGetError()) != EGL14.EGL_SUCCESS) { 575 throw new IllegalStateException(msg + ": EGL error: 0x" + Integer.toHexString(error)); 576 } 577 } 578 checkGlError(String msg)579 private void checkGlError(String msg) { 580 int error; 581 while ((error = GLES20.glGetError()) != GLES20.GL_NO_ERROR) { 582 throw new IllegalStateException( 583 msg + ": GLES20 error: 0x" + Integer.toHexString(error)); 584 } 585 } 586 checkGlDrawError(String msg)587 private void checkGlDrawError(String msg) 588 throws LegacyExceptionUtils.BufferQueueAbandonedException { 589 int error; 590 boolean surfaceAbandoned = false; 591 boolean glError = false; 592 while ((error = GLES20.glGetError()) != GLES20.GL_NO_ERROR) { 593 if (error == GLES20.GL_OUT_OF_MEMORY) { 594 surfaceAbandoned = true; 595 } else { 596 glError = true; 597 } 598 } 599 if (glError) { 600 throw new IllegalStateException( 601 msg + ": GLES20 error: 0x" + Integer.toHexString(error)); 602 } 603 if (surfaceAbandoned) { 604 throw new LegacyExceptionUtils.BufferQueueAbandonedException(); 605 } 606 } 607 608 /** 609 * Save a measurement dump to disk, in 610 * {@code /sdcard/CameraLegacy/durations_<time>_<width1>x<height1>_...txt} 611 */ dumpGlTiming()612 private void dumpGlTiming() { 613 if (mPerfMeasurer == null) return; 614 615 File legacyStorageDir = new File(Environment.getExternalStorageDirectory(), "CameraLegacy"); 616 if (!legacyStorageDir.exists()){ 617 if (!legacyStorageDir.mkdirs()){ 618 Log.e(TAG, "Failed to create directory for data dump"); 619 return; 620 } 621 } 622 623 StringBuilder path = new StringBuilder(legacyStorageDir.getPath()); 624 path.append(File.separator); 625 path.append("durations_"); 626 627 Time now = new Time(); 628 now.setToNow(); 629 path.append(now.format2445()); 630 path.append("_S"); 631 for (EGLSurfaceHolder surface : mSurfaces) { 632 path.append(String.format("_%d_%d", surface.width, surface.height)); 633 } 634 path.append("_C"); 635 for (EGLSurfaceHolder surface : mConversionSurfaces) { 636 path.append(String.format("_%d_%d", surface.width, surface.height)); 637 } 638 path.append(".txt"); 639 mPerfMeasurer.dumpPerformanceData(path.toString()); 640 } 641 setupGlTiming()642 private void setupGlTiming() { 643 if (PerfMeasurement.isGlTimingSupported()) { 644 Log.d(TAG, "Enabling GL performance measurement"); 645 mPerfMeasurer = new PerfMeasurement(); 646 } else { 647 Log.d(TAG, "GL performance measurement not supported on this device"); 648 mPerfMeasurer = null; 649 } 650 } 651 beginGlTiming()652 private void beginGlTiming() { 653 if (mPerfMeasurer == null) return; 654 mPerfMeasurer.startTimer(); 655 } 656 addGlTimestamp(long timestamp)657 private void addGlTimestamp(long timestamp) { 658 if (mPerfMeasurer == null) return; 659 mPerfMeasurer.addTimestamp(timestamp); 660 } 661 endGlTiming()662 private void endGlTiming() { 663 if (mPerfMeasurer == null) return; 664 mPerfMeasurer.stopTimer(); 665 } 666 667 /** 668 * Return the surface texture to draw to - this is the texture use to when producing output 669 * surface buffers. 670 * 671 * @return a {@link SurfaceTexture}. 672 */ getSurfaceTexture()673 public SurfaceTexture getSurfaceTexture() { 674 return mSurfaceTexture; 675 } 676 677 /** 678 * Set a collection of output {@link Surface}s that can be drawn to. 679 * 680 * @param surfaces a {@link Collection} of surfaces. 681 */ configureSurfaces(Collection<Pair<Surface, Size>> surfaces)682 public void configureSurfaces(Collection<Pair<Surface, Size>> surfaces) { 683 releaseEGLContext(); 684 685 if (surfaces == null || surfaces.size() == 0) { 686 Log.w(TAG, "No output surfaces configured for GL drawing."); 687 return; 688 } 689 690 for (Pair<Surface, Size> p : surfaces) { 691 Surface s = p.first; 692 Size surfaceSize = p.second; 693 // If pixel conversions aren't handled by egl, use a pbuffer 694 try { 695 EGLSurfaceHolder holder = new EGLSurfaceHolder(); 696 holder.surface = s; 697 holder.width = surfaceSize.getWidth(); 698 holder.height = surfaceSize.getHeight(); 699 if (LegacyCameraDevice.needsConversion(s)) { 700 mConversionSurfaces.add(holder); 701 // LegacyCameraDevice is the producer of surfaces if it's not handled by EGL, 702 // so LegacyCameraDevice needs to connect to the surfaces. 703 LegacyCameraDevice.connectSurface(s); 704 } else { 705 mSurfaces.add(holder); 706 } 707 } catch (LegacyExceptionUtils.BufferQueueAbandonedException e) { 708 Log.w(TAG, "Surface abandoned, skipping configuration... ", e); 709 } 710 } 711 712 // Set up egl display 713 configureEGLContext(); 714 715 // Set up regular egl surfaces if needed 716 if (mSurfaces.size() > 0) { 717 configureEGLOutputSurfaces(mSurfaces); 718 } 719 720 // Set up pbuffer surface if needed 721 if (mConversionSurfaces.size() > 0) { 722 configureEGLPbufferSurfaces(mConversionSurfaces); 723 } 724 725 try { 726 makeCurrent((mSurfaces.size() > 0) ? mSurfaces.get(0).eglSurface : 727 mConversionSurfaces.get(0).eglSurface); 728 } catch (LegacyExceptionUtils.BufferQueueAbandonedException e) { 729 Log.w(TAG, "Surface abandoned, skipping configuration... ", e); 730 } 731 732 initializeGLState(); 733 mSurfaceTexture = new SurfaceTexture(getTextureId()); 734 735 // Set up performance tracking if enabled 736 if (SystemProperties.getBoolean(LEGACY_PERF_PROPERTY, false)) { 737 setupGlTiming(); 738 } 739 } 740 741 /** 742 * Draw the current buffer in the {@link SurfaceTexture} returned from 743 * {@link #getSurfaceTexture()} into the set of target {@link Surface}s 744 * in the next request from the given {@link CaptureCollector}, or drop 745 * the frame if none is available. 746 * 747 * <p> 748 * Any {@link Surface}s targeted must be a subset of the {@link Surface}s 749 * set in the last {@link #configureSurfaces(java.util.Collection)} call. 750 * </p> 751 * 752 * @param targetCollector the surfaces to draw to. 753 */ drawIntoSurfaces(CaptureCollector targetCollector)754 public void drawIntoSurfaces(CaptureCollector targetCollector) { 755 if ((mSurfaces == null || mSurfaces.size() == 0) 756 && (mConversionSurfaces == null || mConversionSurfaces.size() == 0)) { 757 return; 758 } 759 760 boolean doTiming = targetCollector.hasPendingPreviewCaptures(); 761 checkGlError("before updateTexImage"); 762 763 if (doTiming) { 764 beginGlTiming(); 765 } 766 767 mSurfaceTexture.updateTexImage(); 768 769 long timestamp = mSurfaceTexture.getTimestamp(); 770 771 Pair<RequestHolder, Long> captureHolder = targetCollector.previewCaptured(timestamp); 772 773 // No preview request queued, drop frame. 774 if (captureHolder == null) { 775 if (DEBUG) { 776 Log.d(TAG, "Dropping preview frame."); 777 } 778 if (doTiming) { 779 endGlTiming(); 780 } 781 return; 782 } 783 784 RequestHolder request = captureHolder.first; 785 786 Collection<Surface> targetSurfaces = request.getHolderTargets(); 787 if (doTiming) { 788 addGlTimestamp(timestamp); 789 } 790 791 List<Long> targetSurfaceIds = new ArrayList(); 792 try { 793 targetSurfaceIds = LegacyCameraDevice.getSurfaceIds(targetSurfaces); 794 } catch (LegacyExceptionUtils.BufferQueueAbandonedException e) { 795 Log.w(TAG, "Surface abandoned, dropping frame. ", e); 796 request.setOutputAbandoned(); 797 } 798 799 for (EGLSurfaceHolder holder : mSurfaces) { 800 if (LegacyCameraDevice.containsSurfaceId(holder.surface, targetSurfaceIds)) { 801 try{ 802 LegacyCameraDevice.setSurfaceDimens(holder.surface, holder.width, 803 holder.height); 804 makeCurrent(holder.eglSurface); 805 806 LegacyCameraDevice.setNextTimestamp(holder.surface, captureHolder.second); 807 drawFrame(mSurfaceTexture, holder.width, holder.height, 808 (mFacing == CameraCharacteristics.LENS_FACING_FRONT) ? 809 FLIP_TYPE_HORIZONTAL : FLIP_TYPE_NONE); 810 swapBuffers(holder.eglSurface); 811 } catch (LegacyExceptionUtils.BufferQueueAbandonedException e) { 812 Log.w(TAG, "Surface abandoned, dropping frame. ", e); 813 request.setOutputAbandoned(); 814 } 815 } 816 } 817 for (EGLSurfaceHolder holder : mConversionSurfaces) { 818 if (LegacyCameraDevice.containsSurfaceId(holder.surface, targetSurfaceIds)) { 819 // glReadPixels reads from the bottom of the buffer, so add an extra vertical flip 820 try { 821 makeCurrent(holder.eglSurface); 822 drawFrame(mSurfaceTexture, holder.width, holder.height, 823 (mFacing == CameraCharacteristics.LENS_FACING_FRONT) ? 824 FLIP_TYPE_BOTH : FLIP_TYPE_VERTICAL); 825 } catch (LegacyExceptionUtils.BufferQueueAbandonedException e) { 826 // Should never hit this. 827 throw new IllegalStateException("Surface abandoned, skipping drawFrame...", e); 828 } 829 mPBufferPixels.clear(); 830 GLES20.glReadPixels(/*x*/ 0, /*y*/ 0, holder.width, holder.height, 831 GLES20.GL_RGBA, GLES20.GL_UNSIGNED_BYTE, mPBufferPixels); 832 checkGlError("glReadPixels"); 833 834 try { 835 int format = LegacyCameraDevice.detectSurfaceType(holder.surface); 836 LegacyCameraDevice.setSurfaceDimens(holder.surface, holder.width, 837 holder.height); 838 LegacyCameraDevice.setNextTimestamp(holder.surface, captureHolder.second); 839 LegacyCameraDevice.produceFrame(holder.surface, mPBufferPixels.array(), 840 holder.width, holder.height, format); 841 } catch (LegacyExceptionUtils.BufferQueueAbandonedException e) { 842 Log.w(TAG, "Surface abandoned, dropping frame. ", e); 843 request.setOutputAbandoned(); 844 } 845 } 846 } 847 targetCollector.previewProduced(); 848 849 if (doTiming) { 850 endGlTiming(); 851 } 852 } 853 854 /** 855 * Clean up the current GL context. 856 */ cleanupEGLContext()857 public void cleanupEGLContext() { 858 releaseEGLContext(); 859 } 860 861 /** 862 * Drop all current GL operations on the floor. 863 */ flush()864 public void flush() { 865 // TODO: implement flush 866 Log.e(TAG, "Flush not yet implemented."); 867 } 868 } 869