1 /* 2 * Copyright (C) 2011 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package com.android.gl2cameraeye; 18 19 import java.io.IOException; 20 import java.nio.ByteBuffer; 21 import java.nio.ByteOrder; 22 import java.nio.FloatBuffer; 23 24 import javax.microedition.khronos.egl.EGLConfig; 25 import javax.microedition.khronos.opengles.GL10; 26 27 import android.app.Activity; 28 import android.content.pm.ActivityInfo; 29 import android.os.Bundle; 30 import android.view.MotionEvent; 31 import android.content.Context; 32 import android.util.Log; 33 34 import android.opengl.GLES20; 35 import android.opengl.GLSurfaceView; 36 import android.opengl.GLUtils; 37 import android.opengl.Matrix; 38 39 import android.graphics.SurfaceTexture; 40 41 import android.hardware.Camera; 42 import android.hardware.SensorManager; 43 import android.hardware.SensorEvent; 44 import android.hardware.SensorEventListener; 45 import android.hardware.Sensor; 46 47 public class GL2CameraEye extends Activity { 48 @Override onCreate(Bundle savedInstanceState)49 protected void onCreate(Bundle savedInstanceState) { 50 super.onCreate(savedInstanceState); 51 mGLView = new CamGLSurfaceView(this); 52 setContentView(mGLView); 53 setRequestedOrientation (ActivityInfo.SCREEN_ORIENTATION_LANDSCAPE); 54 } 55 56 @Override onPause()57 protected void onPause() { 58 super.onPause(); 59 mGLView.onPause(); 60 } 61 62 @Override onResume()63 protected void onResume() { 64 super.onResume(); 65 mGLView.onResume(); 66 } 67 68 private GLSurfaceView mGLView; 69 } 70 71 class CamGLSurfaceView extends GLSurfaceView implements SensorEventListener { CamGLSurfaceView(Context context)72 public CamGLSurfaceView(Context context) { 73 super(context); 74 setEGLContextClientVersion(2); 75 mRenderer = new CamRenderer(context); 76 setRenderer(mRenderer); 77 78 mSensorManager = (SensorManager)context.getSystemService(Context.SENSOR_SERVICE); 79 mAcceleration = mSensorManager.getDefaultSensor(Sensor.TYPE_LINEAR_ACCELERATION); 80 } 81 onTouchEvent(final MotionEvent event)82 public boolean onTouchEvent(final MotionEvent event) { 83 queueEvent(new Runnable(){ 84 public void run() { 85 mRenderer.setPosition(event.getX() / getWidth(), 86 event.getY() / getHeight()); 87 }}); 88 return true; 89 } 90 91 @Override onPause()92 public void onPause() { 93 super.onPause(); 94 mCamera.stopPreview(); 95 mCamera.release(); 96 97 mSensorManager.unregisterListener(this); 98 } 99 100 @Override onResume()101 public void onResume() { 102 mCamera = Camera.open(); 103 Camera.Parameters p = mCamera.getParameters(); 104 // No changes to default camera parameters 105 mCamera.setParameters(p); 106 107 queueEvent(new Runnable(){ 108 public void run() { 109 mRenderer.setCamera(mCamera); 110 }}); 111 112 mSensorManager.registerListener(this, mAcceleration, SensorManager.SENSOR_DELAY_GAME); 113 super.onResume(); 114 } 115 onSensorChanged(SensorEvent event)116 public void onSensorChanged(SensorEvent event) { 117 if (event.sensor.getType() == Sensor.TYPE_LINEAR_ACCELERATION) { 118 final float[] accelerationVector = event.values; 119 queueEvent(new Runnable(){ 120 public void run() { 121 mRenderer.setAcceleration(accelerationVector); 122 }}); 123 } 124 } 125 onAccuracyChanged(Sensor sensor, int accuracy)126 public void onAccuracyChanged(Sensor sensor, int accuracy) { 127 // Ignoring sensor accuracy changes. 128 } 129 130 CamRenderer mRenderer; 131 Camera mCamera; 132 133 SensorManager mSensorManager; 134 Sensor mAcceleration; 135 } 136 137 class CamRenderer implements GLSurfaceView.Renderer, SurfaceTexture.OnFrameAvailableListener { 138 CamRenderer(Context context)139 public CamRenderer(Context context) { 140 mContext = context; 141 142 mTriangleVertices = ByteBuffer.allocateDirect(mTriangleVerticesData.length 143 * FLOAT_SIZE_BYTES).order(ByteOrder.nativeOrder()).asFloatBuffer(); 144 mTriangleVertices.put(mTriangleVerticesData).position(0); 145 146 Matrix.setIdentityM(mSTMatrix, 0); 147 Matrix.setIdentityM(mMMatrix, 0); 148 149 float[] defaultAcceleration = {0.f,0.f,0.f}; 150 setAcceleration(defaultAcceleration); 151 mPos[0] = 0.f; 152 mPos[1] = 0.f; 153 mPos[2] = 0.f; 154 mVel[0] = 0.f; 155 mVel[1] = 0.f; 156 mVel[2] = 0.f; 157 158 } 159 160 /* The following set methods are not synchronized, so should only 161 * be called within the rendering thread context. Use GLSurfaceView.queueEvent for safe access. 162 */ setPosition(float x, float y)163 public void setPosition(float x, float y) { 164 /* Map from screen (0,0)-(1,1) to scene coordinates */ 165 mPos[0] = (x*2-1)*mRatio; 166 mPos[1] = (-y)*2+1; 167 mPos[2] = 0.f; 168 mVel[0] = 0; 169 mVel[1] = 0; 170 mVel[2] = 0; 171 } 172 setCamera(Camera camera)173 public void setCamera(Camera camera) { 174 mCamera = camera; 175 Camera.Size previewSize = camera.getParameters().getPreviewSize(); 176 mCameraRatio = (float)previewSize.width/previewSize.height; 177 } 178 setAcceleration(float[] accelerationVector)179 public void setAcceleration(float[] accelerationVector) { 180 mGForce[0] = accelerationVector[0]; 181 mGForce[1] = accelerationVector[1]; 182 mGForce[2] = accelerationVector[2]; 183 } 184 onDrawFrame(GL10 glUnused)185 public void onDrawFrame(GL10 glUnused) { 186 synchronized(this) { 187 if (updateSurface) { 188 mSurface.updateTexImage(); 189 190 mSurface.getTransformMatrix(mSTMatrix); 191 long timestamp = mSurface.getTimestamp(); 192 doPhysics(timestamp); 193 194 updateSurface = false; 195 } 196 } 197 198 // Ignore the passed-in GL10 interface, and use the GLES20 199 // class's static methods instead. 200 GLES20.glClear( GLES20.GL_DEPTH_BUFFER_BIT | GLES20.GL_COLOR_BUFFER_BIT); 201 GLES20.glUseProgram(mProgram); 202 checkGlError("glUseProgram"); 203 204 GLES20.glActiveTexture(GLES20.GL_TEXTURE0); 205 GLES20.glBindTexture(GL_TEXTURE_EXTERNAL_OES, mTextureID); 206 207 mTriangleVertices.position(TRIANGLE_VERTICES_DATA_POS_OFFSET); 208 GLES20.glVertexAttribPointer(maPositionHandle, 3, GLES20.GL_FLOAT, false, 209 TRIANGLE_VERTICES_DATA_STRIDE_BYTES, mTriangleVertices); 210 checkGlError("glVertexAttribPointer maPosition"); 211 GLES20.glEnableVertexAttribArray(maPositionHandle); 212 checkGlError("glEnableVertexAttribArray maPositionHandle"); 213 214 mTriangleVertices.position(TRIANGLE_VERTICES_DATA_UV_OFFSET); 215 GLES20.glVertexAttribPointer(maTextureHandle, 3, GLES20.GL_FLOAT, false, 216 TRIANGLE_VERTICES_DATA_STRIDE_BYTES, mTriangleVertices); 217 checkGlError("glVertexAttribPointer maTextureHandle"); 218 GLES20.glEnableVertexAttribArray(maTextureHandle); 219 checkGlError("glEnableVertexAttribArray maTextureHandle"); 220 221 Matrix.multiplyMM(mMVPMatrix, 0, mVMatrix, 0, mMMatrix, 0); 222 Matrix.multiplyMM(mMVPMatrix, 0, mProjMatrix, 0, mMVPMatrix, 0); 223 224 GLES20.glUniformMatrix4fv(muMVPMatrixHandle, 1, false, mMVPMatrix, 0); 225 GLES20.glUniformMatrix4fv(muSTMatrixHandle, 1, false, mSTMatrix, 0); 226 GLES20.glUniform1f(muCRatioHandle, mCameraRatio); 227 228 GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4); 229 checkGlError("glDrawArrays"); 230 } 231 onSurfaceChanged(GL10 glUnused, int width, int height)232 public void onSurfaceChanged(GL10 glUnused, int width, int height) { 233 // Ignore the passed-in GL10 interface, and use the GLES20 234 // class's static methods instead. 235 GLES20.glViewport(0, 0, width, height); 236 mRatio = (float) width / height; 237 Matrix.frustumM(mProjMatrix, 0, -mRatio, mRatio, -1, 1, 3, 7); 238 } 239 onSurfaceCreated(GL10 glUnused, EGLConfig config)240 public void onSurfaceCreated(GL10 glUnused, EGLConfig config) { 241 // Ignore the passed-in GL10 interface, and use the GLES20 242 // class's static methods instead. 243 244 /* Set up alpha blending and an Android background color */ 245 GLES20.glEnable(GLES20.GL_BLEND); 246 GLES20.glBlendFunc(GLES20.GL_SRC_ALPHA, GLES20.GL_ONE_MINUS_SRC_ALPHA); 247 GLES20.glClearColor(0.643f, 0.776f, 0.223f, 1.0f); 248 249 /* Set up shaders and handles to their variables */ 250 mProgram = createProgram(mVertexShader, mFragmentShader); 251 if (mProgram == 0) { 252 return; 253 } 254 maPositionHandle = GLES20.glGetAttribLocation(mProgram, "aPosition"); 255 checkGlError("glGetAttribLocation aPosition"); 256 if (maPositionHandle == -1) { 257 throw new RuntimeException("Could not get attrib location for aPosition"); 258 } 259 maTextureHandle = GLES20.glGetAttribLocation(mProgram, "aTextureCoord"); 260 checkGlError("glGetAttribLocation aTextureCoord"); 261 if (maTextureHandle == -1) { 262 throw new RuntimeException("Could not get attrib location for aTextureCoord"); 263 } 264 265 muMVPMatrixHandle = GLES20.glGetUniformLocation(mProgram, "uMVPMatrix"); 266 checkGlError("glGetUniformLocation uMVPMatrix"); 267 if (muMVPMatrixHandle == -1) { 268 throw new RuntimeException("Could not get attrib location for uMVPMatrix"); 269 } 270 271 muSTMatrixHandle = GLES20.glGetUniformLocation(mProgram, "uSTMatrix"); 272 checkGlError("glGetUniformLocation uSTMatrix"); 273 if (muMVPMatrixHandle == -1) { 274 throw new RuntimeException("Could not get attrib location for uSTMatrix"); 275 } 276 277 muCRatioHandle = GLES20.glGetUniformLocation(mProgram, "uCRatio"); 278 checkGlError("glGetUniformLocation uCRatio"); 279 if (muMVPMatrixHandle == -1) { 280 throw new RuntimeException("Could not get attrib location for uCRatio"); 281 } 282 283 /* 284 * Create our texture. This has to be done each time the 285 * surface is created. 286 */ 287 288 int[] textures = new int[1]; 289 GLES20.glGenTextures(1, textures, 0); 290 291 mTextureID = textures[0]; 292 GLES20.glBindTexture(GL_TEXTURE_EXTERNAL_OES, mTextureID); 293 checkGlError("glBindTexture mTextureID"); 294 295 // Can't do mipmapping with camera source 296 GLES20.glTexParameterf(GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_MIN_FILTER, 297 GLES20.GL_NEAREST); 298 GLES20.glTexParameterf(GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_MAG_FILTER, 299 GLES20.GL_LINEAR); 300 // Clamp to edge is the only option 301 GLES20.glTexParameteri(GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_WRAP_S, 302 GLES20.GL_CLAMP_TO_EDGE); 303 GLES20.glTexParameteri(GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_WRAP_T, 304 GLES20.GL_CLAMP_TO_EDGE); 305 checkGlError("glTexParameteri mTextureID"); 306 307 /* 308 * Create the SurfaceTexture that will feed this textureID, and pass it to the camera 309 */ 310 311 mSurface = new SurfaceTexture(mTextureID); 312 mSurface.setOnFrameAvailableListener(this); 313 try { 314 mCamera.setPreviewTexture(mSurface); 315 } catch (IOException t) { 316 Log.e(TAG, "Cannot set preview texture target!"); 317 } 318 319 /* Start the camera */ 320 mCamera.startPreview(); 321 322 Matrix.setLookAtM(mVMatrix, 0, 0, 0, 5f, 0f, 0f, 0f, 0f, 1.0f, 0.0f); 323 324 mLastTime = 0; 325 326 synchronized(this) { 327 updateSurface = false; 328 } 329 } 330 onFrameAvailable(SurfaceTexture surface)331 synchronized public void onFrameAvailable(SurfaceTexture surface) { 332 /* For simplicity, SurfaceTexture calls here when it has new 333 * data available. Call may come in from some random thread, 334 * so let's be safe and use synchronize. No OpenGL calls can be done here. 335 */ 336 updateSurface = true; 337 } 338 doPhysics(long timestamp)339 private void doPhysics(long timestamp) { 340 /* 341 * Move the camera surface around based on some simple spring physics with drag 342 */ 343 344 if (mLastTime == 0) 345 mLastTime = timestamp; 346 347 float deltaT = (timestamp - mLastTime)/1000000000.f; // To seconds 348 349 float springStrength = 20.f; 350 float frictionCoeff = 10.f; 351 float mass = 10.f; 352 float gMultiplier = 4.f; 353 /* Only update physics every 30 ms */ 354 if (deltaT > 0.030f) { 355 mLastTime = timestamp; 356 357 float[] totalForce = new float[3]; 358 totalForce[0] = -mPos[0] * springStrength - mVel[0]*frictionCoeff + gMultiplier*mGForce[0]*mass; 359 totalForce[1] = -mPos[1] * springStrength - mVel[1]*frictionCoeff + gMultiplier*mGForce[1]*mass; 360 totalForce[2] = -mPos[2] * springStrength - mVel[2]*frictionCoeff + gMultiplier*mGForce[2]*mass; 361 362 float[] accel = new float[3]; 363 accel[0] = totalForce[0]/mass; 364 accel[1] = totalForce[1]/mass; 365 accel[2] = totalForce[2]/mass; 366 367 /* Not a very accurate integrator */ 368 mVel[0] = mVel[0] + accel[0]*deltaT; 369 mVel[1] = mVel[1] + accel[1]*deltaT; 370 mVel[2] = mVel[2] + accel[2]*deltaT; 371 372 mPos[0] = mPos[0] + mVel[0]*deltaT; 373 mPos[1] = mPos[1] + mVel[1]*deltaT; 374 mPos[2] = mPos[2] + mVel[2]*deltaT; 375 376 Matrix.setIdentityM(mMMatrix, 0); 377 Matrix.translateM(mMMatrix, 0, mPos[0], mPos[1], mPos[2]); 378 } 379 380 } 381 loadShader(int shaderType, String source)382 private int loadShader(int shaderType, String source) { 383 int shader = GLES20.glCreateShader(shaderType); 384 if (shader != 0) { 385 GLES20.glShaderSource(shader, source); 386 GLES20.glCompileShader(shader); 387 int[] compiled = new int[1]; 388 GLES20.glGetShaderiv(shader, GLES20.GL_COMPILE_STATUS, compiled, 0); 389 if (compiled[0] == 0) { 390 Log.e(TAG, "Could not compile shader " + shaderType + ":"); 391 Log.e(TAG, GLES20.glGetShaderInfoLog(shader)); 392 GLES20.glDeleteShader(shader); 393 shader = 0; 394 } 395 } 396 return shader; 397 } 398 createProgram(String vertexSource, String fragmentSource)399 private int createProgram(String vertexSource, String fragmentSource) { 400 int vertexShader = loadShader(GLES20.GL_VERTEX_SHADER, vertexSource); 401 if (vertexShader == 0) { 402 return 0; 403 } 404 int pixelShader = loadShader(GLES20.GL_FRAGMENT_SHADER, fragmentSource); 405 if (pixelShader == 0) { 406 return 0; 407 } 408 409 int program = GLES20.glCreateProgram(); 410 if (program != 0) { 411 GLES20.glAttachShader(program, vertexShader); 412 checkGlError("glAttachShader"); 413 GLES20.glAttachShader(program, pixelShader); 414 checkGlError("glAttachShader"); 415 GLES20.glLinkProgram(program); 416 int[] linkStatus = new int[1]; 417 GLES20.glGetProgramiv(program, GLES20.GL_LINK_STATUS, linkStatus, 0); 418 if (linkStatus[0] != GLES20.GL_TRUE) { 419 Log.e(TAG, "Could not link program: "); 420 Log.e(TAG, GLES20.glGetProgramInfoLog(program)); 421 GLES20.glDeleteProgram(program); 422 program = 0; 423 } 424 } 425 return program; 426 } 427 checkGlError(String op)428 private void checkGlError(String op) { 429 int error; 430 while ((error = GLES20.glGetError()) != GLES20.GL_NO_ERROR) { 431 Log.e(TAG, op + ": glError " + error); 432 throw new RuntimeException(op + ": glError " + error); 433 } 434 } 435 436 private static final int FLOAT_SIZE_BYTES = 4; 437 private static final int TRIANGLE_VERTICES_DATA_STRIDE_BYTES = 5 * FLOAT_SIZE_BYTES; 438 private static final int TRIANGLE_VERTICES_DATA_POS_OFFSET = 0; 439 private static final int TRIANGLE_VERTICES_DATA_UV_OFFSET = 3; 440 private final float[] mTriangleVerticesData = { 441 // X, Y, Z, U, V 442 -1.0f, -1.0f, 0, 0.f, 0.f, 443 1.0f, -1.0f, 0, 1.f, 0.f, 444 -1.0f, 1.0f, 0, 0.f, 1.f, 445 1.0f, 1.0f, 0, 1.f, 1.f, 446 }; 447 448 private FloatBuffer mTriangleVertices; 449 450 private final String mVertexShader = 451 "uniform mat4 uMVPMatrix;\n" + 452 "uniform mat4 uSTMatrix;\n" + 453 "uniform float uCRatio;\n" + 454 "attribute vec4 aPosition;\n" + 455 "attribute vec4 aTextureCoord;\n" + 456 "varying vec2 vTextureCoord;\n" + 457 "varying vec2 vTextureNormCoord;\n" + 458 "void main() {\n" + 459 " vec4 scaledPos = aPosition;\n" + 460 " scaledPos.x = scaledPos.x * uCRatio;\n" + 461 " gl_Position = uMVPMatrix * scaledPos;\n" + 462 " vTextureCoord = (uSTMatrix * aTextureCoord).xy;\n" + 463 " vTextureNormCoord = aTextureCoord.xy;\n" + 464 "}\n"; 465 466 private final String mFragmentShader = 467 "#extension GL_OES_EGL_image_external : require\n" + 468 "precision mediump float;\n" + 469 "varying vec2 vTextureCoord;\n" + 470 "varying vec2 vTextureNormCoord;\n" + 471 "uniform samplerExternalOES sTexture;\n" + 472 "void main() {\n" + 473 " gl_FragColor = texture2D(sTexture, vTextureCoord);\n" + 474 " gl_FragColor.a = 1.0-min(length(vTextureNormCoord-0.5)*2.0,1.0);\n" + 475 "}\n"; 476 477 private float[] mMVPMatrix = new float[16]; 478 private float[] mProjMatrix = new float[16]; 479 private float[] mMMatrix = new float[16]; 480 private float[] mVMatrix = new float[16]; 481 private float[] mSTMatrix = new float[16]; 482 483 private int mProgram; 484 private int mTextureID; 485 private int muMVPMatrixHandle; 486 private int muSTMatrixHandle; 487 private int muCRatioHandle; 488 private int maPositionHandle; 489 private int maTextureHandle; 490 491 private float mRatio = 1.0f; 492 private float mCameraRatio = 1.0f; 493 private float[] mVel = new float[3]; 494 private float[] mPos = new float[3]; 495 private float[] mGForce = new float[3]; 496 497 private long mLastTime; 498 499 private SurfaceTexture mSurface; 500 private Camera mCamera; 501 private boolean updateSurface = false; 502 503 private Context mContext; 504 private static String TAG = "CamRenderer"; 505 506 // Magic key 507 private static int GL_TEXTURE_EXTERNAL_OES = 0x8D65; 508 } 509