1 /*
2  * Copyright (C) 2011 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 package com.android.camera.panorama;
18 
19 import android.util.Log;
20 
21 /**
22  * Class to handle the processing of each frame by Mosaicer.
23  */
24 public class MosaicFrameProcessor {
25     private static final boolean LOGV = true;
26     private static final String TAG = "MosaicFrameProcessor";
27     private static final int NUM_FRAMES_IN_BUFFER = 2;
28     private static final int MAX_NUMBER_OF_FRAMES = 100;
29     private static final int MOSAIC_RET_CODE_INDEX = 10;
30     private static final int FRAME_COUNT_INDEX = 9;
31     private static final int X_COORD_INDEX = 2;
32     private static final int Y_COORD_INDEX = 5;
33     private static final int HR_TO_LR_DOWNSAMPLE_FACTOR = 4;
34     private static final int WINDOW_SIZE = 3;
35 
36     private Mosaic mMosaicer;
37     private boolean mIsMosaicMemoryAllocated = false;
38     private final long [] mFrameTimestamp = new long[NUM_FRAMES_IN_BUFFER];
39     private float mTranslationLastX;
40     private float mTranslationLastY;
41 
42     private int mFillIn = 0;
43     private int mTotalFrameCount = 0;
44     private long mLastProcessedFrameTimestamp = 0;
45     private int mLastProcessFrameIdx = -1;
46     private int mCurrProcessFrameIdx = -1;
47 
48     // Panning rate is in unit of percentage of image content translation / second.
49     // Use the moving average to calculate the panning rate.
50     private float mPanningRateX;
51     private float mPanningRateY;
52 
53     private float[] mDeltaX = new float[WINDOW_SIZE];
54     private float[] mDeltaY = new float[WINDOW_SIZE];
55     private float[] mDeltaTime = new float[WINDOW_SIZE];
56     private int mOldestIdx = 0;
57     private float mTotalTranslationX = 0f;
58     private float mTotalTranslationY = 0f;
59     private float mTotalDeltaTime = 0f;
60 
61     private ProgressListener mProgressListener;
62 
63     private int mPreviewWidth;
64     private int mPreviewHeight;
65     private int mPreviewBufferSize;
66 
67     public interface ProgressListener {
onProgress(boolean isFinished, float panningRateX, float panningRateY, float progressX, float progressY)68         public void onProgress(boolean isFinished, float panningRateX, float panningRateY,
69                 float progressX, float progressY);
70     }
71 
MosaicFrameProcessor(int previewWidth, int previewHeight, int bufSize)72     public MosaicFrameProcessor(int previewWidth, int previewHeight, int bufSize) {
73         mMosaicer = new Mosaic();
74         mPreviewWidth = previewWidth;
75         mPreviewHeight = previewHeight;
76         mPreviewBufferSize = bufSize;
77     }
78 
setProgressListener(ProgressListener listener)79     public void setProgressListener(ProgressListener listener) {
80         mProgressListener = listener;
81     }
82 
reportProgress(boolean hires, boolean cancel)83     public int reportProgress(boolean hires, boolean cancel) {
84         return mMosaicer.reportProgress(hires, cancel);
85     }
86 
initialize()87     public void initialize() {
88         setupMosaicer(mPreviewWidth, mPreviewHeight, mPreviewBufferSize);
89         setStripType(Mosaic.STRIPTYPE_WIDE);
90         reset();
91     }
92 
clear()93     public void clear() {
94         if (mIsMosaicMemoryAllocated) {
95             mIsMosaicMemoryAllocated = false;
96             mMosaicer.freeMosaicMemory();
97         }
98     }
99 
setStripType(int type)100     public void setStripType(int type) {
101         mMosaicer.setStripType(type);
102     }
103 
setupMosaicer(int previewWidth, int previewHeight, int bufSize)104     private void setupMosaicer(int previewWidth, int previewHeight, int bufSize) {
105         Log.v(TAG, "setupMosaicer w, h=" + previewWidth + ',' + previewHeight + ',' + bufSize);
106         mMosaicer.allocateMosaicMemory(previewWidth, previewHeight);
107         mIsMosaicMemoryAllocated = true;
108 
109         mFillIn = 0;
110         if  (mMosaicer != null) {
111             mMosaicer.reset();
112         }
113     }
114 
reset()115     public void reset() {
116         // reset() can be called even if MosaicFrameProcessor is not initialized.
117         // Only counters will be changed.
118         mTotalFrameCount = 0;
119         mFillIn = 0;
120         mLastProcessedFrameTimestamp = 0;
121         mTotalTranslationX = 0;
122         mTranslationLastX = 0;
123         mTotalTranslationY = 0;
124         mTranslationLastY = 0;
125         mTotalDeltaTime = 0;
126         mPanningRateX = 0;
127         mPanningRateY = 0;
128         mLastProcessFrameIdx = -1;
129         mCurrProcessFrameIdx = -1;
130         for (int i = 0; i < WINDOW_SIZE; ++i) {
131             mDeltaX[i] = 0f;
132             mDeltaY[i] = 0f;
133             mDeltaTime[i] = 0f;
134         }
135         mMosaicer.reset();
136     }
137 
createMosaic(boolean highRes)138     public int createMosaic(boolean highRes) {
139         return mMosaicer.createMosaic(highRes);
140     }
141 
getFinalMosaicNV21()142     public byte[] getFinalMosaicNV21() {
143         return mMosaicer.getFinalMosaicNV21();
144     }
145 
146     // Processes the last filled image frame through the mosaicer and
147     // updates the UI to show progress.
148     // When done, processes and displays the final mosaic.
processFrame()149     public void processFrame() {
150         if (!mIsMosaicMemoryAllocated) {
151             // clear() is called and buffers are cleared, stop computation.
152             // This can happen when the onPause() is called in the activity, but still some frames
153             // are not processed yet and thus the callback may be invoked.
154             return;
155         }
156         long t1 = System.currentTimeMillis();
157         mFrameTimestamp[mFillIn] = t1;
158 
159         mCurrProcessFrameIdx = mFillIn;
160         mFillIn = ((mFillIn + 1) % NUM_FRAMES_IN_BUFFER);
161 
162         // Check that we are trying to process a frame different from the
163         // last one processed (useful if this class was running asynchronously)
164         if (mCurrProcessFrameIdx != mLastProcessFrameIdx) {
165             mLastProcessFrameIdx = mCurrProcessFrameIdx;
166 
167             // Access the timestamp associated with it...
168             long timestamp = mFrameTimestamp[mCurrProcessFrameIdx];
169 
170             // TODO: make the termination condition regarding reaching
171             // MAX_NUMBER_OF_FRAMES solely determined in the library.
172             if (mTotalFrameCount < MAX_NUMBER_OF_FRAMES) {
173                 // If we are still collecting new frames for the current mosaic,
174                 // process the new frame.
175                 calculateTranslationRate(timestamp);
176 
177                 // Publish progress of the ongoing processing
178                 if (mProgressListener != null) {
179                     mProgressListener.onProgress(false, mPanningRateX, mPanningRateY,
180                             mTranslationLastX * HR_TO_LR_DOWNSAMPLE_FACTOR / mPreviewWidth,
181                             mTranslationLastY * HR_TO_LR_DOWNSAMPLE_FACTOR / mPreviewHeight);
182                 }
183             } else {
184                 if (mProgressListener != null) {
185                     mProgressListener.onProgress(true, mPanningRateX, mPanningRateY,
186                             mTranslationLastX * HR_TO_LR_DOWNSAMPLE_FACTOR / mPreviewWidth,
187                             mTranslationLastY * HR_TO_LR_DOWNSAMPLE_FACTOR / mPreviewHeight);
188                 }
189             }
190         }
191     }
192 
calculateTranslationRate(long now)193     public void calculateTranslationRate(long now) {
194         float[] frameData = mMosaicer.setSourceImageFromGPU();
195         int ret_code = (int) frameData[MOSAIC_RET_CODE_INDEX];
196         mTotalFrameCount  = (int) frameData[FRAME_COUNT_INDEX];
197         float translationCurrX = frameData[X_COORD_INDEX];
198         float translationCurrY = frameData[Y_COORD_INDEX];
199 
200         if (mLastProcessedFrameTimestamp == 0f) {
201             // First time: no need to update delta values.
202             mTranslationLastX = translationCurrX;
203             mTranslationLastY = translationCurrY;
204             mLastProcessedFrameTimestamp = now;
205             return;
206         }
207 
208         // Moving average: remove the oldest translation/deltaTime and
209         // add the newest translation/deltaTime in
210         int idx = mOldestIdx;
211         mTotalTranslationX -= mDeltaX[idx];
212         mTotalTranslationY -= mDeltaY[idx];
213         mTotalDeltaTime -= mDeltaTime[idx];
214         mDeltaX[idx] = Math.abs(translationCurrX - mTranslationLastX);
215         mDeltaY[idx] = Math.abs(translationCurrY - mTranslationLastY);
216         mDeltaTime[idx] = (now - mLastProcessedFrameTimestamp) / 1000.0f;
217         mTotalTranslationX += mDeltaX[idx];
218         mTotalTranslationY += mDeltaY[idx];
219         mTotalDeltaTime += mDeltaTime[idx];
220 
221         // The panning rate is measured as the rate of the translation percentage in
222         // image width/height. Take the horizontal panning rate for example, the image width
223         // used in finding the translation is (PreviewWidth / HR_TO_LR_DOWNSAMPLE_FACTOR).
224         // To get the horizontal translation percentage, the horizontal translation,
225         // (translationCurrX - mTranslationLastX), is divided by the
226         // image width. We then get the rate by dividing the translation percentage with deltaTime.
227         mPanningRateX = mTotalTranslationX /
228                 (mPreviewWidth / HR_TO_LR_DOWNSAMPLE_FACTOR) / mTotalDeltaTime;
229         mPanningRateY = mTotalTranslationY /
230                 (mPreviewHeight / HR_TO_LR_DOWNSAMPLE_FACTOR) / mTotalDeltaTime;
231 
232         mTranslationLastX = translationCurrX;
233         mTranslationLastY = translationCurrY;
234         mLastProcessedFrameTimestamp = now;
235         mOldestIdx = (mOldestIdx + 1) % WINDOW_SIZE;
236     }
237 }
238