1 /*
2  * Copyright (C) 2018 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #include "ImageConsumer.h"
18 #include <gui/BufferQueue.h>
19 #include "Properties.h"
20 #include "SurfaceTexture.h"
21 #include "renderstate/RenderState.h"
22 #include "renderthread/EglManager.h"
23 #include "renderthread/RenderThread.h"
24 #include "renderthread/VulkanManager.h"
25 #include "utils/Color.h"
26 #include <GrAHardwareBufferUtils.h>
27 #include <GrBackendSurface.h>
28 
29 // Macro for including the SurfaceTexture name in log messages
30 #define IMG_LOGE(x, ...) ALOGE("[%s] " x, st.mName.string(), ##__VA_ARGS__)
31 
32 using namespace android::uirenderer::renderthread;
33 
34 namespace android {
35 
onFreeBufferLocked(int slotIndex)36 void ImageConsumer::onFreeBufferLocked(int slotIndex) {
37     // This callback may be invoked on any thread.
38     mImageSlots[slotIndex].clear();
39 }
40 
onAcquireBufferLocked(BufferItem * item)41 void ImageConsumer::onAcquireBufferLocked(BufferItem* item) {
42     // If item->mGraphicBuffer is not null, this buffer has not been acquired
43     // before, so any prior SkImage is created with a stale buffer. This resets the stale SkImage.
44     if (item->mGraphicBuffer != nullptr) {
45         mImageSlots[item->mSlot].clear();
46     }
47 }
48 
onReleaseBufferLocked(int buf)49 void ImageConsumer::onReleaseBufferLocked(int buf) {
50     mImageSlots[buf].eglFence() = EGL_NO_SYNC_KHR;
51 }
52 
53 /**
54  * AutoBackendTextureRelease manages EglImage/VkImage lifetime. It is a ref-counted object
55  * that keeps GPU resources alive until the last SKImage object using them is destroyed.
56  */
57 class AutoBackendTextureRelease {
58 public:
59     static void releaseProc(SkImage::ReleaseContext releaseContext);
60 
61     AutoBackendTextureRelease(GrContext* context, GraphicBuffer* buffer);
62 
getTexture() const63     const GrBackendTexture& getTexture() const { return mBackendTexture; }
64 
ref()65     void ref() { mUsageCount++; }
66 
67     void unref(bool releaseImage);
68 
getImage()69     inline sk_sp<SkImage> getImage() { return mImage; }
70 
71     void makeImage(sp<GraphicBuffer>& graphicBuffer, android_dataspace dataspace,
72                    GrContext* context);
73 
74     void newBufferContent(GrContext* context);
75 
76 private:
77     // The only way to invoke dtor is with unref, when mUsageCount is 0.
~AutoBackendTextureRelease()78     ~AutoBackendTextureRelease() {}
79 
80     GrBackendTexture mBackendTexture;
81     GrAHardwareBufferUtils::DeleteImageProc mDeleteProc;
82     GrAHardwareBufferUtils::UpdateImageProc mUpdateProc;
83     GrAHardwareBufferUtils::TexImageCtx mImageCtx;
84 
85     // Starting with refcount 1, because the first ref is held by SurfaceTexture. Additional refs
86     // are held by SkImages.
87     int mUsageCount = 1;
88 
89     // mImage is the SkImage created from mBackendTexture.
90     sk_sp<SkImage> mImage;
91 };
92 
AutoBackendTextureRelease(GrContext * context,GraphicBuffer * buffer)93 AutoBackendTextureRelease::AutoBackendTextureRelease(GrContext* context, GraphicBuffer* buffer) {
94     bool createProtectedImage =
95         0 != (buffer->getUsage() & GraphicBuffer::USAGE_PROTECTED);
96     GrBackendFormat backendFormat = GrAHardwareBufferUtils::GetBackendFormat(
97         context,
98         reinterpret_cast<AHardwareBuffer*>(buffer),
99         buffer->getPixelFormat(),
100         false);
101     mBackendTexture = GrAHardwareBufferUtils::MakeBackendTexture(
102         context,
103         reinterpret_cast<AHardwareBuffer*>(buffer),
104         buffer->getWidth(),
105         buffer->getHeight(),
106         &mDeleteProc,
107         &mUpdateProc,
108         &mImageCtx,
109         createProtectedImage,
110         backendFormat,
111         false);
112 }
113 
unref(bool releaseImage)114 void AutoBackendTextureRelease::unref(bool releaseImage) {
115     if (!RenderThread::isCurrent()) {
116         // EGLImage needs to be destroyed on RenderThread to prevent memory leak.
117         // ~SkImage dtor for both pipelines needs to be invoked on RenderThread, because it is not
118         // thread safe.
119         RenderThread::getInstance().queue().post([this, releaseImage]() { unref(releaseImage); });
120         return;
121     }
122 
123     if (releaseImage) {
124         mImage.reset();
125     }
126 
127     mUsageCount--;
128     if (mUsageCount <= 0) {
129         if (mBackendTexture.isValid()) {
130             mDeleteProc(mImageCtx);
131             mBackendTexture = {};
132         }
133         delete this;
134     }
135 }
136 
releaseProc(SkImage::ReleaseContext releaseContext)137 void AutoBackendTextureRelease::releaseProc(SkImage::ReleaseContext releaseContext) {
138     AutoBackendTextureRelease* textureRelease =
139         reinterpret_cast<AutoBackendTextureRelease*>(releaseContext);
140     textureRelease->unref(false);
141 }
142 
makeImage(sp<GraphicBuffer> & graphicBuffer,android_dataspace dataspace,GrContext * context)143 void AutoBackendTextureRelease::makeImage(sp<GraphicBuffer>& graphicBuffer,
144                                           android_dataspace dataspace, GrContext* context) {
145     SkColorType colorType = GrAHardwareBufferUtils::GetSkColorTypeFromBufferFormat(
146         graphicBuffer->getPixelFormat());
147     mImage = SkImage::MakeFromTexture(context,
148         mBackendTexture,
149         kTopLeft_GrSurfaceOrigin,
150         colorType,
151         kPremul_SkAlphaType,
152         uirenderer::DataSpaceToColorSpace(dataspace),
153         releaseProc,
154         this);
155     if (mImage.get()) {
156         // The following ref will be counteracted by releaseProc, when SkImage is discarded.
157         ref();
158     }
159 }
160 
newBufferContent(GrContext * context)161 void AutoBackendTextureRelease::newBufferContent(GrContext* context) {
162     if (mBackendTexture.isValid()) {
163         mUpdateProc(mImageCtx, context);
164     }
165 }
166 
createIfNeeded(sp<GraphicBuffer> graphicBuffer,android_dataspace dataspace,bool forceCreate,GrContext * context)167 void ImageConsumer::ImageSlot::createIfNeeded(sp<GraphicBuffer> graphicBuffer,
168                                               android_dataspace dataspace, bool forceCreate,
169                                               GrContext* context) {
170     if (!mTextureRelease || !mTextureRelease->getImage().get() || dataspace != mDataspace
171             || forceCreate) {
172         if (!graphicBuffer.get()) {
173             clear();
174             return;
175         }
176 
177         if (!mTextureRelease) {
178             mTextureRelease = new AutoBackendTextureRelease(context, graphicBuffer.get());
179         } else {
180             mTextureRelease->newBufferContent(context);
181         }
182 
183         mDataspace = dataspace;
184         mTextureRelease->makeImage(graphicBuffer, dataspace, context);
185     }
186 }
187 
clear()188 void ImageConsumer::ImageSlot::clear() {
189     if (mTextureRelease) {
190         // The following unref counteracts the initial mUsageCount of 1, set by default initializer.
191         mTextureRelease->unref(true);
192         mTextureRelease = nullptr;
193     }
194 }
195 
getImage()196 sk_sp<SkImage> ImageConsumer::ImageSlot::getImage() {
197     return mTextureRelease ? mTextureRelease->getImage() : nullptr;
198 }
199 
dequeueImage(bool * queueEmpty,SurfaceTexture & st,uirenderer::RenderState & renderState)200 sk_sp<SkImage> ImageConsumer::dequeueImage(bool* queueEmpty, SurfaceTexture& st,
201                                            uirenderer::RenderState& renderState) {
202     BufferItem item;
203     status_t err;
204     err = st.acquireBufferLocked(&item, 0);
205     if (err != OK) {
206         if (err != BufferQueue::NO_BUFFER_AVAILABLE) {
207             IMG_LOGE("Error acquiring buffer: %s (%d)", strerror(err), err);
208         } else {
209             int slot = st.mCurrentTexture;
210             if (slot != BufferItem::INVALID_BUFFER_SLOT) {
211                 *queueEmpty = true;
212                 mImageSlots[slot].createIfNeeded(st.mSlots[slot].mGraphicBuffer,
213                         st.mCurrentDataSpace, false, renderState.getRenderThread().getGrContext());
214                 return mImageSlots[slot].getImage();
215             }
216         }
217         return nullptr;
218     }
219 
220     int slot = item.mSlot;
221     if (item.mFence->isValid()) {
222         // Wait on the producer fence for the buffer to be ready.
223         if (uirenderer::Properties::getRenderPipelineType() ==
224             uirenderer::RenderPipelineType::SkiaGL) {
225             err = renderState.getRenderThread().eglManager().fenceWait(item.mFence);
226         } else {
227             err = renderState.getRenderThread().vulkanManager().fenceWait(
228                     item.mFence, renderState.getRenderThread().getGrContext());
229         }
230         if (err != OK) {
231             st.releaseBufferLocked(slot, st.mSlots[slot].mGraphicBuffer, EGL_NO_DISPLAY,
232                                    EGL_NO_SYNC_KHR);
233             return nullptr;
234         }
235     }
236 
237     // Release old buffer.
238     if (st.mCurrentTexture != BufferItem::INVALID_BUFFER_SLOT) {
239         // If needed, set the released slot's fence to guard against a producer accessing the
240         // buffer before the outstanding accesses have completed.
241         sp<Fence> releaseFence;
242         EGLDisplay display = EGL_NO_DISPLAY;
243         if (uirenderer::Properties::getRenderPipelineType() ==
244             uirenderer::RenderPipelineType::SkiaGL) {
245             auto& eglManager = renderState.getRenderThread().eglManager();
246             display = eglManager.eglDisplay();
247             err = eglManager.createReleaseFence(st.mUseFenceSync, &mImageSlots[slot].eglFence(),
248                                                 releaseFence);
249         } else {
250             err = renderState.getRenderThread().vulkanManager().createReleaseFence(
251                     releaseFence, renderState.getRenderThread().getGrContext());
252         }
253         if (OK != err) {
254             st.releaseBufferLocked(slot, st.mSlots[slot].mGraphicBuffer, EGL_NO_DISPLAY,
255                                    EGL_NO_SYNC_KHR);
256             return nullptr;
257         }
258 
259         if (releaseFence.get()) {
260             status_t err = st.addReleaseFenceLocked(
261                     st.mCurrentTexture, st.mSlots[st.mCurrentTexture].mGraphicBuffer, releaseFence);
262             if (err != OK) {
263                 IMG_LOGE("dequeueImage: error adding release fence: %s (%d)", strerror(-err), err);
264                 st.releaseBufferLocked(slot, st.mSlots[slot].mGraphicBuffer, EGL_NO_DISPLAY,
265                                        EGL_NO_SYNC_KHR);
266                 return nullptr;
267             }
268         }
269 
270         // Finally release the old buffer.
271         status_t status = st.releaseBufferLocked(
272                 st.mCurrentTexture, st.mSlots[st.mCurrentTexture].mGraphicBuffer, display,
273                 mImageSlots[st.mCurrentTexture].eglFence());
274         if (status < NO_ERROR) {
275             IMG_LOGE("dequeueImage: failed to release buffer: %s (%d)", strerror(-status), status);
276             err = status;
277             // Keep going, with error raised.
278         }
279     }
280 
281     // Update the state.
282     st.mCurrentTexture = slot;
283     st.mCurrentCrop = item.mCrop;
284     st.mCurrentTransform = item.mTransform;
285     st.mCurrentScalingMode = item.mScalingMode;
286     st.mCurrentTimestamp = item.mTimestamp;
287     st.mCurrentDataSpace = item.mDataSpace;
288     st.mCurrentFence = item.mFence;
289     st.mCurrentFenceTime = item.mFenceTime;
290     st.mCurrentFrameNumber = item.mFrameNumber;
291     st.computeCurrentTransformMatrixLocked();
292 
293     *queueEmpty = false;
294     mImageSlots[slot].createIfNeeded(st.mSlots[slot].mGraphicBuffer, item.mDataSpace, true,
295         renderState.getRenderThread().getGrContext());
296     return mImageSlots[slot].getImage();
297 }
298 
299 } /* namespace android */
300